ftl: extended nightly tests

Change-Id: I8dc3e895f6b360f3ad81f157db502dd58629c5e6
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Signed-off-by: Wojciech Malikowski <wojciech.malikowski@intel.com>
Reviewed-on: https://review.gerrithub.io/c/442878
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

scripts/gen_ftl.sh

@@ -5,26 +5,18 @@ set -e
 rootdir=$(readlink -f $(dirname $0))/..
 
 function usage {
-	echo "Replaces FTL_* variables in config files inside the config/ directory."
-	echo "The following varaibles are replaced:"
-	echo "- FTL_CONF_DIR - config directory"
-	echo "- FTL_TRANSPORT_ADDR - SSD's PCIe address (defaults to first lnvm device)"
-	echo "- FTL_BDEV_NAME - name of the bdev"
-	echo "- FTL_BDEV_PUNITS - bdev's parallel unit range (e.g. 0-3)"
-	echo "- FTL_BDEV_UUID - bdev's uuid (used when in restore mode)"
-	echo
 	echo "Usage: $0 -a TRANSPORT_ADDR -n BDEV_NAME -l PUNITS [-u UUID]"
 	echo "UUID is required when restoring device state"
+	echo
+	echo "TRANSPORT_ADDR - SSD's PCIe address"
+	echo "BDEV_NAME - name of the bdev"
+	echo "PUNITS - bdev's parallel unit range (e.g. 0-3)"
+	echo "UUID - bdev's uuid (used when in restore mode)"
 }
 
 function generate_config {
-	fname=$1
+	echo "[Ftl]"
-	output=${1%.in}
+	echo "  TransportID \"trtype:PCIe traddr:$1\" $2 $3 $4"
-
-	cp $fname $output
-	for var in ${!vmap[@]}; do
-		sed -i "s,$var,${vmap[$var]},g" $output
-	done
 }
 
 while getopts ":a:n:l:m:u:" arg; do
@@ -45,13 +37,4 @@ if [[ -z "$addr" || -z "$name" || -z "$punits" ]]; then
 	exit 1
 fi
 
-declare -A vmap
+generate_config $addr $name $punits $uuid
-vmap[FTL_CONF_DIR]=$rootdir/test/ftl/config
-vmap[FTL_TRANSPORT_ADDR]=$addr
-vmap[FTL_BDEV_NAME]=$name
-vmap[FTL_BDEV_PUNITS]=$punits
-vmap[FTL_BDEV_UUID]=${uuid:-}
-
-for file in $(find $rootdir/test/ftl/config -type f -iname "*.in"); do
-	generate_config $file
-done

test/common/autotest_common.sh

@@ -65,6 +65,7 @@ fi
 : ${SPDK_TEST_FTL=0}; export SPDK_TEST_FTL
 : ${SPDK_TEST_BDEV_FTL=0}; export SPDK_TEST_BDEV_FTL
 : ${SPDK_TEST_OCF=1}; export SPDK_TEST_OCF
+: ${SPDK_TEST_FTL_EXTENDED=0}; export SPDK_TEST_FTL_EXTENDED
 
 if [ -z "$DEPENDENCY_DIR" ]; then
 	export DEPENDENCY_DIR=/home/sys_sgsw

test/ftl/.gitignore

@@ -1 +0,0 @@
-.testfile_*

test/ftl/config/fio/drive-prep.fio

@@ -0,0 +1,15 @@
+[drive_prep]
+ioengine=spdk_bdev
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
+thread=1
+
+direct=1
+buffered=0
+size=100%
+randrepeat=0
+norandommap
+bs=4k
+iodepth=128
+numjobs=1
+rw=write

test/ftl/config/fio/randr.fio

@@ -0,0 +1,19 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
+direct=1
+thread=1
+buffered=0
+size=100%
+randrepeat=0
+time_based
+norandommap
+
+[test]
+stonewall
+bs=4k
+numjobs=4
+rw=randread
+iodepth=128
+runtime=1200

test/ftl/config/fio/randrw.fio

@@ -0,0 +1,20 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
+direct=1
+thread=1
+buffered=0
+size=100%
+randrepeat=0
+time_based
+norandommap
+
+[test]
+stonewall
+bs=4k
+numjobs=4
+rw=randrw
+rwmixread=70
+iodepth=32
+runtime=1200

test/ftl/config/fio/randw-verify-depth128.fio

@@ -1,6 +1,7 @@
 [global]
 ioengine=spdk_bdev
-spdk_conf=FTL_CONF_DIR/ftl.conf
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
 thread=1
 direct=1
 iodepth=128
@@ -11,7 +12,6 @@ verify_dump=0
 verify_state_save=0
 verify_fatal=1
 bs=4k
-filename=FTL_BDEV_NAME
 random_distribution=normal
 serialize_overlap=1
 io_size=256M

test/ftl/config/fio/randw-verify-j2.fio

@@ -1,6 +1,7 @@
 [global]
 ioengine=spdk_bdev
-spdk_conf=FTL_CONF_DIR/ftl.conf
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
 thread=1
 direct=1
 iodepth=128
@@ -12,7 +13,6 @@ verify_state_save=0
 verify_backlog=5000
 verify_fatal=1
 bs=4k
-filename=FTL_BDEV_NAME
 random_distribution=normal
 serialize_overlap=1
 io_size=256M

test/ftl/config/fio/randw-verify-qd128-ext.fio

@@ -0,0 +1,20 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
+thread=1
+direct=1
+iodepth=128
+rw=randwrite
+verify=crc32c
+do_verify=1
+verify_dump=0
+verify_state_save=0
+verify_fatal=1
+bs=4k
+random_distribution=normal
+serialize_overlap=1
+
+[test]
+io_size=64G
+numjobs=1

test/ftl/config/fio/randw-verify.fio

@@ -1,6 +1,7 @@
 [global]
 ioengine=spdk_bdev
-spdk_conf=FTL_CONF_DIR/ftl.conf
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
 thread=1
 direct=1
 iodepth=1
@@ -13,7 +14,6 @@ verify_state_save=0
 verify_backlog=16
 verify_fatal=1
 bs=68k
-filename=FTL_BDEV_NAME
 random_distribution=normal
 
 [test]

test/ftl/config/fio/randw.fio

@@ -0,0 +1,18 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=${FTL_BDEV_CONF}
+filename=${FTL_BDEV_NAME}
+direct=1
+thread=1
+buffered=0
+size=100%
+randrepeat=0
+time_based
+norandommap
+
+[test]
+bs=4k
+numjobs=1
+rw=randwrite
+iodepth=64
+runtime=1200

test/ftl/config/ftl.conf.in

@@ -1,2 +0,0 @@
-[Ftl]
- TransportID "trtype:PCIe traddr:FTL_TRANSPORT_ADDR" FTL_BDEV_NAME "FTL_BDEV_PUNITS" FTL_BDEV_UUID

test/ftl/fio.sh

@@ -4,19 +4,36 @@ set -e
 
 testdir=$(readlink -f $(dirname $0))
 rootdir=$(readlink -f $testdir/../..)
+plugindir=$rootdir/examples/bdev/fio_plugin
 
 source $rootdir/test/common/autotest_common.sh
 
-tests=(randw-verify randw-verify-j2 randw-verify-depth128)
+declare -A suite
-plugindir=$rootdir/examples/bdev/fio_plugin
+suite['basic']='randw-verify randw-verify-j2 randw-verify-depth128'
+suite['extended']='drive-prep randw-verify-qd128-ext randw randr randrw'
+
 device=$1
+tests=${suite[$2]}
+uuid=$3
 
 if [ ! -d /usr/src/fio ]; then
 	echo "FIO not available"
 	exit 1
 fi
 
-$rootdir/scripts/gen_ftl.sh -a $device -n nvme0 -l 0-3
+if [ -z "$tests" ]; then
+	echo "Invalid test suite '$2'"
+	exit 1
+fi
+
+export FTL_BDEV_CONF=$testdir/config/ftl.conf
+export FTL_BDEV_NAME=nvme0
+
+if [ -z "$uuid" ]; then
+	$rootdir/scripts/gen_ftl.sh -a $device -n nvme0 -l 0-3 > $FTL_BDEV_CONF
+else
+	$rootdir/scripts/gen_ftl.sh -a $device -n nvme0 -l 0-3 -u $uuid > $FTL_BDEV_CONF
+fi
 
 for test in ${tests[@]}; do
 	timing_enter $test

test/ftl/ftl.sh

@@ -4,13 +4,10 @@ set -e
 
 testdir=$(readlink -f $(dirname $0))
 rootdir=$(readlink -f $testdir/../..)
+rpc_py=$rootdir/scripts/rpc.py
 
 source $rootdir/test/common/autotest_common.sh
 
-function ftl_kill() {
-	rm -f $testdir/.testfile_*
-}
-
 vendor_id='0x1d1d'
 device_id='0x1f1f'
 device=$(lspci -d ${vendor_id}:${device_id} | cut -d' ' -f 1)
@@ -20,20 +17,32 @@ if [ -z "$device" ]; then
 	exit 0
 fi
 
-trap "ftl_kill; exit 1" SIGINT SIGTERM EXIT
-
 timing_enter ftl
 timing_enter fio
 
-run_test suite $testdir/fio.sh $device
+run_test suite $testdir/fio.sh $device basic
 
 timing_exit fio
 
 timing_enter restore
-run_test suite $testdir/restore.sh $device $uuid
+run_test suite $testdir/restore.sh $device
 timing_exit restore
 
-timing_exit ftl
+if [ $SPDK_TEST_FTL_EXTENDED -eq 1 ]; then
+	$rootdir/test/app/bdev_svc/bdev_svc &
+	bdev_svc_pid=$!
 
-trap - SIGINT SIGTERM EXIT
+	trap "killprocess $bdev_svc_pid; exit 1" SIGINT SIGTERM EXIT
-ftl_kill
+
+	waitforlisten $bdev_svc_pid
+	uuid=$($rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3 | jq -r '.uuid')
+	killprocess $bdev_svc_pid
+
+	trap - SIGINT SIGTERM EXIT
+
+	timing_enter fio_extended
+	run_test suite $testdir/fio.sh $device extended $uuid
+	timing_exit fio_extended
+fi
+
+timing_exit ftl