bdev_ftl: Added FTL bdev functional tests

This patch introduces functional tests for FTL bdev.

The tests cover various I/O workflows and check data integrity. Several
scripts have been added to test the FTL library:
 * generate_config.sh - prepares configuration scripts for specified
 device
 * restore.sh - tests restoring device's state from the SSD
 * fio.sh - runs tests based on fio and fio_plugin

The tests are run from autotest.sh when the SPDK_TEST_BDEV_FTL flag is
set.

Change-Id: I561d99ed35fe91eadd3756789cc99afe2da8c1db
Signed-off-by: Mateusz Kozlowski <mateusz.kozlowski@intel.com>
Signed-off-by: Wojciech Malikowski <wojciech.malikowski@intel.com>
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-on: https://review.gerrithub.io/c/431330
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>

autotest.sh

@@ -197,6 +197,10 @@ if [ $SPDK_TEST_OCF -eq 1 ]; then
 	run_test suite ./test/ocf/ocf.sh
 fi
 
+if [ $SPDK_TEST_BDEV_FTL -eq 1 ]; then
+	run_test suite ./test/ftl/ftl.sh
+fi
+
 run_test suite ./test/json_config/json_config.sh
 
 timing_enter cleanup

scripts/gen_ftl.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -e
+
+rootdir=$(readlink -f $(dirname $0))/..
+
+function usage {
+	echo "Replaces FTL_* variables in config files inside the config/ directory."
+	echo "The following varaibles are replaced:"
+	echo "- FTL_CONF_DIR - config directory"
+	echo "- FTL_TRANSPORT_ADDR - SSD's PCIe address (defaults to first lnvm device)"
+	echo "- FTL_BDEV_NAME - name of the bdev"
+	echo "- FTL_BDEV_PUNITS - bdev's parallel unit range (e.g. 0-3)"
+	echo "- FTL_BDEV_UUID - bdev's uuid (used when in restore mode)"
+	echo
+	echo "Usage: $0 -a TRANSPORT_ADDR -n BDEV_NAME -l PUNITS [-u UUID]"
+	echo "UUID is required when restoring device state"
+}
+
+function generate_config {
+	fname=$1
+	output=${1%.in}
+
+	cp $fname $output
+	for var in ${!vmap[@]}; do
+		sed -i "s,$var,${vmap[$var]},g" $output
+	done
+}
+
+while getopts ":a:n:l:m:u:" arg; do
+	case "$arg" in
+		a)	addr=$OPTARG	;;
+		n)	name=$OPTARG	;;
+		l)	punits=$OPTARG	;;
+		u)	uuid=$OPTARG	;;
+		h)	usage
+			exit 0		;;
+		*)	usage
+			exit 1		;;
+	esac
+done
+
+if [[ -z "$addr" || -z "$name" || -z "$punits" ]]; then
+	usage
+	exit 1
+fi
+
+declare -A vmap
+vmap[FTL_CONF_DIR]=$rootdir/test/ftl/config
+vmap[FTL_TRANSPORT_ADDR]=$addr
+vmap[FTL_BDEV_NAME]=$name
+vmap[FTL_BDEV_PUNITS]=$punits
+vmap[FTL_BDEV_UUID]=${uuid:-}
+
+for file in $(find $rootdir/test/ftl/config -type f -iname "*.in"); do
+	generate_config $file
+done

test/common/autotest_common.sh

@@ -62,6 +62,7 @@ fi
 : ${SPDK_RUN_INSTALLED_DPDK=1}; export SPDK_RUN_INSTALLED_DPDK
 : ${SPDK_TEST_CRYPTO=1}; export SPDK_TEST_CRYPTO
 : ${SPDK_TEST_FTL=0}; export SPDK_TEST_FTL
+: ${SPDK_TEST_BDEV_FTL=0}; export SPDK_TEST_BDEV_FTL
 : ${SPDK_TEST_OCF=1}; export SPDK_TEST_OCF
 
 if [ -z "$DEPENDENCY_DIR" ]; then
@@ -262,7 +263,7 @@ function timing_finish() {
 }
 
 function create_test_list() {
-	grep -rsh --exclude="autotest_common.sh" --exclude="$rootdir/test/common/autotest_common.sh" -e "report_test_completion" $rootdir | sed 's/report_test_completion//g; s/[[:blank:]]//g; s/"//g;' > $output_dir/all_tests.txt || true
+	grep -rshI --exclude="autotest_common.sh" --exclude="$rootdir/test/common/autotest_common.sh" -e "report_test_completion" $rootdir | sed 's/report_test_completion//g; s/[[:blank:]]//g; s/"//g;' > $output_dir/all_tests.txt || true
 }
 
 function report_test_completion() {

test/ftl/.gitignore

@@ -0,0 +1 @@
+.testfile_*

test/ftl/config/.gitignore

@@ -0,0 +1,2 @@
+ftl.conf
+fio/*.fio

test/ftl/config/fio/randw-verify-depth128.fio.in

@@ -0,0 +1,20 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=FTL_CONF_DIR/ftl.conf
+thread=1
+direct=1
+iodepth=128
+rw=randwrite
+verify=crc32c
+do_verify=1
+verify_dump=0
+verify_state_save=0
+verify_fatal=1
+bs=4k
+filename=FTL_BDEV_NAME
+random_distribution=normal
+serialize_overlap=1
+io_size=5G
+
+[test]
+numjobs=1

test/ftl/config/fio/randw-verify-j2.fio.in

@@ -0,0 +1,25 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=FTL_CONF_DIR/ftl.conf
+thread=1
+direct=1
+iodepth=128
+rw=randwrite
+verify=crc32c
+do_verify=1
+verify_dump=0
+verify_state_save=0
+verify_backlog=5000
+verify_fatal=1
+bs=4k
+filename=FTL_BDEV_NAME
+random_distribution=normal
+serialize_overlap=1
+io_size=1G
+
+[first_half]
+offset=0%
+size=50%
+
+[second_half]
+offset=50%

test/ftl/config/fio/randw-verify.fio.in

@@ -0,0 +1,20 @@
+[global]
+ioengine=spdk_bdev
+spdk_conf=FTL_CONF_DIR/ftl.conf
+thread=1
+direct=1
+iodepth=1
+rw=randwrite
+size=4G
+verify=crc32c
+do_verify=1
+verify_dump=0
+verify_state_save=0
+verify_backlog=16
+verify_fatal=1
+bs=68k
+filename=FTL_BDEV_NAME
+random_distribution=normal
+
+[test]
+numjobs=1

test/ftl/config/ftl.conf.in

@@ -0,0 +1,2 @@
+[Ftl]
+ TransportID "trtype:PCIe traddr:FTL_TRANSPORT_ADDR" FTL_BDEV_NAME "FTL_BDEV_PUNITS" FTL_BDEV_UUID

test/ftl/fio.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+
+source $rootdir/test/common/autotest_common.sh
+
+tests=(randw-verify randw-verify-j2 randw-verify-depth128)
+plugindir=$rootdir/examples/bdev/fio_plugin
+device=$1
+
+if [ ! -d /usr/src/fio ]; then
+	echo "FIO not available"
+	exit 1
+fi
+
+$rootdir/scripts/gen_ftl.sh -a $device -n nvme0 -l 0-3
+
+for test in ${tests[@]}; do
+	timing_enter $test
+	LD_PRELOAD=$plugindir/fio_plugin /usr/src/fio/fio $testdir/config/fio/$test.fio
+	timing_exit $test
+done
+
+report_test_completion ftl_fio

test/ftl/ftl.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -e
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+
+source $rootdir/test/common/autotest_common.sh
+
+function ftl_kill() {
+	rm -f $testdir/.testfile_*
+}
+
+vendor_id='0x1d1d'
+device_id='0x1f1f'
+device=$(lspci -d ${vendor_id}:${device_id} | cut -d' ' -f 1)
+
+if [ -z "$device" ]; then
+	echo "Could not find FTL device. Tests skipped."
+	exit 0
+fi
+
+trap "ftl_kill; exit 1" SIGINT SIGTERM EXIT
+
+timing_enter ftl
+timing_enter fio
+
+run_test suite $testdir/fio.sh $device
+
+timing_exit fio
+
+timing_enter restore
+run_test suite $testdir/restore.sh $device $uuid
+timing_exit restore
+
+timing_exit ftl
+
+trap - SIGINT SIGTERM EXIT
+ftl_kill

test/ftl/restore.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -e
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+rpc_py=$rootdir/scripts/rpc.py
+
+source $rootdir/test/common/autotest_common.sh
+
+mount_dir=$(mktemp -d)
+device=$1
+uuid=$2
+
+restore_kill() {
+	if mount | grep $mount_dir; then
+		umount $mount_dir
+	fi
+	rm -rf $mount_dir
+	rm -f $testdir/testfile.md5
+
+	$rpc_py delete_ftl_bdev -b nvme0
+	killprocess $svcpid
+	rmmod nbd || true
+}
+
+trap "restore_kill; exit 1" SIGINT SIGTERM EXIT
+
+$rootdir/test/app/bdev_svc/bdev_svc --max-delay=0 & svcpid=$!
+# Wait until bdev_svc starts
+waitforlisten $svcpid
+
+if [ -n "$uuid" ]; then
+	$rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3 -u $uuid
+else
+	uuid=$($rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3 | jq -r '.uuid')
+fi
+
+# Load the nbd driver
+modprobe nbd
+$rpc_py start_nbd_disk nvme0 /dev/nbd0
+waitfornbd nbd0
+
+# Prepare the disk by creating ext4 fs and putting a file on it
+mkfs.ext4 -F /dev/nbd0
+mount /dev/nbd0 $mount_dir
+dd if=/dev/urandom of=$mount_dir/testfile bs=4K count=256K
+sync
+mount -o remount /dev/nbd0 $mount_dir
+md5sum $mount_dir/testfile > $testdir/testfile.md5
+
+# Kill bdev service and start it again
+umount $mount_dir
+killprocess $svcpid
+
+$rootdir/test/app/bdev_svc/bdev_svc --max-delay=0 & svcpid=$!
+# Wait until bdev_svc starts
+waitforlisten $svcpid
+$rpc_py construct_ftl_bdev -b nvme0 -a $device -l 0-3 -u $uuid
+
+$rpc_py start_nbd_disk nvme0 /dev/nbd0
+
+mount /dev/nbd0 $mount_dir
+md5sum -c $testdir/testfile.md5
+
+report_test_completion occsd_restore
+
+trap - SIGINT SIGTERM EXIT
+restore_kill