From 25503b963079f2b683a10a1b40b17a538a25b196 Mon Sep 17 00:00:00 2001
From: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Date: Mon, 13 Jun 2022 10:43:37 +0200
Subject: [PATCH] FTL: Add write tests

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I184247316c1b85ddecc8e8b48422de606521dc36
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13323
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
---
 test/ftl/common.sh                            | 48 ++++++++++++
 test/ftl/config/fio/randw-verify-depth128.fio | 20 +++++
 test/ftl/config/fio/randw-verify-j2.fio       | 25 +++++++
 .../config/fio/randw-verify-qd2048-ext.fio    | 24 ++++++
 test/ftl/config/fio/randw-verify.fio          | 20 +++++
 test/ftl/fio.sh                               | 67 +++++++++++++++++
 test/ftl/ftl.sh                               | 75 +++++++++++++++++++
 7 files changed, 279 insertions(+)
 create mode 100644 test/ftl/common.sh
 create mode 100644 test/ftl/config/fio/randw-verify-depth128.fio
 create mode 100644 test/ftl/config/fio/randw-verify-j2.fio
 create mode 100644 test/ftl/config/fio/randw-verify-qd2048-ext.fio
 create mode 100644 test/ftl/config/fio/randw-verify.fio
 create mode 100755 test/ftl/fio.sh
 create mode 100755 test/ftl/ftl.sh

diff --git a/test/ftl/common.sh b/test/ftl/common.sh
new file mode 100644
index 000000000..612e68368
--- /dev/null
+++ b/test/ftl/common.sh
@@ -0,0 +1,48 @@
+# Common utility functions to be sourced by the libftl test scripts
+
+function clear_lvols() {
+	stores=$("$rootdir/scripts/rpc.py" bdev_lvol_get_lvstores | jq -r ".[] | .uuid")
+	for lvs in $stores; do
+		"$rootdir/scripts/rpc.py" bdev_lvol_delete_lvstore -u $lvs
+	done
+}
+
+function create_nv_cache_bdev() {
+	local name=$1
+	local cache_bdf=$2
+	local base_bdev=$3
+
+	# use 5% space of base bdev
+	local size=$(($(get_bdev_size $base_bdev) * 5 / 100))
+
+	# Create NVMe bdev on specified device and split it so that it has the desired size
+	local nvc_bdev
+	nvc_bdev=$($rootdir/scripts/rpc.py bdev_nvme_attach_controller -b $name -t PCIe -a $cache_bdf)
+
+	local nvc_size
+	nvc_size=$(get_bdev_size $nvc_bdev)
+	if [[ $size -gt $nvc_size ]]; then
+		size=nvc_size
+	fi
+	$rootdir/scripts/rpc.py bdev_split_create $nvc_bdev -s $size 1
+}
+
+function create_base_bdev() {
+	local name=$1
+	local base_bdf=$2
+	local size=$3
+
+	# Create NVMe bdev on specified device and split it so that it has the desired size
+	local base_bdev
+	base_bdev=$($rootdir/scripts/rpc.py bdev_nvme_attach_controller -b $name -t PCIe -a $base_bdf)
+
+	local base_size
+	base_size=$(get_bdev_size $base_bdev)
+	if [[ $size -le $base_size ]]; then
+		$rootdir/scripts/rpc.py bdev_split_create $base_bdev -s $size 1
+	else
+		clear_lvols
+		lvs=$($rootdir/scripts/rpc.py bdev_lvol_create_lvstore $base_bdev lvs)
+		$rootdir/scripts/rpc.py bdev_lvol_create ${base_bdev}p0 $size -t -u $lvs
+	fi
+}
diff --git a/test/ftl/config/fio/randw-verify-depth128.fio b/test/ftl/config/fio/randw-verify-depth128.fio
new file mode 100644
index 000000000..a932d1b36
--- /dev/null
+++ b/test/ftl/config/fio/randw-verify-depth128.fio
@@ -0,0 +1,20 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=${FTL_JSON_CONF}
+filename=${FTL_BDEV_NAME}
+thread=1
+direct=1
+iodepth=128
+rw=randwrite
+verify=crc32c
+do_verify=0
+verify_dump=0
+verify_state_save=0
+verify_fatal=1
+bs=4k
+random_distribution=normal
+serialize_overlap=1
+io_size=256M
+
+[test]
+numjobs=1
diff --git a/test/ftl/config/fio/randw-verify-j2.fio b/test/ftl/config/fio/randw-verify-j2.fio
new file mode 100644
index 000000000..fbbf284be
--- /dev/null
+++ b/test/ftl/config/fio/randw-verify-j2.fio
@@ -0,0 +1,25 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=${FTL_JSON_CONF}
+filename=${FTL_BDEV_NAME}
+thread=1
+direct=1
+iodepth=128
+rw=randwrite
+verify=crc32c
+do_verify=0
+verify_dump=0
+verify_state_save=0
+verify_backlog=5000
+verify_fatal=1
+bs=4k
+random_distribution=normal
+serialize_overlap=1
+io_size=256M
+
+[first_half]
+offset=0%
+size=50%
+
+[second_half]
+offset=50%
diff --git a/test/ftl/config/fio/randw-verify-qd2048-ext.fio b/test/ftl/config/fio/randw-verify-qd2048-ext.fio
new file mode 100644
index 000000000..f12e41532
--- /dev/null
+++ b/test/ftl/config/fio/randw-verify-qd2048-ext.fio
@@ -0,0 +1,24 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=${FTL_JSON_CONF}
+filename=${FTL_BDEV_NAME}
+thread=1
+direct=1
+iodepth=2048
+size=100%
+
+[write]
+numjobs=1
+rw=randwrite
+verify=crc32c
+do_verify=0
+verify_dump=0
+verify_state_save=0
+verify_fatal=1
+bs=512k
+serialize_overlap=1
+norandommap
+
+[read]
+rw=randread
+bs=512k
diff --git a/test/ftl/config/fio/randw-verify.fio b/test/ftl/config/fio/randw-verify.fio
new file mode 100644
index 000000000..efb8432a1
--- /dev/null
+++ b/test/ftl/config/fio/randw-verify.fio
@@ -0,0 +1,20 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=${FTL_JSON_CONF}
+filename=${FTL_BDEV_NAME}
+thread=1
+direct=1
+iodepth=1
+rw=randwrite
+size=256M
+verify=crc32c
+do_verify=0
+verify_dump=0
+verify_state_save=0
+verify_backlog=16
+verify_fatal=1
+bs=68k
+random_distribution=normal
+
+[test]
+numjobs=1
diff --git a/test/ftl/fio.sh b/test/ftl/fio.sh
new file mode 100755
index 000000000..5d942d2ac
--- /dev/null
+++ b/test/ftl/fio.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+source $rootdir/test/common/autotest_common.sh
+source $testdir/common.sh
+
+declare -A suite
+suite['basic']='randw-verify randw-verify-j2 randw-verify-depth128'
+suite['extended']='randw-verify-qd2048-ext'
+
+rpc_py=$rootdir/scripts/rpc.py
+
+fio_kill() {
+	killprocess $svcpid
+	rm -f $FTL_JSON_CONF
+}
+
+device=$1
+cache_device=$2
+tests=${suite[$3]}
+uuid=$4
+timeout=240
+
+if [[ $CONFIG_FIO_PLUGIN != y ]]; then
+	echo "FIO not available"
+	exit 1
+fi
+
+if [ -z "$tests" ]; then
+	echo "Invalid test suite '$2'"
+	exit 1
+fi
+
+export FTL_BDEV_NAME=ftl0
+export FTL_JSON_CONF=$testdir/config/ftl.json
+
+trap "fio_kill; exit 1" SIGINT SIGTERM EXIT
+
+"$SPDK_BIN_DIR/spdk_tgt" -m 7 &
+svcpid=$!
+waitforlisten $svcpid
+
+split_bdev=$(create_base_bdev nvme0 $device $((1024 * 101)))
+nv_cache=$(create_nv_cache_bdev nvc0 $cache_device $split_bdev)
+
+$rpc_py -t $timeout bdev_ftl_create -b ftl0 -d $split_bdev -c $nv_cache
+
+waitforbdev ftl0
+
+(
+	echo '{"subsystems": ['
+	$rpc_py save_subsystem_config -n bdev
+	echo ']}'
+	# Temporary hack so tests are always creating new instance, until clean restore is reintroduced later
+) | sed 's/"uuid": "[a-f0-9\-]\{36\}"/"uuid": "00000000-0000-0000-0000-000000000000"/g' > $FTL_JSON_CONF
+
+killprocess $svcpid
+trap - SIGINT SIGTERM EXIT
+
+for test in ${tests}; do
+	timing_enter $test
+	fio_bdev $testdir/config/fio/$test.fio
+	timing_exit $test
+done
+
+rm -f $FTL_JSON_CONF
diff --git a/test/ftl/ftl.sh b/test/ftl/ftl.sh
new file mode 100755
index 000000000..1a5e88895
--- /dev/null
+++ b/test/ftl/ftl.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+source $rootdir/test/common/autotest_common.sh
+source $testdir/common.sh
+
+rpc_py=$rootdir/scripts/rpc.py
+
+function at_ftl_exit() {
+	killprocess "$spdk_tgt_pid"
+
+	# delete any created lvols of the base device
+	if [[ -n $device ]]; then
+		"$rootdir/build/bin/spdk_tgt" &
+		spdk_tgt_pid=$!
+		waitforlisten "$spdk_tgt_pid"
+		"$rpc_py" bdev_nvme_attach_controller -b nvme0 -t PCIe -a $device
+		clear_lvols
+		killprocess "$spdk_tgt_pid"
+	fi
+
+	# restore original driver
+	$rootdir/scripts/setup.sh reset
+}
+
+trap 'at_ftl_exit' SIGINT SIGTERM EXIT
+
+# Bind device to vfio/uio driver before testing
+PCI_ALLOWED="$device" PCI_BLOCKED="" DRIVER_OVERRIDE="" $rootdir/scripts/setup.sh
+
+"$rootdir/build/bin/spdk_tgt" --wait-for-rpc &
+spdk_tgt_pid=$!
+waitforlisten "$spdk_tgt_pid"
+
+$rpc_py bdev_set_options -d
+$rpc_py framework_start_init
+
+"$rpc_py" load_subsystem_config -j <($rootdir/scripts/gen_nvme.sh)
+
+# 5GiB minimum for cache device
+cache_size=$((5 * 1024 * 1024 * 1024 / 4096))
+cache_disks=$("$rpc_py" bdev_get_bdevs | jq -r ".[] | select(.md_size==64 and .zoned == false and .num_blocks >= $cache_size).driver_specific.nvme[].pci_address")
+for disk in $cache_disks; do
+	nv_cache=$disk
+	break
+done
+
+if [ -z "$nv_cache" ]; then
+	echo "Couldn't find NVMe device to be used as non-volatile cache"
+	exit 1
+fi
+
+# 5GiB minimum for base device (will be thin provisioned to 100GiB if necessary - it's enough for basic tests)
+base_size=$((5 * 1024 * 1024 * 1024 / 4096))
+base_disks=$("$rpc_py" bdev_get_bdevs | jq -r ".[] | select(.driver_specific.nvme[0].pci_address!=\"$nv_cache\" and .zoned == false and .num_blocks >= $base_size).driver_specific.nvme[].pci_address")
+for disk in $base_disks; do
+	device=$disk
+	break
+done
+
+killprocess "$spdk_tgt_pid"
+
+if [ -z "$device" ]; then
+	echo "Couldn't find NVMe device to be used as base device"
+	exit 1
+fi
+
+if [[ -z $SPDK_TEST_FTL_NIGHTLY ]]; then
+	run_test "ftl_fio_basic" $testdir/fio.sh $device $nv_cache basic
+fi
+
+if [ $SPDK_TEST_FTL_EXTENDED -eq 1 ]; then
+	run_test "ftl_fio_extended" $testdir/fio.sh $device $nv_cache extended
+fi