From 70c2bd8858704f72a2b0486fcf6d536544d4a278 Mon Sep 17 00:00:00 2001 From: Kozlowski Mateusz Date: Tue, 31 May 2022 14:16:40 +0200 Subject: [PATCH] FTL: Add restore tests Additional tests testing clean shutdown/startup capability (both fast and slow shutdown paths). Additionally FIO tests can now run on the same FTL instance, instead of creating a new one for each test. Signed-off-by: Kozlowski Mateusz Signed-off-by: Artur Paszkiewicz Change-Id: I20cbd2fe0303b3e75bf7efca1623e223e2611f17 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13365 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- test/ftl/fio.sh | 10 +++-- test/ftl/ftl.sh | 2 + test/ftl/restore.sh | 105 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 3 deletions(-) create mode 100755 test/ftl/restore.sh diff --git a/test/ftl/fio.sh b/test/ftl/fio.sh index 8859c6875..e7e78acdf 100755 --- a/test/ftl/fio.sh +++ b/test/ftl/fio.sh @@ -52,7 +52,12 @@ fi l2p_dram_size_mb=$(($(get_bdev_size $split_bdev) * l2p_percentage / 100 / 1024)) -$rpc_py -t $timeout bdev_ftl_create -b ftl0 -d $split_bdev -c $nv_cache --l2p_dram_limit $l2p_dram_size_mb +if [ -z "$uuid" ]; then + # First FTL creation takes longer due to scrubbing, so extending the timeout + $rpc_py -t $timeout bdev_ftl_create -b ftl0 -d $split_bdev -c $nv_cache --l2p_dram_limit $l2p_dram_size_mb +else + $rpc_py bdev_ftl_create -b ftl0 -d $split_bdev -c $nv_cache -u $uuid --l2p_dram_limit $l2p_dram_size_mb +fi waitforbdev ftl0 @@ -60,8 +65,7 @@ waitforbdev ftl0 echo '{"subsystems": [' $rpc_py save_subsystem_config -n bdev echo ']}' - # Temporary hack so tests are always creating new instance, until clean restore is reintroduced later -) | sed 's/"uuid": "[a-f0-9\-]\{36\}"/"uuid": "00000000-0000-0000-0000-000000000000"/g' > $FTL_JSON_CONF +) > $FTL_JSON_CONF killprocess $svcpid trap - SIGINT SIGTERM EXIT diff --git a/test/ftl/ftl.sh b/test/ftl/ftl.sh index 85575078c..1f58a13e3 100755 --- a/test/ftl/ftl.sh +++ b/test/ftl/ftl.sh @@ -70,6 +70,8 @@ fi if [[ -z $SPDK_TEST_FTL_NIGHTLY ]]; then run_test "ftl_fio_basic" $testdir/fio.sh $device $nv_cache basic run_test "ftl_bdevperf" $testdir/bdevperf.sh $device $nv_cache + run_test "ftl_restore" $testdir/restore.sh -c $nv_cache $device + run_test "ftl_restore_fast" $testdir/restore.sh -f -c $nv_cache $device fi if [ $SPDK_TEST_FTL_EXTENDED -eq 1 ]; then diff --git a/test/ftl/restore.sh b/test/ftl/restore.sh new file mode 100755 index 000000000..224bc4ad1 --- /dev/null +++ b/test/ftl/restore.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../..) +source $rootdir/test/common/autotest_common.sh +source $testdir/common.sh + +rpc_py=$rootdir/scripts/rpc.py + +mount_dir=$(mktemp -d) + +while getopts ':u:c:f' opt; do + case $opt in + u) uuid=$OPTARG ;; + c) nv_cache=$OPTARG ;; + f) fast_shutdown=1 ;; + ?) echo "Usage: $0 [-f] [-u UUID] [-c NV_CACHE_PCI_BDF] BASE_PCI_BDF" && exit 1 ;; + esac +done +shift $((OPTIND - 1)) +device=$1 +timeout=240 + +restore_kill() { + if mount | grep $mount_dir; then + umount $mount_dir + fi + rm -rf $mount_dir + rm -f $testdir/testfile.md5 + rm -f $testdir/testfile2.md5 + rm -f $testdir/config/ftl.json + + killprocess $svcpid + rmmod nbd || true + remove_shm +} + +trap "restore_kill; exit 1" SIGINT SIGTERM EXIT + +"$SPDK_BIN_DIR/spdk_tgt" & +svcpid=$! +# Wait until spdk_tgt starts +waitforlisten $svcpid + +split_bdev=$(create_base_bdev nvme0 $device $((1024 * 101))) +if [ -n "$nv_cache" ]; then + nvc_bdev=$(create_nv_cache_bdev nvc0 $nv_cache $split_bdev) +fi + +l2p_dram_size_mb=$(($(get_bdev_size $split_bdev) * 10 / 100 / 1024)) +ftl_construct_args="bdev_ftl_create -b ftl0 -d $split_bdev --l2p_dram_limit $l2p_dram_size_mb" + +[ -n "$uuid" ] && ftl_construct_args+=" -u $uuid" +[ -n "$nv_cache" ] && ftl_construct_args+=" -c $nvc_bdev" + +$rpc_py -t $timeout $ftl_construct_args + +# Load the nbd driver +modprobe nbd +$rpc_py nbd_start_disk ftl0 /dev/nbd0 +waitfornbd nbd0 + +$rpc_py save_config > $testdir/config/ftl.json + +# Prepare the disk by creating ext4 fs and putting a file on it +make_filesystem ext4 /dev/nbd0 +mount /dev/nbd0 $mount_dir +dd if=/dev/urandom of=$mount_dir/testfile bs=4K count=256K +sync +mount -o remount /dev/nbd0 $mount_dir +md5sum $mount_dir/testfile > $testdir/testfile.md5 +umount $mount_dir + +# Kill bdev service and start it again +if [ "$fast_shutdown" -eq "1" ]; then + $rpc_py bdev_ftl_delete -b ftl0 --fast_shutdown +else + $rpc_py bdev_ftl_delete -b ftl0 +fi + +killprocess $svcpid + +"$SPDK_BIN_DIR/spdk_tgt" -L ftl_init & +svcpid=$! +# Wait until spdk_tgt starts +waitforlisten $svcpid + +$rpc_py load_config < $testdir/config/ftl.json +waitfornbd nbd0 + +mount /dev/nbd0 $mount_dir + +# Write second file, to make sure writer thread has restored properly +dd if=/dev/urandom of=$mount_dir/testfile2 bs=4K count=256K +md5sum $mount_dir/testfile2 > $testdir/testfile2.md5 + +# Make sure second file will be read from disk +echo 3 > /proc/sys/vm/drop_caches + +# Check both files have proper data +md5sum -c $testdir/testfile.md5 +md5sum -c $testdir/testfile2.md5 + +trap - SIGINT SIGTERM EXIT +restore_kill