From ea80f87e0eb705f4c5a22d1eb761ba14262cc1b2 Mon Sep 17 00:00:00 2001 From: Kozlowski Mateusz Date: Fri, 27 May 2022 15:57:45 +0200 Subject: [PATCH] ftl: Dirty shutdown tests Add additional tests checking against dirty shutdown recovery. Also adds 'write after write' test - checking if two simultaneous writes to the same LBAs return the same data before and after dirty shutdown. Signed-off-by: Kozlowski Mateusz Signed-off-by: Artur Paszkiewicz Change-Id: Idcf9b51d9c00d0d065f7e9655387668f5eeb646d Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13376 Reviewed-by: Jim Harris Reviewed-by: Ben Walker Tested-by: SPDK CI Jenkins --- test/common/skipped_tests.txt | 1 + test/ftl/config/fio/write_after_write.fio | 20 +++++ test/ftl/dirty_shutdown.sh | 92 +++++++++++++++++++++++ test/ftl/ftl.sh | 3 + test/ftl/write_after_write.sh | 88 ++++++++++++++++++++++ 5 files changed, 204 insertions(+) create mode 100644 test/ftl/config/fio/write_after_write.fio create mode 100755 test/ftl/dirty_shutdown.sh create mode 100755 test/ftl/write_after_write.sh diff --git a/test/common/skipped_tests.txt b/test/common/skipped_tests.txt index 1c7eaab62..3cc43d17d 100644 --- a/test/common/skipped_tests.txt +++ b/test/common/skipped_tests.txt @@ -3,6 +3,7 @@ # cases ftl_dirty_shutdown ftl_restore_fast +ftl_write_after_write ftl_fio_basic ftl_fio_extended ftl_fio_nightly diff --git a/test/ftl/config/fio/write_after_write.fio b/test/ftl/config/fio/write_after_write.fio new file mode 100644 index 000000000..53f928fcd --- /dev/null +++ b/test/ftl/config/fio/write_after_write.fio @@ -0,0 +1,20 @@ +[global] +ioengine=spdk_bdev +spdk_json_conf=${FTL_JSON_CONF} +filename=${FTL_BDEV_NAME} +thread=1 +direct=1 +random_distribution=normal +io_size=5G + +[job1] +iodepth=512 +rw=write +bs=32k +buffer_pattern="abcdef" + +[job2] +iodepth=512 +rw=write +bs=64k +buffer_pattern="qwert" diff --git a/test/ftl/dirty_shutdown.sh b/test/ftl/dirty_shutdown.sh new file mode 100755 index 000000000..e74d6c914 --- /dev/null +++ b/test/ftl/dirty_shutdown.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../..) +source $rootdir/test/common/autotest_common.sh +source $testdir/common.sh + +rpc_py=$rootdir/scripts/rpc.py + +while getopts ':u:c:' opt; do + case $opt in + u) uuid=$OPTARG ;; + c) nv_cache=$OPTARG ;; + ?) echo "Usage: $0 [-u UUID] [-c NV_CACHE_PCI_BDF] BASE_PCI_BDF" && exit 1 ;; + esac +done +shift $((OPTIND - 1)) + +device=$1 +timeout=240 + +chunk_size=262144 +data_size=$((chunk_size * 2)) + +restore_kill() { + rm -f $testdir/config/ftl.json + rm -f $testdir/testfile + rm -f $testdir/testfile2 + rm -f $testdir/testfile.md5 + rm -f $testdir/testfile2.md5 + + killprocess $svcpid || true + rmmod nbd || true + remove_shm +} + +trap "restore_kill; exit 1" SIGINT SIGTERM EXIT + +"$SPDK_BIN_DIR/spdk_tgt" & +svcpid=$! +# Wait until spdk_tgt starts +waitforlisten $svcpid + +split_bdev=$(create_base_bdev nvme0 $device $((1024 * 101))) + +if [ -n "$nv_cache" ]; then + nvc_bdev=$(create_nv_cache_bdev nvc0 $nv_cache $split_bdev) +fi + +l2p_dram_size_mb=$(($(get_bdev_size $split_bdev) * 10 / 100 / 1024)) +ftl_construct_args="bdev_ftl_create -b ftl0 -d $split_bdev --l2p_dram_limit $l2p_dram_size_mb" + +[ -n "$uuid" ] && ftl_construct_args+=" -u $uuid" +[ -n "$nv_cache" ] && ftl_construct_args+=" -c $nvc_bdev" + +$rpc_py -t $timeout $ftl_construct_args + +( + echo '{"subsystems": [' + $rpc_py save_subsystem_config -n bdev + echo ']}' +) > $testdir/config/ftl.json + +# Load the nbd driver +modprobe nbd +$rpc_py nbd_start_disk ftl0 /dev/nbd0 +waitfornbd nbd0 + +# Write and calculate checksum of the data written +dd if=/dev/urandom of=$testdir/testfile bs=4K count=$data_size +md5sum $testdir/testfile > $testdir/testfile.md5 +dd if=$testdir/testfile of=/dev/nbd0 bs=4K count=$data_size oflag=dsync +$rpc_py nbd_stop_disk /dev/nbd0 + +# Force kill bdev service (dirty shutdown) and start it again +kill -9 $svcpid +rm -f /dev/shm/spdk_tgt_trace.pid$svcpid + +# Write extra data after restore +dd if=/dev/urandom of=$testdir/testfile2 bs=4K count=$chunk_size +"$SPDK_BIN_DIR/spdk_dd" --if=$testdir/testfile2 --ob=ftl0 --count=$chunk_size --seek=$data_size --json=$testdir/config/ftl.json +# Save md5 data +md5sum $testdir/testfile2 > $testdir/testfile2.md5 + +# Verify that the checksum matches and the data is consistent +"$SPDK_BIN_DIR/spdk_dd" --ib=ftl0 --of=$testdir/testfile --count=$data_size --json=$testdir/config/ftl.json +md5sum -c $testdir/testfile.md5 +"$SPDK_BIN_DIR/spdk_dd" --ib=ftl0 --of=$testdir/testfile2 --count=$chunk_size --skip=$data_size --json=$testdir/config/ftl.json +md5sum -c $testdir/testfile2.md5 + +trap - SIGINT SIGTERM EXIT +restore_kill diff --git a/test/ftl/ftl.sh b/test/ftl/ftl.sh index 6cc9e7dcc..f2ab99aa8 100755 --- a/test/ftl/ftl.sh +++ b/test/ftl/ftl.sh @@ -71,10 +71,13 @@ if [[ -z $SPDK_TEST_FTL_NIGHTLY ]]; then run_test "ftl_fio_basic" $testdir/fio.sh $device $nv_cache basic run_test "ftl_bdevperf" $testdir/bdevperf.sh $device $nv_cache run_test "ftl_restore" $testdir/restore.sh -c $nv_cache $device + run_test "ftl_dirty_shutdown" $testdir/dirty_shutdown.sh -c $nv_cache $device fi if [ $SPDK_TEST_FTL_EXTENDED -eq 1 ]; then run_test "ftl_restore_fast" $testdir/restore.sh -f -c $nv_cache $device + run_test "ftl_dirty_shutdown" $testdir/dirty_shutdown.sh -c $nv_cache $device + run_test "ftl_write_after_write" $testdir/write_after_write.sh $device $nv_cache run_test "ftl_fio_extended" $testdir/fio.sh $device $nv_cache extended fi diff --git a/test/ftl/write_after_write.sh b/test/ftl/write_after_write.sh new file mode 100755 index 000000000..d538ea56b --- /dev/null +++ b/test/ftl/write_after_write.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../..) +source $rootdir/test/common/autotest_common.sh +source $testdir/common.sh + +rpc_py=$rootdir/scripts/rpc.py + +fio_kill() { + rm -f $testdir/testfile.md5 + rm -f $testdir/config/ftl.json + + killprocess $svcpid + rmmod nbd || true +} + +device=$1 +cache_device=$2 +timeout=240 +data_size=$((262144 * 5)) + +if [[ $CONFIG_FIO_PLUGIN != y ]]; then + echo "FIO not available" + exit 1 +fi + +export FTL_BDEV_NAME=ftl0 +export FTL_JSON_CONF=$testdir/config/ftl.json + +trap "fio_kill; exit 1" SIGINT SIGTERM EXIT + +"$SPDK_BIN_DIR/spdk_tgt" & +svcpid=$! +waitforlisten $svcpid + +split_bdev=$(create_base_bdev nvme0 $device $((1024 * 101))) +nv_cache=$(create_nv_cache_bdev nvc0 $cache_device $split_bdev) + +l2p_percentage=60 +l2p_dram_size_mb=$(($(get_bdev_size $split_bdev) * l2p_percentage / 100 / 1024)) + +$rpc_py -t $timeout bdev_ftl_create -b ftl0 -d $split_bdev -c $nv_cache --l2p_dram_limit $l2p_dram_size_mb + +waitforbdev ftl0 + +( + echo '{"subsystems": [' + $rpc_py save_subsystem_config -n bdev + echo ']}' +) > $FTL_JSON_CONF + +killprocess $svcpid + +fio_bdev $testdir/config/fio/write_after_write.fio + +"$SPDK_BIN_DIR/spdk_tgt" -L ftl_init & +svcpid=$! +waitforlisten $svcpid + +$rpc_py load_config < $FTL_JSON_CONF +# Load the nbd driver +modprobe nbd +$rpc_py nbd_start_disk ftl0 /dev/nbd0 +waitfornbd nbd0 + +$rpc_py save_config > $testdir/config/ftl.json + +# Calculate checksum of the data written +dd if=/dev/nbd0 bs=4K count=$data_size | md5sum > $testdir/testfile.md5 +$rpc_py nbd_stop_disk /dev/nbd0 + +# Force kill bdev service (dirty shutdown) and start it again +kill -9 $svcpid +rm -f /dev/shm/spdk_tgt_trace.pid$svcpid + +"$SPDK_BIN_DIR/spdk_tgt" -L ftl_init & +svcpid=$! +waitforlisten $svcpid + +$rpc_py load_config < $testdir/config/ftl.json +waitfornbd nbd0 + +# Verify that the checksum matches and the data is consistent +dd if=/dev/nbd0 bs=4K count=$data_size | md5sum -c $testdir/testfile.md5 + +trap - SIGINT SIGTERM EXIT +fio_kill