test/nvmf: Test bdev nvme connection and reconnect timeout parameters

ctrlr_loss_timeout_sec and fast_io_fail_timeout_sec are
tested with bdev_nvme_attach_controller RPC parameters

Signed-off-by: Monica Kenguva <monica.kenguva@intel.com>
Change-Id: Ia89a7998732abc9974427e86cf64ad17dd8fdf58
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12016
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>

test/nvmf/host/timeout.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+source $rootdir/test/nvmf/common.sh
+
+MALLOC_BDEV_SIZE=64
+MALLOC_BLOCK_SIZE=512
+
+rpc_py="$rootdir/scripts/rpc.py"
+
+bdevperf_rpc_sock=/var/tmp/bdevperf.sock
+
+nvmftestinit
+
+nvmfappstart -m 0x3
+
+$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192
+$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0
+$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001
+$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0
+$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+
+$rootdir/test/bdev/bdevperf/bdevperf -m 0x4 -z -r $bdevperf_rpc_sock -q 128 -o 4096 -w verify -t 20 -f &> $testdir/try.txt &
+bdevperf_pid=$!
+
+trap 'process_shm --id $NVMF_APP_SHM_ID; rm -f $testdir/try.txt; killprocess $bdevperf_pid; nvmftestfini; exit 1' SIGINT SIGTERM EXIT
+waitforlisten $bdevperf_pid $bdevperf_rpc_sock
+
+function get_bdev() {
+	$rpc_py -s $bdevperf_rpc_sock bdev_get_bdevs | jq -r '.[].name'
+}
+
+function get_controller() {
+	$rpc_py -s $bdevperf_rpc_sock bdev_nvme_get_controllers | jq -r '.[].name'
+}
+
+# case 1 test ctrlr_loss_timeout_sec time to try reconnecting to a ctrlr before deleting it
+# ctrlr_loss_timeout_sec is 10 reconnect_delay_sec is 5
+$rpc_py -s $bdevperf_rpc_sock bdev_nvme_set_options -r -1
+$rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b NVMe0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT -f ipv4 -n nqn.2016-06.io.spdk:cnode1 -l 10 -o 5
+
+$rootdir/test/bdev/bdevperf/bdevperf.py -s $bdevperf_rpc_sock perform_tests &
+rpc_pid=$!
+
+sleep 1
+
+$rpc_py nvmf_subsystem_remove_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+sleep 5
+[[ "$(get_controller)" == "NVMe0" ]]
+[[ "$(get_bdev)" == "NVMe0n1" ]]
+
+# wait for the ctrlr_loss_timeout_sec time 10 sec and check bdevs and controller are deleted
+sleep 10
+[[ "$(get_controller)" == "" ]]
+[[ "$(get_bdev)" == "" ]]
+
+wait $rpc_pid
+
+cat $testdir/try.txt
+killprocess $bdevperf_pid
+rm -f $testdir/try.txt
+
+# Case 2 test fast_io_fail_timeout_sec
+# Time to wait until ctrlr is reconnected before failing I/O to ctrlr
+$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+
+$rootdir/test/bdev/bdevperf/bdevperf -m 0x4 -z -r $bdevperf_rpc_sock -q 128 -o 4096 -w verify -t 20 -f &> $testdir/try.txt &
+bdevperf_pid=$!
+
+trap 'process_shm --id $NVMF_APP_SHM_ID; rm -f $testdir/try.txt; killprocess $bdevperf_pid; nvmftestfini; exit 1' SIGINT SIGTERM EXIT
+waitforlisten $bdevperf_pid $bdevperf_rpc_sock
+
+# ctrlr_loss_timeout_sec is 10 fast_io_fail_timeout_sec is 2 -o reconnect_delay_sec is 1
+$rpc_py -s $bdevperf_rpc_sock bdev_nvme_set_options -r -1
+$rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b NVMe0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT \
+	-f ipv4 -n nqn.2016-06.io.spdk:cnode1 -l 10 -u 2 -o 1
+
+$rootdir/test/bdev/bdevperf/bdevperf.py -s $bdevperf_rpc_sock perform_tests &
+rpc_pid=$!
+
+sleep 1
+$rpc_py nvmf_subsystem_remove_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+
+# ctrlr should try to reconnect and I/O submitted should be queued until the listener is added back before 5 sec fast_io_fail_timeout_sec
+sleep 1
+$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+wait $rpc_pid
+cat $testdir/try.txt
+cat /dev/null > $testdir/try.txt
+
+$rootdir/test/bdev/bdevperf/bdevperf.py -s $bdevperf_rpc_sock perform_tests &
+rpc_pid=$!
+sleep 1
+$rpc_py nvmf_subsystem_remove_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+# bdevperf fails to process the I/O fast_io_fail_timeout_sec expires at 2 sec
+sleep 5
+$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
+wait $rpc_pid
+
+cat $testdir/try.txt
+killprocess $bdevperf_pid
+rm -f $testdir/try.txt
+
+$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1
+
+trap - SIGINT SIGTERM EXIT
+
+nvmftestfini

test/nvmf/nvmf.sh

@@ -84,6 +84,7 @@ if [[ $SPDK_TEST_USDT -eq 1 ]]; then
 	run_test "nvmf_multipath" test/nvmf/host/multipath.sh "${TEST_ARGS[@]}"
 fi
 run_test "nvmf_discovery" test/nvmf/host/discovery.sh "${TEST_ARGS[@]}"
+run_test "nvmf_timeout" test/nvmf/host/timeout.sh "${TEST_ARGS[@]}"
 
 # TODO: disabled due to intermittent failures (RDMA_CM_EVENT_UNREACHABLE/ETIMEDOUT)
 #run_test test/nvmf/host/identify_kernel_nvmf.sh $TEST_ARGS