From 91751e1e4091b75e6ba10f15e5960f4103eed06b Mon Sep 17 00:00:00 2001 From: Monica Kenguva Date: Tue, 16 Nov 2021 11:02:27 +0900 Subject: [PATCH] test/nvmf: Multipath testing with ANA states support Signed-off-by: Monica Kenguva Change-Id: Ide3af4fd027450eeac523e7078ba41ab0572ed80 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10197 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- scripts/vagrant/autorun-spdk.conf | 1 + test/common/autotest_common.sh | 6 ++ test/nvmf/host/multipath.sh | 114 ++++++++++++++++++++++++++++++ test/nvmf/nvmf.sh | 3 + 4 files changed, 124 insertions(+) create mode 100755 test/nvmf/host/multipath.sh diff --git a/scripts/vagrant/autorun-spdk.conf b/scripts/vagrant/autorun-spdk.conf index 18ab49740..bb2a910a8 100644 --- a/scripts/vagrant/autorun-spdk.conf +++ b/scripts/vagrant/autorun-spdk.conf @@ -18,6 +18,7 @@ SPDK_TEST_VHOST=0 SPDK_TEST_VHOST_INIT=0 SPDK_TEST_BLOCKDEV=1 SPDK_TEST_URING=0 +SPDK_TEST_USDT=0 # doesn't work on vm SPDK_TEST_IOAT=0 SPDK_TEST_BLOBFS=0 diff --git a/test/common/autotest_common.sh b/test/common/autotest_common.sh index ad10e4354..1111da40d 100755 --- a/test/common/autotest_common.sh +++ b/test/common/autotest_common.sh @@ -151,6 +151,8 @@ export SPDK_AUTOTEST_X export SPDK_TEST_RAID5 : ${SPDK_TEST_URING=0} export SPDK_TEST_URING +: ${SPDK_TEST_USDT=0} +export SPDK_TEST_USDT : ${SPDK_TEST_USE_IGB_UIO:=0} export SPDK_TEST_USE_IGB_UIO : ${SPDK_TEST_SCHEDULER:=0} @@ -370,6 +372,10 @@ function get_config_params() { config_params+=' --with-rdma' fi + if [ $SPDK_TEST_USDT -eq 1 ]; then + config_params+=" --with-usdt" + fi + if [ $(uname -s) == "FreeBSD" ]; then intel="hw.model: Intel" cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) diff --git a/test/nvmf/host/multipath.sh b/test/nvmf/host/multipath.sh new file mode 100755 index 000000000..ac303d7c5 --- /dev/null +++ b/test/nvmf/host/multipath.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" +bpf_sh="$rootdir/scripts/bpftrace.sh" + +bdevperf_rpc_sock=/var/tmp/bdevperf.sock + +# NQN prefix to use for subsystem NQNs +NQN=nqn.2016-06.io.spdk:cnode1 + +nvmftestinit + +nvmfappstart -m 0x3 +nvmfapp_pid=$! + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 +# Set -r to enable ANA reporting feature +$rpc_py nvmf_create_subsystem $NQN -a -s SPDK00000000000001 -r -m 2 +$rpc_py nvmf_subsystem_add_ns $NQN Malloc0 +$rpc_py nvmf_subsystem_add_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT +$rpc_py nvmf_subsystem_add_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT + +$rootdir/test/bdev/bdevperf/bdevperf -m 0x4 -z -r $bdevperf_rpc_sock -q 128 -o 4096 -w verify -t 90 &> $testdir/try.txt & +bdevperf_pid=$! + +trap 'process_shm --id $NVMF_APP_SHM_ID; rm -f $testdir/try.txt; killprocess $bdevperf_pid; nvmftestfini; exit 1' SIGINT SIGTERM EXIT +waitforlisten $bdevperf_pid $bdevperf_rpc_sock + +# Create a controller and set multipath behavior +# bdev_retry_count is set to -1 means infinite reconnects +$rpc_py -s $bdevperf_rpc_sock bdev_nvme_set_options -r -1 +# -l -1 ctrlr_loss_timeout_sec -1 means infinite reconnects +# -o 10 reconnect_delay_sec time to delay a reconnect retry is limited to 10 sec +$rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b Nvme0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT -f ipv4 -n $NQN -l -1 -o 10 +$rpc_py -s $bdevperf_rpc_sock bdev_nvme_attach_controller -b Nvme0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT -f ipv4 -n $NQN -x multipath -l -1 -o 10 + +function set_ANA_state() { + $rpc_py nvmf_subsystem_listener_set_ana_state $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT -n $1 + $rpc_py nvmf_subsystem_listener_set_ana_state $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT -n $2 +} + +# check for io on the expected ANA state port +function confirm_io_on_port() { + $bpf_sh $nvmfapp_pid $rootdir/scripts/bpf/nvmf_path.bt &> $testdir/trace.txt & + dtrace_pid=$! + sleep 6 + active_port=$($rpc_py nvmf_subsystem_get_listeners $NQN | jq -r '.[] | select (.ana_states[0].ana_state=="'$1'") | .address.trsvcid') + cat $testdir/trace.txt + port=$(cut < $testdir/trace.txt -d ']' -f1 | awk '$1=="@path[10.0.0.2," {print $2}' | sed -n '1p') + [[ "$active_port" == "$port" ]] + [[ "$port" == "$2" ]] + kill $dtrace_pid + rm -f $testdir/trace.txt +} + +$rootdir/test/bdev/bdevperf/bdevperf.py -t 120 -s $bdevperf_rpc_sock perform_tests & +rpc_pid=$! + +sleep 1 + +# Set ANA state to each listener +set_ANA_state non_optimized optimized +# Check the IO on expected port with ANA state set +confirm_io_on_port "optimized" $NVMF_SECOND_PORT + +# Check traffic paths with different ANA states +set_ANA_state non_optimized inaccessible +confirm_io_on_port "non_optimized" $NVMF_PORT + +set_ANA_state inaccessible optimized +confirm_io_on_port "optimized" $NVMF_SECOND_PORT + +# Not expecting the io on any port +set_ANA_state inaccessible inaccessible +confirm_io_on_port "" "" + +set_ANA_state non_optimized optimized +confirm_io_on_port "optimized" $NVMF_SECOND_PORT + +# Remove listener to monitor multipath function +$rpc_py nvmf_subsystem_remove_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT +sleep 1 + +# Expect IO on alternate path +confirm_io_on_port "non_optimized" $NVMF_PORT + +# Add listener back with optimized state, traffic should switch to optimized port +$rpc_py nvmf_subsystem_add_listener $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT +$rpc_py nvmf_subsystem_listener_set_ana_state $NQN -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_SECOND_PORT -n optimized + +# wait for the io to switch to second port +sleep 2 +confirm_io_on_port "optimized" $NVMF_SECOND_PORT + +wait $rpc_pid +cat $testdir/try.txt + +killprocess $bdevperf_pid + +$rpc_py nvmf_delete_subsystem $NQN + +trap - SIGINT SIGTERM EXIT + +rm -f $testdir/try.txt +nvmftestfini diff --git a/test/nvmf/nvmf.sh b/test/nvmf/nvmf.sh index 0187f460d..860daf9d9 100755 --- a/test/nvmf/nvmf.sh +++ b/test/nvmf/nvmf.sh @@ -72,6 +72,9 @@ timing_enter host run_test "nvmf_identify" test/nvmf/host/identify.sh "${TEST_ARGS[@]}" run_test "nvmf_perf" test/nvmf/host/perf.sh "${TEST_ARGS[@]}" run_test "nvmf_failover" test/nvmf/host/failover.sh "${TEST_ARGS[@]}" +if [[ $SPDK_TEST_USDT -eq 1 ]]; then + run_test "nvmf_multipath" test/nvmf/host/multipath.sh "${TEST_ARGS[@]}" +fi run_test "nvmf_discovery" test/nvmf/host/discovery.sh "${TEST_ARGS[@]}" # TODO: disabled due to intermittent failures (RDMA_CM_EVENT_UNREACHABLE/ETIMEDOUT)