From 8078db7a9e30905a4d46a8205178b1b53bdf3e55 Mon Sep 17 00:00:00 2001 From: Karol Latecki Date: Tue, 22 Nov 2022 13:48:37 +0100 Subject: [PATCH] test/nvmf: reduce number of RDMA IO queues when using ConnectX-5 NICs Reduce maximum number of queues when connecting with ConnectX-5 NICs. When using host systems with nproc > 64 connecting with default options and creating all IO queues takes too much time and results in keep-alive timeout. Change-Id: I3b3a000fad3a69d4ce5657df494245be2b2a0a81 Signed-off-by: Karol Latecki Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15571 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Tomasz Zawadzki --- test/nvmf/common.sh | 16 ++++++++++++++++ test/nvmf/target/connect_disconnect.sh | 5 +++-- test/nvmf/target/filesystem.sh | 2 +- test/nvmf/target/fio.sh | 2 +- test/nvmf/target/initiator_timeout.sh | 2 +- test/nvmf/target/multiconnection.sh | 2 +- test/nvmf/target/multipath.sh | 4 ++-- test/nvmf/target/nmic.sh | 4 ++-- test/nvmf/target/nvme_cli.sh | 2 +- test/nvmf/target/rpc.sh | 10 +++++----- test/nvmf/target/srq_overwhelm.sh | 3 ++- 11 files changed, 35 insertions(+), 17 deletions(-) diff --git a/test/nvmf/common.sh b/test/nvmf/common.sh index 4f8e33426..aea7456d5 100644 --- a/test/nvmf/common.sh +++ b/test/nvmf/common.sh @@ -6,6 +6,7 @@ NVMF_IP_LEAST_ADDR=8 NVMF_TCP_IP_ADDRESS="127.0.0.1" NVMF_TRANSPORT_OPTS="" NVMF_SERIAL=SPDK00000000000001 +NVME_CONNECT="nvme connect" NET_TYPE=${NET_TYPE:-phy-fallback} function build_nvmf_app_args() { @@ -291,6 +292,7 @@ function gather_supported_nvmf_pci_devs() { x722+=(${pci_bus_cache["$intel:0x37d2"]}) # ConnectX-5 mlx+=(${pci_bus_cache["$mellanox:0x1017"]}) + mlx+=(${pci_bus_cache["$mellanox:0x1019"]}) # ConnectX-4 mlx+=(${pci_bus_cache["$mellanox:0x1015"]}) mlx+=(${pci_bus_cache["$mellanox:0x1013"]}) @@ -325,6 +327,20 @@ function gather_supported_nvmf_pci_devs() { echo "$pci not bound, needs ${pci_mod_resolved["$pci"]}" pci_drivers["${pci_mod_resolved["$pci"]}"]=1 fi + if [[ ${pci_ids_device["$pci"]} == "0x1017" ]] \ + || [[ ${pci_ids_device["$pci"]} == "0x1019" ]] \ + || [[ $TEST_TRANSPORT == rdma ]]; then + # Reduce maximum number of queues when connecting with + # ConnectX-5 NICs. When using host systems with nproc > 64 + # connecting with default options (where default equals to + # number of host online CPUs) creating all IO queues + # takes too much time and results in keep-alive timeout. + # See: + # https://github.com/spdk/spdk/issues/2772 + # 0x1017 - MT27800 Family ConnectX-5 + # 0x1019 - MT28800 Family ConnectX-5 Ex + NVME_CONNECT="nvme connect -i 15" + fi done if ((${#pci_drivers[@]} > 0)); then diff --git a/test/nvmf/target/connect_disconnect.sh b/test/nvmf/target/connect_disconnect.sh index 12c4e4202..8eb60feb9 100755 --- a/test/nvmf/target/connect_disconnect.sh +++ b/test/nvmf/target/connect_disconnect.sh @@ -22,14 +22,15 @@ $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPOR if [ $RUN_NIGHTLY -eq 1 ]; then num_iterations=100 - IO_QUEUES="-i 8" + # Reduce number of IO queues to shorten connection time + NVME_CONNECT="nvme connect -i 8" else num_iterations=10 fi set +x for i in $(seq 1 $num_iterations); do - nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" $IO_QUEUES + $NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial "$NVMF_SERIAL" nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" waitforserial_disconnect "$NVMF_SERIAL" diff --git a/test/nvmf/target/filesystem.sh b/test/nvmf/target/filesystem.sh index 946dac101..440c326ff 100755 --- a/test/nvmf/target/filesystem.sh +++ b/test/nvmf/target/filesystem.sh @@ -54,7 +54,7 @@ function nvmf_filesystem_part() { malloc_size=$(($(get_bdev_size $malloc_name) * 1024 * 1024)) - nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + $NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial "$NVMF_SERIAL" nvme_name=$(lsblk -l -o NAME,SERIAL | grep -oP "([\w]*)(?=\s+${NVMF_SERIAL})") diff --git a/test/nvmf/target/fio.sh b/test/nvmf/target/fio.sh index 2c324bd45..4b0442c3c 100755 --- a/test/nvmf/target/fio.sh +++ b/test/nvmf/target/fio.sh @@ -40,7 +40,7 @@ $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 raid0 # Append the concat0 bdev into subsystem $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 concat0 -nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +$NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial $NVMF_SERIAL 4 diff --git a/test/nvmf/target/initiator_timeout.sh b/test/nvmf/target/initiator_timeout.sh index 223ab9b46..077081951 100755 --- a/test/nvmf/target/initiator_timeout.sh +++ b/test/nvmf/target/initiator_timeout.sh @@ -23,7 +23,7 @@ $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Delay0 $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT -nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +$NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial "$NVMF_SERIAL" diff --git a/test/nvmf/target/multiconnection.sh b/test/nvmf/target/multiconnection.sh index bbee0ee07..cd257992b 100755 --- a/test/nvmf/target/multiconnection.sh +++ b/test/nvmf/target/multiconnection.sh @@ -23,7 +23,7 @@ for i in $(seq 1 $NVMF_SUBSYS); do done for i in $(seq 1 $NVMF_SUBSYS); do - nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode${i}" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + $NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode${i}" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial SPDK$i done diff --git a/test/nvmf/target/multipath.sh b/test/nvmf/target/multipath.sh index 1c22bb080..c6ee9feab 100755 --- a/test/nvmf/target/multipath.sh +++ b/test/nvmf/target/multipath.sh @@ -52,8 +52,8 @@ $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_SECOND_TARGET_IP -s $NVMF_PORT -nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" -g -G -nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_SECOND_TARGET_IP" -s "$NVMF_PORT" -g -G +$NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" -g -G +$NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_SECOND_TARGET_IP" -s "$NVMF_PORT" -g -G waitforserial "$NVMF_SERIAL" diff --git a/test/nvmf/target/nmic.sh b/test/nvmf/target/nmic.sh index 46a7279b3..d7bd5b78b 100755 --- a/test/nvmf/target/nmic.sh +++ b/test/nvmf/target/nmic.sh @@ -35,8 +35,8 @@ fi echo "test case2: host connect to nvmf target in multiple paths" $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s "$NVMF_SECOND_PORT" -nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" -nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_SECOND_PORT" +$NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +$NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_SECOND_PORT" waitforserial "$NVMF_SERIAL" diff --git a/test/nvmf/target/nvme_cli.sh b/test/nvmf/target/nvme_cli.sh index baeb15d2e..3f32268a4 100755 --- a/test/nvmf/target/nvme_cli.sh +++ b/test/nvmf/target/nvme_cli.sh @@ -31,7 +31,7 @@ $rpc_py nvmf_subsystem_add_listener discovery -t $TEST_TRANSPORT -a $NVMF_FIRST_ nvme discover -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT" devs=($(get_nvme_devs)) nvme_num_before_connection=${#devs[@]} -nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +$NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial $NVMF_SERIAL 2 if [[ -z $(get_nvme_devs) ]]; then diff --git a/test/nvmf/target/rpc.sh b/test/nvmf/target/rpc.sh index 4d089fed3..01dd5553a 100755 --- a/test/nvmf/target/rpc.sh +++ b/test/nvmf/target/rpc.sh @@ -52,22 +52,22 @@ $rpc_py nvmf_subsystem_allow_any_host -d nqn.2016-06.io.spdk:cnode1 $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT # This connect should fail - the host NQN is not allowed -NOT nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +NOT $NVME_CONNECT -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" # Add the host NQN and verify that the connect succeeds $rpc_py nvmf_subsystem_add_host nqn.2016-06.io.spdk:cnode1 nqn.2016-06.io.spdk:host1 -nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +$NVME_CONNECT -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial "$NVMF_SERIAL" nvme disconnect -n nqn.2016-06.io.spdk:cnode1 waitforserial_disconnect "$NVMF_SERIAL" # Remove the host and verify that the connect fails $rpc_py nvmf_subsystem_remove_host nqn.2016-06.io.spdk:cnode1 nqn.2016-06.io.spdk:host1 -NOT nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +NOT $NVME_CONNECT -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" # Allow any host and verify that the connect succeeds $rpc_py nvmf_subsystem_allow_any_host -e nqn.2016-06.io.spdk:cnode1 -nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +$NVME_CONNECT -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial "$NVMF_SERIAL" nvme disconnect -n nqn.2016-06.io.spdk:cnode1 waitforserial_disconnect "$NVMF_SERIAL" @@ -80,7 +80,7 @@ for i in $(seq 1 $loops); do $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 -n 5 $rpc_py nvmf_subsystem_allow_any_host nqn.2016-06.io.spdk:cnode1 - nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + $NVME_CONNECT -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforserial "$NVMF_SERIAL" diff --git a/test/nvmf/target/srq_overwhelm.sh b/test/nvmf/target/srq_overwhelm.sh index 0185a41f3..fc95df8b9 100755 --- a/test/nvmf/target/srq_overwhelm.sh +++ b/test/nvmf/target/srq_overwhelm.sh @@ -7,6 +7,7 @@ source $rootdir/test/nvmf/common.sh MALLOC_BDEV_SIZE=64 MALLOC_BLOCK_SIZE=512 +NVME_CONNECT="nvme connect -i 16" nvmftestinit @@ -20,7 +21,7 @@ for i in $(seq 0 5); do $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc$i $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$i Malloc$i $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$i -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT - nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode${i}" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" -i 16 + $NVME_CONNECT -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode${i}" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" waitforblk "nvme${i}n1" done