From 8bb27faff6b9dc3877db80c1ee547272bfd95944 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Wed, 4 Aug 2021 09:12:37 +0200 Subject: [PATCH] test/common: Always wait min 2s in waitforserial() It seems like there's some race in the kernel when we try to delete_controller (nvme disconnect) right after the new nvme subsystem is connected. This results in a block subsystem left with lingering nvme devices which are not usable and which start to affect the nvmf suite. They also can't be removed either unless the kernel is rebooted. To workaround it make sure that we wait long enough for all of the subsystems to be in a sane state before we attempt to stress the connect<->disconnect path. Mitigates #2060. Signed-off-by: Michal Berger Change-Id: I9299ecfc760e334504730aab6f19d338fad88081 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9059 Reviewed-by: Pawel Piatek Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins --- test/common/autotest_common.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/common/autotest_common.sh b/test/common/autotest_common.sh index e23ac1f21..c67c47819 100755 --- a/test/common/autotest_common.sh +++ b/test/common/autotest_common.sh @@ -1033,18 +1033,17 @@ function waitforserial() { nvme_device_counter=$2 fi - while [ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]; do - [ $i -lt 15 ] || break - i=$((i + 1)) + # Wait initially for min 2s to make sure all devices are ready for use. It seems + # that we may be racing with a kernel where in some cases immediate disconnect may + # leave dangling subsystem with no-op block devices which can't be used nor removed + # (unless kernel is rebooted) and which start to negatively affect all the tests. + sleep 2 + while ((i++ <= 15)); do + (($(lsblk -l -o NAME,SERIAL | grep -c "$1") == nvme_device_counter)) && return 0 echo "Waiting for devices" sleep 1 done - - if [[ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]]; then - return 1 - fi - - return 0 + return 1 } function waitforserial_disconnect() {