test/nvme: Simplify hotplug test

Instead of relaying on vhost VM image, which may be incompatible with
the host system, perform the hotplug test locally against detecte
nvme device.

Also, limit this test to Linux since it never was intended to run
on other platforms in the CI due to qemu dependency.

Fixes #2150

Signed-off-by: Michal Berger <michalx.berger@intel.com>
Change-Id: Ie048343ceb52c1a45193a6c857bc3a09fa1ec3fd
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9461
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

autotest.sh

@@ -230,7 +230,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		# enabled, it catches SEGV earlier than our handler which
 		# breaks the hotplug logic.
 		if [ $SPDK_RUN_ASAN -eq 0 ]; then
-			run_test "nvme_hotplug" test/nvme/hotplug.sh root
+			run_test "nvme_hotplug" test/nvme/hotplug.sh
 		fi
 	fi
 

test/nvme/hotplug.sh

@@ -4,136 +4,64 @@ testdir=$(readlink -f $(dirname $0))
 rootdir=$(readlink -f $testdir/../..)
 source $rootdir/test/common/autotest_common.sh
 
-if [ -z "${DEPENDENCY_DIR}" ]; then
+[[ $(uname -s) == Linux ]] || exit 0
-	echo DEPENDENCY_DIR not defined!
-	exit 1
-fi
 
-function ssh_vm() {
+cleanup() {
-	xtrace_disable
+	[[ -e /proc/$hotplug_pid/status ]] || return 0
-	sshpass -p "$password" ssh -o PubkeyAuthentication=no \
+	kill "$hotplug_pid"
-		-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 10022 root@localhost "$@"
-	xtrace_restore
 }
 
-function monitor_cmd() {
+remove_attach_helper() {
-	echo "$@" | nc localhost 4444 | tail --lines=+2 | (grep -v '^(qemu) ' || true)
+	local hotplug_events=$1
-}
+	local hotplug_wait=$2
+	local dev
 
-function get_online_devices_count() {
+	# We need to make sure we wait long enough for hotplug to initialize the devices
-	ssh_vm "lspci | grep -c NVM"
+	# and start IO - if we start removing devices before that happens we will end up
-}
+	# stepping on hotplug's toes forcing it to fail to report proper count of given
+	# events.
+	sleep "$hotplug_wait"
 
-function wait_for_devices_ready() {
+	while ((hotplug_events--)); do
-	count=$(get_online_devices_count)
+		for dev in "${nvmes[@]}"; do
+			echo 1 > "/sys/bus/pci/devices/$dev/remove"
+		done
 
-	while [ $count -ne 4 ]; do
+		echo 1 > "/sys/bus/pci/rescan"
-		echo "waitting for all devices online"
+
-		count=$(get_online_devices_count)
+		# setup.sh masks failures while writing to sysfs interfaces so let's do
+		# this on our own to make sure there's anything for the hotplug to reattach
+		# to.
+		for dev in "${nvmes[@]}"; do
+			echo "$dev" > "/sys/bus/pci/devices/$dev/driver/unbind"
+			echo "$dev" > "/sys/bus/pci/drivers/${pci_bus_driver["$dev"]}/bind"
+		done
+		# Wait now for hotplug to reattach to the devices
+		sleep "$hotplug_wait"
 	done
 }
 
-function insert_devices() {
+hotplug() {
-	for i in {0..3}; do
+	local hotplug_events=3
-		monitor_cmd "device_add nvme,drive=drive$i,id=nvme$i,serial=nvme$i"
+	local hotplug_wait=6 # This should be enough for more stubborn nvmes in the CI
-	done
+
-	wait_for_devices_ready
+	remove_attach_helper "$hotplug_events" "$hotplug_wait" &
-	ssh_vm "scripts/setup.sh"
+	hotplug_pid=$!
+
+	"$SPDK_EXAMPLE_DIR/hotplug" \
+		-i 0 \
+		-t $((hotplug_events * hotplug_wait + hotplug_wait)) \
+		-n $((hotplug_events * nvme_count)) \
+		-r $((hotplug_events * nvme_count))
 }
 
-function remove_devices() {
+"$rootdir/scripts/setup.sh"
-	for i in {0..3}; do
+nvmes=($(nvme_in_userspace)) nvme_count=${#nvmes[@]}
-		monitor_cmd "device_del nvme$i"
-	done
-}
 
-function devices_delete() {
+xtrace_disable
-	for i in {0..3}; do
+cache_pci_bus_sysfs
-		rm "$SPDK_TEST_STORAGE/nvme$i.img"
+xtrace_restore
-	done
-}
 
-password=$1
+trap "cleanup" EXIT
-base_img=$DEPENDENCY_DIR/spdk_test_image.qcow2
-qemu_pidfile=$HOME/qemupid
 
-if [ ! -e "$base_img" ]; then
+hotplug
-	echo "Hotplug VM image not found; skipping test"
-	exit 0
-fi
-
-timing_enter start_qemu
-
-for i in {0..3}; do
-	dd if=/dev/zero of="$SPDK_TEST_STORAGE/nvme$i.img" bs=1M count=1024
-done
-
-qemu-system-x86_64 \
-	-daemonize -display none -m 8192 \
-	-pidfile "$qemu_pidfile" \
-	-hda "$base_img" \
-	-net user,hostfwd=tcp::10022-:22 \
-	-net nic \
-	-cpu host \
-	-smp cores=16,sockets=1 \
-	--enable-kvm \
-	-chardev socket,id=mon0,host=localhost,port=4444,server,nowait \
-	-mon chardev=mon0,mode=readline \
-	-drive format=raw,file="$SPDK_TEST_STORAGE/nvme0.img",if=none,id=drive0 \
-	-drive format=raw,file="$SPDK_TEST_STORAGE/nvme1.img",if=none,id=drive1 \
-	-drive format=raw,file="$SPDK_TEST_STORAGE/nvme2.img",if=none,id=drive2 \
-	-drive format=raw,file="$SPDK_TEST_STORAGE/nvme3.img",if=none,id=drive3 \
-	-snapshot
-
-timing_exit start_qemu
-
-timing_enter wait_for_vm
-ssh_vm 'echo ready'
-timing_exit wait_for_vm
-
-timing_enter copy_repo
-files_to_copy="scripts "
-files_to_copy+="include/spdk/pci_ids.h "
-files_to_copy+="build/examples/hotplug "
-files_to_copy+="build/lib "
-
-# Select which dpdk libs to copy in case we're not building with
-# spdk/dpdk submodule
-if [[ -n "$SPDK_RUN_EXTERNAL_DPDK" ]]; then
-	files_to_copy+="-C $SPDK_RUN_EXTERNAL_DPDK/../.. dpdk/build/lib"
-else
-	files_to_copy+="dpdk/build/lib "
-fi
-
-(
-	cd "$rootdir"
-	tar -cf - $files_to_copy
-) | (ssh_vm "tar -xf -")
-timing_exit copy_repo
-
-insert_devices
-
-timing_enter hotplug_test
-
-ssh_vm "LD_LIBRARY_PATH=/root//build/lib:/root/dpdk/build/lib:$LD_LIBRARY_PATH build/examples/hotplug -i 0 -t 25 -n 4 -r 8" &
-example_pid=$!
-
-sleep 6
-remove_devices
-sleep 4
-insert_devices
-sleep 6
-remove_devices
-devices_delete
-
-timing_enter wait_for_example
-wait $example_pid
-timing_exit wait_for_example
-
-trap - SIGINT SIGTERM EXIT
-
-qemupid=$(awk '{printf $0}' "$qemu_pidfile")
-kill -9 $qemupid
-rm "$qemu_pidfile"
-
-timing_exit hotplug_test