hotplug: add new software hotplug test

Add a new test to sw_hotplug.sh based on SPDK target
in addition to examples/hotplug application.

Change-Id: I352ee4dfe934fa88082101bb92c6db41da10c10c
Signed-off-by: Krzysztof Karas <krzysztof.karas@intel.com>

Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10088
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

test/nvme/sw_hotplug.sh

@@ -11,9 +11,11 @@ cleanup() {
 	kill "$hotplug_pid"
 }
 
+# Helper function to remove/attach cotrollers
 remove_attach_helper() {
 	local hotplug_events=$1
 	local hotplug_wait=$2
+	local use_bdev=$3
 	local dev
 
 	# We need to make sure we wait long enough for hotplug to initialize the devices
@@ -25,10 +27,37 @@ remove_attach_helper() {
 	while ((hotplug_events--)); do
 		for dev in "${nvmes[@]}"; do
 			echo 1 > "/sys/bus/pci/devices/$dev/remove"
+			if $use_bdev; then
+				sleep "$hotplug_wait"
+				rpc_cmd bdev_get_bdevs \
+					| jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \
+					| NOT grep $dev
+			fi
 		done
 
 		echo 1 > "/sys/bus/pci/rescan"
 
+		if $use_bdev; then
+			local retries_left=10
+
+			"$rootdir/scripts/setup.sh"
+
+			while [ "$(rpc_cmd bdev_get_bdevs \
+				| jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \
+				| wc -l)" -lt "$nvme_count" ]; do
+				[ $retries_left -gt 0 ] || break
+				retries_left=$((retries_left - 1))
+				sleep 1
+			done
+
+			# Bdev name has already changed, search for the bdev's PCI addresses
+			for dev in "${nvmes[@]}"; do
+				rpc_cmd bdev_get_bdevs \
+					| jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \
+					| grep $dev
+			done
+		fi
+
 		# setup.sh masks failures while writing to sysfs interfaces so let's do
 		# this on our own to make sure there's anything for the hotplug to reattach
 		# to.
@@ -36,35 +65,70 @@ remove_attach_helper() {
 			echo "$dev" > "/sys/bus/pci/devices/$dev/driver/unbind"
 			echo "$dev" > "/sys/bus/pci/drivers/${pci_bus_driver["$dev"]}/bind"
 		done
+
 		# Wait now for hotplug to reattach to the devices
 		sleep "$hotplug_wait"
 	done
 }
 
 run_hotplug() {
-	"$rootdir/scripts/setup.sh"
-
-	local hotplug_events=3
-	local hotplug_wait=6 # This should be enough for more stubborn nvmes in the CI
-	local nvmes=($(nvme_in_userspace))
-	local nvme_count=${#nvmes[@]}
-
-	xtrace_disable
-	cache_pci_bus_sysfs
-	xtrace_restore
-
 	trap "cleanup" EXIT
 
-	remove_attach_helper "$hotplug_events" "$hotplug_wait" &
+	remove_attach_helper "$hotplug_events" "$hotplug_wait" false &
 	hotplug_pid=$!
 
 	"$SPDK_EXAMPLE_DIR/hotplug" \
 		-i 0 \
-		-t $((hotplug_events * hotplug_wait + hotplug_wait)) \
+		-t $((hotplug_events * hotplug_wait + hotplug_wait * 3)) \
 		-n $((hotplug_events * nvme_count)) \
 		-r $((hotplug_events * nvme_count)) \
 		-l warning
 }
 
+# SPDK target hotplug
+tgt_run_hotplug() {
+	local i=0
+	local dev
+
+	$SPDK_BIN_DIR/spdk_tgt &
+	spdk_tgt_pid=$!
+
+	trap 'killprocess ${spdk_tgt_pid}; echo 1 > /sys/bus/pci/rescan; exit 1' SIGINT SIGTERM EXIT
+	waitforlisten $spdk_tgt_pid
+
+	for dev in "${nvmes[@]}"; do
+		rpc_cmd bdev_nvme_attach_controller -b "Nvme0$i" -t PCIe -a $dev
+		waitforbdev "Nvme0${i}n1" "$hotplug_wait"
+		((i = i + 1))
+	done
+
+	rpc_cmd bdev_nvme_set_hotplug -e
+
+	remove_attach_helper "$hotplug_events" "$hotplug_wait" true
+	# Verify reregistering hotplug poller
+	rpc_cmd bdev_nvme_set_hotplug -d
+	rpc_cmd bdev_nvme_set_hotplug -e
+
+	remove_attach_helper "$hotplug_events" "$hotplug_wait" true
+
+	trap - SIGINT SIGTERM EXIT
+	killprocess $spdk_tgt_pid
+}
+
+# Preparation
+"$rootdir/scripts/setup.sh"
+
+hotplug_wait=6
+hotplug_events=3
+nvmes=($(nvme_in_userspace))
+nvme_count=${#nvmes[@]}
+
+xtrace_disable
+cache_pci_bus_sysfs
+xtrace_restore
+
 # Run pci bus hotplug test
 run_hotplug
+
+# Run SPDK target based hotplug
+tgt_run_hotplug