diff --git a/test/nvme/sw_hotplug.sh b/test/nvme/sw_hotplug.sh index 7ac9e01c2..49193f33f 100755 --- a/test/nvme/sw_hotplug.sh +++ b/test/nvme/sw_hotplug.sh @@ -11,9 +11,11 @@ cleanup() { kill "$hotplug_pid" } +# Helper function to remove/attach cotrollers remove_attach_helper() { local hotplug_events=$1 local hotplug_wait=$2 + local use_bdev=$3 local dev # We need to make sure we wait long enough for hotplug to initialize the devices @@ -25,10 +27,37 @@ remove_attach_helper() { while ((hotplug_events--)); do for dev in "${nvmes[@]}"; do echo 1 > "/sys/bus/pci/devices/$dev/remove" + if $use_bdev; then + sleep "$hotplug_wait" + rpc_cmd bdev_get_bdevs \ + | jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \ + | NOT grep $dev + fi done echo 1 > "/sys/bus/pci/rescan" + if $use_bdev; then + local retries_left=10 + + "$rootdir/scripts/setup.sh" + + while [ "$(rpc_cmd bdev_get_bdevs \ + | jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \ + | wc -l)" -lt "$nvme_count" ]; do + [ $retries_left -gt 0 ] || break + retries_left=$((retries_left - 1)) + sleep 1 + done + + # Bdev name has already changed, search for the bdev's PCI addresses + for dev in "${nvmes[@]}"; do + rpc_cmd bdev_get_bdevs \ + | jq '.[] | .driver_specific | .nvme | .[] | .pci_address' \ + | grep $dev + done + fi + # setup.sh masks failures while writing to sysfs interfaces so let's do # this on our own to make sure there's anything for the hotplug to reattach # to. @@ -36,35 +65,70 @@ remove_attach_helper() { echo "$dev" > "/sys/bus/pci/devices/$dev/driver/unbind" echo "$dev" > "/sys/bus/pci/drivers/${pci_bus_driver["$dev"]}/bind" done + # Wait now for hotplug to reattach to the devices sleep "$hotplug_wait" done } run_hotplug() { - "$rootdir/scripts/setup.sh" - - local hotplug_events=3 - local hotplug_wait=6 # This should be enough for more stubborn nvmes in the CI - local nvmes=($(nvme_in_userspace)) - local nvme_count=${#nvmes[@]} - - xtrace_disable - cache_pci_bus_sysfs - xtrace_restore - trap "cleanup" EXIT - remove_attach_helper "$hotplug_events" "$hotplug_wait" & + remove_attach_helper "$hotplug_events" "$hotplug_wait" false & hotplug_pid=$! "$SPDK_EXAMPLE_DIR/hotplug" \ -i 0 \ - -t $((hotplug_events * hotplug_wait + hotplug_wait)) \ + -t $((hotplug_events * hotplug_wait + hotplug_wait * 3)) \ -n $((hotplug_events * nvme_count)) \ -r $((hotplug_events * nvme_count)) \ -l warning } +# SPDK target hotplug +tgt_run_hotplug() { + local i=0 + local dev + + $SPDK_BIN_DIR/spdk_tgt & + spdk_tgt_pid=$! + + trap 'killprocess ${spdk_tgt_pid}; echo 1 > /sys/bus/pci/rescan; exit 1' SIGINT SIGTERM EXIT + waitforlisten $spdk_tgt_pid + + for dev in "${nvmes[@]}"; do + rpc_cmd bdev_nvme_attach_controller -b "Nvme0$i" -t PCIe -a $dev + waitforbdev "Nvme0${i}n1" "$hotplug_wait" + ((i = i + 1)) + done + + rpc_cmd bdev_nvme_set_hotplug -e + + remove_attach_helper "$hotplug_events" "$hotplug_wait" true + # Verify reregistering hotplug poller + rpc_cmd bdev_nvme_set_hotplug -d + rpc_cmd bdev_nvme_set_hotplug -e + + remove_attach_helper "$hotplug_events" "$hotplug_wait" true + + trap - SIGINT SIGTERM EXIT + killprocess $spdk_tgt_pid +} + +# Preparation +"$rootdir/scripts/setup.sh" + +hotplug_wait=6 +hotplug_events=3 +nvmes=($(nvme_in_userspace)) +nvme_count=${#nvmes[@]} + +xtrace_disable +cache_pci_bus_sysfs +xtrace_restore + # Run pci bus hotplug test run_hotplug + +# Run SPDK target based hotplug +tgt_run_hotplug