diff --git a/test/common/skipped_tests.txt b/test/common/skipped_tests.txt index d6fc3176a..2c823a4ee 100644 --- a/test/common/skipped_tests.txt +++ b/test/common/skipped_tests.txt @@ -95,3 +95,4 @@ sma_nvmf_tcp sma_plugins sma_discovery sma_vhost +sma_vfiouser_qemu diff --git a/test/sma/sma.sh b/test/sma/sma.sh index afc45480d..df177658e 100755 --- a/test/sma/sma.sh +++ b/test/sma/sma.sh @@ -6,6 +6,7 @@ rootdir=$(readlink -f "$testdir/../..") source "$rootdir/test/common/autotest_common.sh" run_test "sma_nvmf_tcp" $testdir/nvmf_tcp.sh +run_test "sma_vfiouser_qemu" $testdir/vfiouser_qemu.sh run_test "sma_plugins" $testdir/plugins.sh run_test "sma_discovery" $testdir/discovery.sh run_test "sma_vhost" $testdir/vhost_blk.sh diff --git a/test/sma/vfiouser_qemu.sh b/test/sma/vfiouser_qemu.sh new file mode 100755 index 000000000..5f74c6ef8 --- /dev/null +++ b/test/sma/vfiouser_qemu.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f "$(dirname "$0")") +rootdir=$(readlink -f "$testdir/../..") + +source "$rootdir/test/common/autotest_common.sh" +source "$rootdir/test/vhost/common.sh" +source "$testdir/common.sh" + +function create_device() { + local pfid=${1:-1} + local vfid=${2:-0} + + "$rootdir/scripts/sma-client.py" <<- EOF + { + "method": "CreateDevice", + "params": { + "nvme": { + "physical_id": "$pfid", + "virtual_id": "$vfid" + } + } + } + EOF +} + +function delete_device() { + "$rootdir/scripts/sma-client.py" <<- EOF + { + "method": "DeleteDevice", + "params": { + "handle": "$1" + } + } + EOF +} + +function attach_volume() { + "$rootdir/scripts/sma-client.py" <<- EOF + { + "method": "AttachVolume", + "params": { + "device_handle": "$1", + "volume": { + "volume_id": "$(uuid2base64 $2)" + } + } + } + EOF +} + +function detach_volume() { + "$rootdir/scripts/sma-client.py" <<- EOF + { + "method": "DetachVolume", + "params": { + "device_handle": "$1", + "volume_id": "$(uuid2base64 $2)" + } + } + EOF +} + +function vm_count_nvme() { + vm_exec $1 "grep -l SPDK /sys/class/nvme/*/model" | wc -l +} + +function vm_check_subsys_volume() { + local vm_id=$1 + local nqn=$2 + local uuid=$3 + + nvme="$(vm_exec $vm_id "grep -l $nqn /sys/class/nvme/*/subsysnqn" | awk -F/ '{print $5}')" + if [[ -z "$nvme" ]]; then + error "FAILED no NVMe on vm=$vm_id with nqn=$nqn" + return 1 + fi + + tmpuuid="$(vm_exec $vm_id "grep -l $uuid /sys/class/nvme/$nvme/nvme*/uuid")" + if [[ -z "$tmpuuid" ]]; then + return 1 + fi +} + +function vm_check_subsys_nqn() { + sleep 1 + nqn=$(vm_exec $1 "grep -l $2 /sys/class/nvme/*/subsysnqn") + if [[ -z "$nqn" ]]; then + error "FAILED no NVMe on vm=$1 with nqn=$2" + return 1 + fi +} + +function cleanup() { + vm_kill_all + killprocess $tgtpid + killprocess $smapid + if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi +} + +trap "cleanup; exit 1" SIGINT SIGTERM EXIT + +# SSH VM Password +VM_PASSWORD=root +vm_no=0 + +VFO_ROOT_PATH="/tmp/sma/vfio-user/qemu" + +if [ -e "${VFO_ROOT_PATH}" ]; then rm -rf "${VFO_ROOT_PATH}"; fi +mkdir -p "${VFO_ROOT_PATH}" + +# Cleanup old VM: +used_vms=$vm_no +vm_kill_all + +vm_setup --os="$VM_IMAGE" --disk-type=virtio --force=$vm_no --qemu-args="-qmp tcp:localhost:10005,server,nowait -device pci-bridge,chassis_nr=1,id=pci.spdk.0" + +# Run pre-configured VM and wait for them to start +vm_run $vm_no +vm_wait_for_boot 300 $vm_no + +# Start SPDK +$rootdir/build/bin/spdk_tgt & +tgtpid=$! +waitforlisten $tgtpid + +# Prepare the target +rpc_cmd bdev_null_create null0 100 4096 +rpc_cmd bdev_null_create null1 100 4096 + +# Start SMA server +$rootdir/scripts/sma.py -c <( + cat <<- EOF + devices: + - name: 'vfiouser' + params: + buses: + - name: 'pci.spdk.0' + count: 32 + qmp_addr: 127.0.0.1 + qmp_port: 10005 + EOF +) & +smapid=$! + +# Wait until the SMA starts listening +sma_waitforlisten + +# Make sure a TCP transport has been created +rpc_cmd nvmf_get_transports --trtype VFIOUSER + +# Make sure no nvme subsystems are present +[[ $(vm_exec ${vm_no} nvme list-subsys -o json | jq -r '.Subsystems | length') -eq 0 ]] + +# Create a couple of devices and verify them via RPC and SSH +device0=$(create_device 0 0 | jq -r '.handle') +rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 +vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-0 + +# Check that there are two subsystems (1 created above + discovery) +[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] + +device1=$(create_device 1 0 | jq -r '.handle') +rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 +rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 +[[ "$device0" != "$device1" ]] +vm_check_subsys_nqn $vm_no nqn.2016-06.io.spdk:vfiouser-1 + +# Check that there are three subsystems (2 created above + discovery) +[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] + +# Verify the method is idempotent and sending the same gRPCs won't create new +# devices and will return the same IDs +tmp0=$(create_device 0 0 | jq -r '.handle') +tmp1=$(create_device 1 0 | jq -r '.handle') + +[[ $(vm_count_nvme ${vm_no}) -eq 2 ]] + +[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 3 ]] +[[ "$tmp0" == "$device0" ]] +[[ "$tmp1" == "$device1" ]] + +# Now remove them verifying via RPC +delete_device "$device0" +NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 +rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 +[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 2 ]] +[[ $(vm_count_nvme ${vm_no}) -eq 1 ]] + +delete_device "$device1" +NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 +NOT rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 +[[ $(rpc_cmd nvmf_get_subsystems | jq -r '. | length') -eq 1 ]] +[[ $(vm_count_nvme ${vm_no}) -eq 0 ]] + +# Finally check that removing a non-existing device is also sucessful +delete_device "$device0" +delete_device "$device1" + +# Check volume attach/detach +device0=$(create_device 0 0 | jq -r '.handle') +device1=$(create_device 1 0 | jq -r '.handle') +uuid0=$(rpc_cmd bdev_get_bdevs -b null0 | jq -r '.[].uuid') +uuid1=$(rpc_cmd bdev_get_bdevs -b null1 | jq -r '.[].uuid') + +# Attach the first volume to a first subsystem +attach_volume "$device0" "$uuid0" +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] +vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 + +attach_volume "$device1" "$uuid1" +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] +vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 + +# Attach the same device again and see that it won't fail +attach_volume "$device0" "$uuid0" +attach_volume "$device1" "$uuid1" +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] +vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 +NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 +vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 +NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 + +# Cross detach volumes and verify they not fail and have not been removed from the subsystems +detach_volume "$device0" "$uuid1" +detach_volume "$device1" "$uuid0" +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 1 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 1 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces[0].uuid') == "$uuid0" ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces[0].uuid') == "$uuid1" ]] +vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 +NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid1 +vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 +NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid0 + +# Detach volumes and verify they have been removed from the subsystems +detach_volume "$device0" "$uuid0" +detach_volume "$device1" "$uuid1" +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-0 | jq -r '.[0].namespaces | length') -eq 0 ]] +[[ $(rpc_cmd nvmf_get_subsystems nqn.2016-06.io.spdk:vfiouser-1 | jq -r '.[0].namespaces | length') -eq 0 ]] +NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-0 $uuid0 +NOT vm_check_subsys_volume $vm_no nqn.2016-06.io.spdk:vfiouser-1 $uuid1 + +# Detach volumes once again and verify they will not fail +detach_volume "$device0" "$uuid0" +detach_volume "$device1" "$uuid1" +detach_volume "$device0" "$uuid1" +detach_volume "$device1" "$uuid0" + +delete_device "$device0" +delete_device "$device1" + +cleanup +trap - SIGINT SIGTERM EXIT