test/sma: add discovery test

The test checks that it's possible to attach remote volumes through
discovery service.  The tests starts up three SPDK instances: one is being
managed by SMA, while the other two are acting as remote storage nodes.
The test verifies that volumes can be successfully discvered and
attached to a device and that the connections to the discovery
subsystems are refcounted correctly.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I3488b822880b99ef9cfd70e03de3ed9054c13901
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12414
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

test/common/skipped_tests.txt

@@ -93,3 +93,4 @@ zoned_fio
 # SMA tests - disabled in CI for now
 sma_nvmf_tcp
 sma_plugins
+sma_discovery

test/sma/common.sh

@@ -10,3 +10,10 @@ function sma_waitforlisten() {
 	done
 	return 1
 }
+
+function uuid2base64() {
+	python <<- EOF
+		import base64, uuid
+		print(base64.b64encode(uuid.UUID("$1").bytes).decode())
+	EOF
+}

test/sma/discovery.sh

@@ -0,0 +1,404 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../..")
+
+source "$rootdir/test/common/autotest_common.sh"
+source "$testdir/common.sh"
+
+rpc_py="$rootdir/scripts/rpc.py"
+sma_py="$rootdir/scripts/sma-client.py"
+
+t1sock='/var/tmp/spdk.sock1'
+t2sock='/var/tmp/spdk.sock2'
+invalid_port=8008
+t1dscport=8009
+t2dscport1=8010
+t2dscport2=8011
+t1nqn='nqn.2016-06.io.spdk:node1'
+t2nqn='nqn.2016-06.io.spdk:node2'
+hostnqn='nqn.2016-06.io.spdk:host0'
+
+function cleanup() {
+	killprocess $smapid
+	killprocess $tgtpid
+	killprocess $t1pid
+	killprocess $t2pid
+}
+
+function format_endpoints() {
+	local eps=("$@")
+	for ((i = 0; i < ${#eps[@]}; i++)); do
+		cat <<- EOF
+			{
+				"trtype": "tcp",
+				"traddr": "127.0.0.1",
+				"trsvcid": "${eps[i]}"
+			}
+		EOF
+		if ! ((i + 1 == ${#@})); then
+			echo ,
+		fi
+	done
+}
+
+function format_volume() {
+	local volume_id=$1
+	shift
+
+	cat <<- EOF
+		"volume": {
+			"volume_id": "$(uuid2base64 $volume_id)",
+			"nvmf": {
+				"hostnqn": "$hostnqn",
+				"discovery": {
+					"discovery_endpoints": [
+						$(format_endpoints "$@")
+					]
+				}
+			}
+		}
+	EOF
+}
+
+function create_device() {
+	local nqn=$1
+	local volume_id=$2
+	local volume=
+
+	shift
+	if [[ -n "$volume_id" ]]; then
+		volume="$(format_volume "$@"),"
+	fi
+
+	$sma_py <<- EOF
+		{
+			"method": "CreateDevice",
+			"params": {
+				$volume
+				"nvmf_tcp": {
+					"subnqn": "$nqn",
+					"adrfam": "ipv4",
+					"traddr": "127.0.0.1",
+					"trsvcid": "4419"
+				}
+			}
+		}
+	EOF
+}
+
+function delete_device() {
+	$sma_py <<- EOF
+		{
+			"method": "DeleteDevice",
+			"params": {
+				"handle": "$1"
+			}
+		}
+	EOF
+}
+
+function attach_volume() {
+	local device_id=$1
+
+	shift
+	$sma_py <<- EOF
+		{
+			"method": "AttachVolume",
+			"params": {
+				$(format_volume "$@"),
+				"device_handle": "$device_id"
+			}
+		}
+	EOF
+}
+
+function detach_volume() {
+	$sma_py <<- EOF
+		{
+			"method": "DetachVolume",
+			"params": {
+				"device_handle": "$1",
+				"volume_id": "$(uuid2base64 $2)"
+			}
+		}
+	EOF
+}
+
+trap "cleanup; exit 1" SIGINT SIGTERM EXIT
+
+# Start two remote targets
+$rootdir/build/bin/spdk_tgt -r $t1sock &
+t1pid=$!
+$rootdir/build/bin/spdk_tgt -r $t2sock &
+t2pid=$!
+
+# One target that the SMA will configure
+$rootdir/build/bin/spdk_tgt &
+tgtpid=$!
+
+# And finally the SMA itself
+$rootdir/scripts/sma.py -c <(
+	cat <<- EOF
+		discovery_timeout: 5
+		devices:
+		  - name: 'nvmf_tcp'
+	EOF
+) &
+smapid=$!
+
+waitforlisten $t1pid
+waitforlisten $t2pid
+
+# Prepare the targets.  The first one has a single subsystem with a single volume and a single
+# discovery listener.  The second one also has a single subsystem, but has two volumes attached to
+# it and has two discovery listeners.
+t1uuid=$(uuidgen)
+t2uuid=$(uuidgen)
+t2uuid2=$(uuidgen)
+
+$rpc_py -s $t1sock <<- EOF
+	nvmf_create_transport -t tcp
+	bdev_null_create null0 128 4096 -u $t1uuid
+	nvmf_create_subsystem $t1nqn -s SPDK00000000000001 -d SPDK_Controller1
+	nvmf_subsystem_add_host $t1nqn $hostnqn
+	nvmf_subsystem_add_ns $t1nqn $t1uuid
+	nvmf_subsystem_add_listener $t1nqn -t tcp -a 127.0.0.1 -s 4420
+	nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t1dscport
+EOF
+
+$rpc_py -s $t2sock <<- EOF
+	nvmf_create_transport -t tcp
+	bdev_null_create null0 128 4096 -u $t2uuid
+	bdev_null_create null1 128 4096 -u $t2uuid2
+	nvmf_create_subsystem $t2nqn -s SPDK00000000000001 -d SPDK_Controller1
+	nvmf_subsystem_add_host $t2nqn $hostnqn
+	nvmf_subsystem_add_ns $t2nqn $t2uuid
+	nvmf_subsystem_add_ns $t2nqn $t2uuid2
+	nvmf_subsystem_add_listener $t2nqn -t tcp -a 127.0.0.1 -s 4421
+	nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport1
+	nvmf_subsystem_add_listener discovery -t tcp -a 127.0.0.1 -s $t2dscport2
+EOF
+
+# Wait until the SMA starts listening
+sma_waitforlisten
+
+localnqn='nqn.2016-06.io.spdk:local0'
+
+# Create a device
+device_id=$(create_device $localnqn | jq -r '.handle')
+
+# Check that it's been created
+$rpc_py nvmf_get_subsystems $localnqn
+
+# Attach a volume specifying both targets
+attach_volume $device_id $t1uuid $t1dscport $t2dscport1
+
+# Check that a connection has been made to discovery services on both targets
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]
+
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1
+
+# Check that the volume was attached to the device
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]
+
+# Attach the other volume, this time specify only single target
+attach_volume $device_id $t2uuid $t2dscport1
+
+# Check that both volumes are attached to the device
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]]
+$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid
+$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid
+
+# Detach the first volume
+detach_volume $device_id $t1uuid
+
+# Check that there's a connection to a single target now (because we've only specified a single
+# target when connecting the other volume).
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1
+# Check that the volume was actually removed
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]]
+
+# Detach the other volume
+detach_volume $device_id $t2uuid
+
+# And verify it's gone too
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]]
+
+# Check that specifying an invalid volume UUID results in an error
+NOT attach_volume $device_id $(uuidgen) $t1dscport
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]]
+
+# Attach them again, this time both volumes specify both targets
+volumes=($t1uuid $t2uuid)
+for volume_id in "${volumes[@]}"; do
+	attach_volume $device_id $volume_id $t1dscport $t2dscport1
+done
+
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1
+$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t1uuid
+$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid
+
+# Detach one and see that both targets are still connected
+detach_volume $device_id $t1uuid
+
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 2 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t2dscport1
+
+# Delete the device and verify that this also causes the volumes to be disconnected
+delete_device $device_id
+
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]
+NOT $rpc_py nvmf_get_subsystems $localnqn
+
+# Create a device and attach a volume immediately
+device_id=$(create_device $localnqn $t1uuid $t1dscport | jq -r '.handle')
+
+# Verify that there's a connection to the target and the volume is attached to the device
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]
+
+# Make sure it's also possible to detach it
+detach_volume $device_id $t1uuid
+
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 0 ]]
+
+# Check that discovery referrals work correctly
+attach_volume $device_id $t2uuid $t2dscport1 $t2dscport2
+
+# Check that only a single connection has been made
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t2uuid" ]]
+
+# Add the other volume from the same target/subsystem, but use a single discovery endpoint
+attach_volume $device_id $t2uuid2 $t2dscport2
+
+# Check that the volume was attached to the subsystem, but no extra connection has been made
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 2 ]]
+$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid
+$rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[].uuid' | grep $t2uuid2
+
+# Reset the device
+delete_device $device_id
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 0 ]]
+
+device_id=$(create_device $localnqn | jq -r '.handle')
+
+# Check subsystem NQN verification, start with a valid one
+$sma_py <<- EOF
+	{
+		"method": "AttachVolume",
+		"params": {
+			"volume": {
+				"volume_id": "$(uuid2base64 $t1uuid)",
+				"nvmf": {
+					"hostnqn": "$hostnqn",
+					"subnqn": "$t1nqn",
+					"discovery": {
+						"discovery_endpoints": [
+							{
+								"trtype": "tcp",
+								"traddr": "127.0.0.1",
+								"trsvcid": "$t1dscport"
+							}
+						]
+					}
+				}
+			},
+			"device_handle": "$device_id"
+		}
+	}
+EOF
+
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]
+
+# Then check incorrect subnqn
+NOT $sma_py <<- EOF
+	{
+		"method": "AttachVolume",
+		"params": {
+			"volume": {
+				"volume_id": "$(uuid2base64 $t2uuid)",
+				"nvmf": {
+					"hostnqn": "$hostnqn",
+					"subnqn": "${t2nqn}-invalid",
+					"discovery": {
+						"discovery_endpoints": [
+							{
+								"trtype": "tcp",
+								"traddr": "127.0.0.1",
+								"trsvcid": "$t2dscport1"
+							}
+						]
+					}
+				}
+			},
+			"device_handle": "$device_id"
+		}
+	}
+EOF
+
+# Verify the volume hasn't been attached
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]
+
+# Check incorrect hostnqn
+NOT $sma_py <<- EOF
+	{
+		"method": "AttachVolume",
+		"params": {
+			"volume": {
+				"volume_id": "$(uuid2base64 $t2uuid)",
+				"nvmf": {
+					"hostnqn": "${hostnqn}-invalid",
+					"discovery": {
+						"discovery_endpoints": [
+							{
+								"trtype": "tcp",
+								"traddr": "127.0.0.1",
+								"trsvcid": "$t2dscport1"
+							}
+						]
+					}
+				}
+			},
+			"device_handle": "$device_id"
+		}
+	}
+EOF
+
+# Verify that the volume wasn't attached
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces | length') -eq 1 ]]
+[[ $($rpc_py nvmf_get_subsystems $localnqn | jq -r '.[].namespaces[0].uuid') == "$t1uuid" ]]
+
+# Check that the the attach will fail if there's nobody listening on the discovery endpoint
+NOT attach_volume $device_id $(uuidgen) $invalid_port
+[[ $($rpc_py bdev_nvme_get_discovery_info | jq -r '. | length') -eq 1 ]]
+$rpc_py bdev_nvme_get_discovery_info | jq -r '.[].trid.trsvcid' | grep $t1dscport
+
+delete_device $device_id
+
+cleanup
+trap - SIGINT SIGTERM EXIT

test/sma/nvmf_tcp.sh

@@ -11,13 +11,6 @@ function cleanup() {
 	killprocess $smapid
 }
 
-function uuid2base64() {
-	python <<- EOF
-		import base64, uuid
-		print(base64.b64encode(uuid.UUID("$1").bytes).decode())
-	EOF
-}
-
 function create_device() {
 	"$rootdir/scripts/sma-client.py" <<- EOF
 		{

test/sma/sma.sh

@@ -7,3 +7,4 @@ source "$rootdir/test/common/autotest_common.sh"
 
 run_test "sma_nvmf_tcp" $testdir/nvmf_tcp.sh
 run_test "sma_plugins" $testdir/plugins.sh
+run_test "sma_discovery" $testdir/discovery.sh