bdev/nvme: RPC call can now specify remote NVMe-oF devices

Change-Id: Ief3cf71489f0725249d51851b8d3270cb4e034db
Signed-off-by: Ziye Yang <ziye.yang@intel.com>

autotest.sh

@@ -135,6 +135,7 @@ if [ $(uname -s) = Linux ] && [ $SPDK_TEST_ISCSI -eq 1 ]; then
 	fi
 	run_test ./test/iscsi_tgt/ext4test/ext4test.sh
 	run_test ./test/iscsi_tgt/rbd/rbd.sh
+	run_test ./test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh
 	timing_exit iscsi_tgt
 
 	run_test test/lib/iscsi/iscsi.sh

lib/bdev/nvme/blockdev_nvme.c

@@ -543,6 +543,7 @@ probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 	}
 
 	if (!claim_device) {
+		SPDK_ERRLOG("Not claiming device at %s\n", trid->traddr);
 		return false;
 	}
 
@@ -675,11 +676,13 @@ spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
 	probe_ctx.count = 1;
 	probe_ctx.trids[0] = *trid;
 	if (spdk_nvme_probe(trid, &probe_ctx, probe_cb, attach_cb, NULL)) {
+		SPDK_ERRLOG("Failed to probe for new devices\n");
 		return -1;
 	}
 
 	nvme_ctrlr = nvme_ctrlr_get(trid);
 	if (!nvme_ctrlr) {
+		SPDK_ERRLOG("Failed to find new NVMe controller\n");
 		return -1;
 	}
 

lib/bdev/nvme/blockdev_nvme_rpc.c

@@ -34,23 +34,38 @@
 #include <string.h>
 
 #include "blockdev_nvme.h"
+
+#include "spdk/string.h"
 #include "spdk/rpc.h"
 #include "spdk/util.h"
 
 #include "spdk_internal/log.h"
 
 struct rpc_construct_nvme {
-	char *pci_address;
+	char *trtype;
+	char *adrfam;
+	char *traddr;
+	char *trsvcid;
+	char *subnqn;
 };
 
 static void
 free_rpc_construct_nvme(struct rpc_construct_nvme *req)
 {
-	free(req->pci_address);
+	free(req->trtype);
+	free(req->adrfam);
+	free(req->traddr);
+	free(req->trsvcid);
+	free(req->subnqn);
 }
 
 static const struct spdk_json_object_decoder rpc_construct_nvme_decoders[] = {
-	{"pci_address", offsetof(struct rpc_construct_nvme, pci_address), spdk_json_decode_string},
+	{"trtype", offsetof(struct rpc_construct_nvme, trtype), spdk_json_decode_string},
+	{"traddr", offsetof(struct rpc_construct_nvme, traddr), spdk_json_decode_string},
+
+	{"adrfam", offsetof(struct rpc_construct_nvme, adrfam), spdk_json_decode_string, true},
+	{"trsvcid", offsetof(struct rpc_construct_nvme, trsvcid), spdk_json_decode_string, true},
+	{"subnqn", offsetof(struct rpc_construct_nvme, subnqn), spdk_json_decode_string, true},
 };
 
 #define NVME_MAX_BLOCKDEVS_PER_RPC 32
@@ -66,16 +81,43 @@ spdk_rpc_construct_nvme_bdev(struct spdk_jsonrpc_server_conn *conn,
 	const char *names[NVME_MAX_BLOCKDEVS_PER_RPC];
 	size_t count = 0;
 	size_t i;
+	int rc;
 
 	if (spdk_json_decode_object(params, rpc_construct_nvme_decoders,
 				    SPDK_COUNTOF(rpc_construct_nvme_decoders),
 				    &req)) {
-		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n");
+		SPDK_ERRLOG("spdk_json_decode_object failed\n");
 		goto invalid;
 	}
 
-	trid.trtype = SPDK_NVME_TRANSPORT_PCIE;
+	/* Parse trtype */
-	snprintf(trid.traddr, sizeof(trid.traddr), "%s", req.pci_address);
+	rc = spdk_nvme_transport_id_parse_trtype(&trid.trtype, req.trtype);
+	if (rc < 0) {
+		SPDK_ERRLOG("Failed to parse trtype: %s\n", req.trtype);
+		goto invalid;
+	}
+
+	/* Parse traddr */
+	snprintf(trid.traddr, sizeof(trid.traddr), "%s", req.traddr);
+
+	/* Parse adrfam */
+	if (req.adrfam) {
+		rc = spdk_nvme_transport_id_parse_adrfam(&trid.adrfam, req.adrfam);
+		if (rc < 0) {
+			SPDK_ERRLOG("Failed to parse adrfam: %s\n", req.adrfam);
+			goto invalid;
+		}
+	}
+
+	/* Parse trsvcid */
+	if (req.trsvcid) {
+		snprintf(trid.trsvcid, sizeof(trid.trsvcid), "%s", req.trsvcid);
+	}
+
+	/* Parse subnqn */
+	if (req.subnqn) {
+		snprintf(trid.subnqn, sizeof(trid.subnqn), "%s", req.subnqn);
+	}
 
 	count = NVME_MAX_BLOCKDEVS_PER_RPC;
 	if (spdk_bdev_nvme_create(&trid, names, &count)) {

scripts/rpc.py

@@ -173,10 +173,26 @@ p.add_argument('fname', help='Path to device or file (ex: /dev/sda)')
 p.set_defaults(func=construct_aio_bdev)
 
 def construct_nvme_bdev(args):
-    params = {'pci_address': args.pci_address}
+    params = {'trtype': args.trtype,
-    print_array(jsonrpc_call('construct_nvme_bdev', params))
+              'traddr': args.traddr}
+
+    if args.adrfam:
+        params['adrfam'] = args.adrfam
+
+    if args.trsvcid:
+        params['trsvcid'] = args.trsvcid
+
+    if args.subnqn:
+        params['subnqn'] = args.subnqn
+
+    jsonrpc_call('construct_nvme_bdev', params)
+
 p = subparsers.add_parser('construct_nvme_bdev', help='Add bdev with nvme backend')
-p.add_argument('pci_address', help='PCI address domain:bus:device.function')
+p.add_argument('-t', '--trtype', help='NVMe-oF target trtype: e.g., rdma, pcie', required=True)
+p.add_argument('-a', '--traddr', help='NVMe-oF target address: e.g., an ip address or BDF', required=True)
+p.add_argument('-f', '--adrfam', help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')
+p.add_argument('-s', '--trsvcid', help='NVMe-oF target trsvcid: e.g., a port number')
+p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn')
 p.set_defaults(func=construct_nvme_bdev)
 
 def construct_rbd_bdev(args):

test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+set -e
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$testdir/../../..
+source $rootdir/scripts/autotest_common.sh
+source $rootdir/test/nvmf/common.sh
+
+if [ -z "$TARGET_IP" ]; then
+	echo "TARGET_IP not defined in environment"
+	exit 1
+fi
+
+if [ -z "$INITIATOR_IP" ]; then
+	echo "INITIATOR_IP not defined in environment"
+	exit 1
+fi
+
+if ! rdma_nic_available; then
+	echo "no NIC for nvmf test"
+	exit 0
+fi
+
+rpc_py="python $rootdir/scripts/rpc.py"
+fio_py="python $rootdir/scripts/fio.py"
+
+ISCSI_PORT=3260
+NVMF_PORT=4420
+
+timing_enter nvme_remote
+
+# Start the NVMf target
+$rootdir/app/nvmf_tgt/nvmf_tgt -c $rootdir/test/nvmf/nvmf.conf -m 0x2 -p 1 -s 512 &
+nvmfpid=$!
+echo "NVMf target launched. pid: $nvmfpid"
+trap "killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT
+waitforlisten $nvmfpid 5260
+echo "NVMf target has started."
+bdevs=$($rpc_py construct_malloc_bdev 64 512)
+$rpc_py construct_nvmf_subsystem Virtual nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -s SPDK00000000000001 -n "$bdevs"
+echo "NVMf subsystem created."
+
+# Start the iSCSI target
+$rootdir/app/iscsi_tgt/iscsi_tgt -c $testdir/iscsi.conf -m 0x1 -p 0 -s 512 &
+iscsipid=$!
+echo "iSCSI target launched. pid: $iscsipid"
+trap "process_core; killprocess $iscsipid; exit 1" SIGINT SIGTERM EXIT
+# The configuration file for the iSCSI target told it to use port 5261 for RPC
+waitforlisten $iscsipid 5261
+echo "iSCSI target has started."
+
+echo "Creating an iSCSI target node."
+$rpc_py -p 5261 add_portal_group 1 $TARGET_IP:$ISCSI_PORT
+$rpc_py -p 5261 add_initiator_group 1 ALL $INITIATOR_IP/32
+$rpc_py -p 5261 construct_nvme_bdev -t "rdma" -f "ipv4" -a 192.168.100.8 -s $NVMF_PORT -n nqn.2016-06.io.spdk:cnode1
+$rpc_py -p 5261 construct_target_node Target1 Target1_alias 'Nvme0n1:0' '1:1' 64 1 0 0 0
+sleep 1
+
+echo "Logging in to iSCSI target."
+iscsiadm -m discovery -t sendtargets -p $TARGET_IP:$ISCSI_PORT
+iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT
+trap "iscsicleanup; process_core; killprocess $pid; exit 1" SIGINT SIGTERM EXIT
+sleep 1
+
+echo "Running FIO"
+$fio_py 4096 1 randrw 1 verify
+
+rm -f ./local-job0-0-verify.state
+trap - SIGINT SIGTERM EXIT
+
+iscsicleanup
+killprocess $iscsipid
+$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1
+killprocess $nvmfpid
+
+timing_exit nvme_remote

test/iscsi_tgt/nvme_remote/iscsi.conf

@@ -0,0 +1,15 @@
+[Global]
+  LogFacility "local7"
+
+[iSCSI]
+  NodeBase "iqn.2016-06.io.spdk"
+  AuthFile /usr/local/etc/spdk/auth.conf
+  Timeout 30
+  DiscoveryAuthMethod Auto
+  MaxSessions 16
+  ImmediateData Yes
+  ErrorRecoveryLevel 0
+
+[Rpc]
+  Enable Yes
+  Listen 127.0.0.1:5261

test/iscsi_tgt/rpc_config/rpc_config.py

@@ -399,7 +399,7 @@ def verify_add_nvme_bdev_rpc_methods(rpc_py):
     output = check_output(["lspci", "-mm", "-nn"])
     addrs = re.findall('^([0-9]{2}:[0-9]{2}.[0-9]) "Non-Volatile memory controller \[0108\]".*-p02', output, re.MULTILINE)
     for addr in addrs:
-        ctrlr_address = "0000:{}".format(addr)
+        ctrlr_address = "-t pcie -a 0000:{}".format(addr)
         rpc.construct_nvme_bdev(ctrlr_address)
         print "add nvme device passed first time"
         test_pass = 0