bdev/nvme: Names are now explicitly assigned by the user

Names for the NVMe bdevs are now assigned by the user.
This means the same name will always be assigned to the
same device, even across restarts.

Change-Id: If9825ec9abcb5236b4671bc44a825e4f0d704fe3
Signed-off-by: Ben Walker <benjamin.walker@intel.com>

doc/iscsi/getting_started.md

@@ -102,10 +102,12 @@ the kernel to avoid interrupts and context switching.
   # NVMe Device Whitelist
   # Users may specify which NVMe devices to claim by their transport id.
   # See spdk_nvme_transport_id_parse() in spdk/nvme.h for the correct format.
-  # The devices will be assigned names in the format NvmeXnY, where X starts at 0 and
-  # increases by 1 for each entry and Y is the namespace id, which starts at 1.
-  TransportID "trtype:PCIe traddr:0000:00:00.0"
-  TransportID "trtype:PCIe traddr:0000:01:00.0"
+  # The second argument is the assigned name, which can be referenced from
+  # other sections in the configuration file. For NVMe devices, a namespace
+  # is automatically appended to each name in the format <YourName>nY, where
+  # Y is the NSID (starts at 1).
+  TransportID "trtype:PCIe traddr:0000:00:00.0" Nvme0
+  TransportID "trtype:PCIe traddr:0000:01:00.0" Nvme1
 
   # The number of attempts per I/O when an I/O fails. Do not include
   # this key to get the default behavior.

etc/spdk/iscsi.conf.in

@@ -90,10 +90,12 @@
   # NVMe Device Whitelist
   # Users may specify which NVMe devices to claim by their transport id.
   # See spdk_nvme_transport_id_parse() in spdk/nvme.h for the correct format.
-  # The devices will be assigned names in the format NvmeXnY, where X starts at 0 and
-  # increases by 1 for each entry and Y is the namespace id, which starts at 1.
-  TransportID "trtype:PCIe traddr:0000:00:00.0"
-  TransportID "trtype:PCIe traddr:0000:01:00.0"
+  # The second argument is the assigned name, which can be referenced from
+  # other sections in the configuration file. For NVMe devices, a namespace
+  # is automatically appended to each name in the format <YourName>nY, where
+  # Y is the NSID (starts at 1).
+  TransportID "trtype:PCIe traddr:0000:00:00.0" Nvme0
+  TransportID "trtype:PCIe traddr:0000:01:00.0" Nvme1
 
   # The number of attempts per I/O when an I/O fails. Do not include
   # this key to get the default behavior.
@@ -174,5 +176,5 @@
   AuthMethod Auto
   AuthGroup AuthGroup1
   UseDigest Auto
-  LUN0 Nvme0
+  LUN0 Nvme0n1
   QueueDepth 32

etc/spdk/nvmf.conf.in

@@ -74,11 +74,24 @@
   AcceptorPollRate 10000
 
 [Nvme]
+  # NVMe Device Whitelist
+  # Users may specify which NVMe devices to claim by their transport id.
+  # See spdk_nvme_transport_id_parse() in spdk/nvme.h for the correct format.
+  # The second argument is the assigned name, which can be referenced from
+  # other sections in the configuration file. For NVMe devices, a namespace
+  # is automatically appended to each name in the format <YourName>nY, where
+  # Y is the NSID (starts at 1).
+  TransportId "trtype:PCIe traddr:0000:00:00.0" Nvme0
+  TransportId "trtype:PCIe traddr:0000:01:00.0" Nvme1
+
+  # The number of attempts per I/O when an I/O fails. Do not include
+  # this key to get the default behavior.
+  NvmeRetryCount 4
   # Registers the application to receive timeout callback and to reset the controller.
   ResetControllerOnTimeout Yes
   # Timeout value.
   NvmeTimeoutValue 30
-  # Set how often the admin queue is polled for asynchronous events
+  # Set how often the admin queue is polled for asynchronous events.
   # Units in microseconds.
   AdminPollRate 100000
 
@@ -134,6 +147,8 @@
   Listen RDMA 192.168.2.21:4420
   Host nqn.2016-06.io.spdk:init
   SN SPDK00000000000001
+  Namespace Nvme0n1
+  Namespace Nvme1n1
   Namespace Malloc0
   Namespace Malloc1
   Namespace AIO0

etc/spdk/vhost.conf.in

@@ -69,10 +69,12 @@
   # NVMe Device Whitelist
   # Users may specify which NVMe devices to claim by their transport id.
   # See spdk_nvme_transport_id_parse() in spdk/nvme.h for the correct format.
-  # The devices will be assigned names in the format NvmeXnY, where X starts at 0 and
-  # increases by 1 for each entry and Y is the namespace id, which starts at 1.
-  TransportID "trtype:PCIe traddr:0000:00:00.0"
-  TransportID "trtype:PCIe traddr:0000:01:00.0"
+  # The second argument is the assigned name, which can be referenced from
+  # other sections in the configuration file. For NVMe devices, a namespace
+  # is automatically appended to each name in the format <YourName>nY, where
+  # Y is the NSID (starts at 1).
+  TransportID "trtype:PCIe traddr:0000:00:00.0" Nvme0
+  TransportID "trtype:PCIe traddr:0000:01:00.0" Nvme1
 
   # The number of attempts per I/O when an I/O fails. Do not include
   # this key to get the default behavior.

lib/bdev/nvme/blockdev_nvme.c

@@ -63,13 +63,13 @@ struct nvme_ctrlr {
 	 */
 	struct spdk_nvme_ctrlr		*ctrlr;
 	struct spdk_nvme_transport_id	trid;
+	const char			*name;
+	int				ref;
 
 	struct spdk_poller		*adminq_timer_poller;
 
 	/** linked list pointer for device list */
 	TAILQ_ENTRY(nvme_ctrlr)	tailq;
-	int				id;
-	int				ref;
 };
 
 struct nvme_bdev {
@@ -108,10 +108,9 @@ enum data_direction {
 struct nvme_probe_ctx {
 	size_t count;
 	struct spdk_nvme_transport_id trids[NVME_MAX_CONTROLLERS];
+	const char *names[NVME_MAX_CONTROLLERS];
 };
 
-static int nvme_controller_index = 0;
-static int num_controllers = -1;
 static int g_reset_controller_on_timeout = 0;
 static int g_timeout = 0;
 static int g_nvme_adminq_poll_timeout_us = 0;
@@ -123,7 +122,7 @@ static pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
 static TAILQ_HEAD(, nvme_ctrlr)	g_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_ctrlrs);
 static TAILQ_HEAD(, nvme_bdev) g_nvme_bdevs = TAILQ_HEAD_INITIALIZER(g_nvme_bdevs);
 
-static void nvme_ctrlr_create_bdevs(struct nvme_ctrlr *nvme_ctrlr, int ctrlr_id);
+static void nvme_ctrlr_create_bdevs(struct nvme_ctrlr *nvme_ctrlr);
 static int bdev_nvme_library_init(void);
 static void bdev_nvme_library_fini(void);
 static int bdev_nvme_queue_cmd(struct nvme_bdev *bdev, struct spdk_nvme_qpair *qpair,
@@ -580,28 +579,44 @@ static void
 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 	  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
 {
-	struct nvme_ctrlr *dev;
+	struct nvme_ctrlr *nvme_ctrlr;
+	struct nvme_probe_ctx *ctx = cb_ctx;
+	const char *name = NULL;
+	size_t i;
 
-	dev = malloc(sizeof(struct nvme_ctrlr));
-	if (dev == NULL) {
-		SPDK_ERRLOG("Failed to allocate device struct\n");
+	for (i = 0; i < ctx->count; i++) {
+		if (spdk_nvme_transport_id_compare(trid, &ctx->trids[i]) == 0) {
+			name = strdup(ctx->names[i]);
+			break;
+		}
+	}
+
+	if (!name) {
+		SPDK_ERRLOG("Failed to assign name to NVMe device\n");
 		return;
 	}
 
-	dev->adminq_timer_poller = NULL;
-	dev->ctrlr = ctrlr;
-	dev->ref = 0;
-	dev->trid = *trid;
-	dev->id = nvme_controller_index++;
+	nvme_ctrlr = calloc(1, sizeof(*nvme_ctrlr));
+	if (nvme_ctrlr == NULL) {
+		SPDK_ERRLOG("Failed to allocate device struct\n");
+		free((void *)name);
+		return;
+	}
 
-	nvme_ctrlr_create_bdevs(dev, dev->id);
+	nvme_ctrlr->adminq_timer_poller = NULL;
+	nvme_ctrlr->ctrlr = ctrlr;
+	nvme_ctrlr->ref = 0;
+	nvme_ctrlr->trid = *trid;
+	nvme_ctrlr->name = name;
 
-	spdk_poller_register(&dev->adminq_timer_poller, bdev_nvme_poll_adminq, ctrlr,
+	nvme_ctrlr_create_bdevs(nvme_ctrlr);
+
+	spdk_poller_register(&nvme_ctrlr->adminq_timer_poller, bdev_nvme_poll_adminq, ctrlr,
 			     spdk_app_get_current_core(), g_nvme_adminq_poll_timeout_us);
 
 	spdk_io_device_register(ctrlr, bdev_nvme_create_cb, bdev_nvme_destroy_cb,
 				sizeof(struct nvme_io_channel));
-	TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, dev, tailq);
+	TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, nvme_ctrlr, tailq);
 
 	if (g_reset_controller_on_timeout) {
 		spdk_nvme_ctrlr_register_timeout_callback(ctrlr, g_timeout,
@@ -661,6 +676,7 @@ blockdev_nvme_hotplug(void *arg)
 
 int
 spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
+		      const char *base_name,
 		      const char **names, size_t *count)
 {
 	struct nvme_probe_ctx	probe_ctx;
@@ -675,6 +691,7 @@ spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
 
 	probe_ctx.count = 1;
 	probe_ctx.trids[0] = *trid;
+	probe_ctx.names[0] = base_name;
 	if (spdk_nvme_probe(trid, &probe_ctx, probe_cb, attach_cb, NULL)) {
 		SPDK_ERRLOG("Failed to probe for new devices\n");
 		return -1;
@@ -697,8 +714,8 @@ spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
 				names[j] = nvme_bdev->disk.name;
 				j++;
 			} else {
-				assert(false);
-				break;
+				SPDK_ERRLOG("Unable to return all names of created bdevs\n");
+				return -1;
 			}
 		}
 	}
@@ -712,7 +729,8 @@ bdev_nvme_library_init(void)
 {
 	struct spdk_conf_section *sp;
 	const char *val;
-	int i, rc;
+	int rc;
+	size_t i;
 	struct nvme_probe_ctx probe_ctx = {};
 
 	sp = spdk_conf_find_section(NULL, "Nvme");
@@ -736,6 +754,15 @@ bdev_nvme_library_init(void)
 			SPDK_ERRLOG("Unable to parse TransportID: %s\n", val);
 			return -1;
 		}
+
+		val = spdk_conf_section_get_nmval(sp, "TransportID", i, 1);
+		if (val == NULL) {
+			SPDK_ERRLOG("No name provided for TransportID\n");
+			return -1;
+		}
+
+		probe_ctx.names[i] = val;
+
 		probe_ctx.count++;
 	}
 
@@ -787,7 +814,7 @@ bdev_nvme_library_fini(void)
 }
 
 static void
-nvme_ctrlr_create_bdevs(struct nvme_ctrlr *nvme_ctrlr, int ctrlr_id)
+nvme_ctrlr_create_bdevs(struct nvme_ctrlr *nvme_ctrlr)
 {
 	struct nvme_bdev	*bdev;
 	struct spdk_nvme_ctrlr	*ctrlr = nvme_ctrlr->ctrlr;
@@ -816,7 +843,7 @@ nvme_ctrlr_create_bdevs(struct nvme_ctrlr *nvme_ctrlr, int ctrlr_id)
 		nvme_ctrlr->ref++;
 
 		snprintf(bdev->disk.name, SPDK_BDEV_MAX_NAME_LENGTH,
-			 "Nvme%dn%d", ctrlr_id, spdk_nvme_ns_get_id(ns));
+			 "%sn%d", nvme_ctrlr->name, spdk_nvme_ns_get_id(ns));
 		snprintf(bdev->disk.product_name, SPDK_BDEV_MAX_PRODUCT_NAME_LENGTH,
 			 "NVMe disk");
 
@@ -972,15 +999,7 @@ bdev_nvme_unmap(struct nvme_bdev *nbdev, struct spdk_io_channel *ch,
 static void
 bdev_nvme_get_spdk_running_config(FILE *fp)
 {
-	fprintf(fp,
-		"\n"
-		"[Nvme]\n");
-	if (num_controllers != -1) {
-		fprintf(fp, "  NumControllers %d\n", num_controllers);
-	}
-	fprintf(fp, "  # Set how often the admin queue is polled for asynchronous events.\n"
-		"  # Units in microseconds.\n"
-		"  AdminPollRate %d\n", g_nvme_adminq_poll_timeout_us);
+	/* TODO */
 }
 
 SPDK_LOG_REGISTER_TRACE_FLAG("bdev_nvme", SPDK_TRACE_BDEV_NVME)

lib/bdev/nvme/blockdev_nvme.h

@@ -41,6 +41,7 @@
 #define NVME_MAX_CONTROLLERS 16
 
 int spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
+			  const char *base_name,
 			  const char **names, size_t *count);
 
 #endif // SPDK_BLOCKDEV_NVME_H

lib/bdev/nvme/blockdev_nvme_rpc.c

@@ -42,6 +42,7 @@
 #include "spdk_internal/log.h"
 
 struct rpc_construct_nvme {
+	char *name;
 	char *trtype;
 	char *adrfam;
 	char *traddr;
@@ -52,6 +53,7 @@ struct rpc_construct_nvme {
 static void
 free_rpc_construct_nvme(struct rpc_construct_nvme *req)
 {
+	free(req->name);
 	free(req->trtype);
 	free(req->adrfam);
 	free(req->traddr);
@@ -60,6 +62,7 @@ free_rpc_construct_nvme(struct rpc_construct_nvme *req)
 }
 
 static const struct spdk_json_object_decoder rpc_construct_nvme_decoders[] = {
+	{"name", offsetof(struct rpc_construct_nvme, name), spdk_json_decode_string},
 	{"trtype", offsetof(struct rpc_construct_nvme, trtype), spdk_json_decode_string},
 	{"traddr", offsetof(struct rpc_construct_nvme, traddr), spdk_json_decode_string},
 
@@ -79,7 +82,7 @@ spdk_rpc_construct_nvme_bdev(struct spdk_jsonrpc_server_conn *conn,
 	struct spdk_json_write_ctx *w;
 	struct spdk_nvme_transport_id trid = {};
 	const char *names[NVME_MAX_BLOCKDEVS_PER_RPC];
-	size_t count = 0;
+	size_t count;
 	size_t i;
 	int rc;
 
@@ -120,13 +123,12 @@ spdk_rpc_construct_nvme_bdev(struct spdk_jsonrpc_server_conn *conn,
 	}
 
 	count = NVME_MAX_BLOCKDEVS_PER_RPC;
-	if (spdk_bdev_nvme_create(&trid, names, &count)) {
+	if (spdk_bdev_nvme_create(&trid, req.name, names, &count)) {
 		goto invalid;
 	}
 
-	free_rpc_construct_nvme(&req);
-
 	if (id == NULL) {
+		free_rpc_construct_nvme(&req);
 		return;
 	}
 
@@ -137,6 +139,9 @@ spdk_rpc_construct_nvme_bdev(struct spdk_jsonrpc_server_conn *conn,
 	}
 	spdk_json_write_array_end(w);
 	spdk_jsonrpc_end_result(conn, w);
+
+	free_rpc_construct_nvme(&req);
+
 	return;
 
 invalid:

scripts/gen_nvme.sh

@@ -17,6 +17,6 @@ esac
 echo "[Nvme]"
 i=0
 for bdf in $bdfs; do
-        echo "  TransportID \"trtype:PCIe traddr:$bdf\""
+        echo "  TransportID \"trtype:PCIe traddr:$bdf\" Nvme$i"
         let i=i+1
 done

scripts/rpc.py

@@ -173,7 +173,8 @@ p.add_argument('fname', help='Path to device or file (ex: /dev/sda)')
 p.set_defaults(func=construct_aio_bdev)
 
 def construct_nvme_bdev(args):
-    params = {'trtype': args.trtype,
+    params = {'name': args.name,
+              'trtype': args.trtype,
               'traddr': args.traddr}
 
     if args.adrfam:
@@ -188,6 +189,7 @@ def construct_nvme_bdev(args):
     jsonrpc_call('construct_nvme_bdev', params)
 
 p = subparsers.add_parser('construct_nvme_bdev', help='Add bdev with nvme backend')
+p.add_argument('-b', '--name', help="Name of the bdev", required=True)
 p.add_argument('-t', '--trtype', help='NVMe-oF target trtype: e.g., rdma, pcie', required=True)
 p.add_argument('-a', '--traddr', help='NVMe-oF target address: e.g., an ip address or BDF', required=True)
 p.add_argument('-f', '--adrfam', help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')

test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

@@ -53,7 +53,7 @@ echo "iSCSI target has started."
 echo "Creating an iSCSI target node."
 $rpc_py -p 5261 add_portal_group 1 $TARGET_IP:$ISCSI_PORT
 $rpc_py -p 5261 add_initiator_group 1 ALL $INITIATOR_IP/32
-$rpc_py -p 5261 construct_nvme_bdev -t "rdma" -f "ipv4" -a 192.168.100.8 -s $NVMF_PORT -n nqn.2016-06.io.spdk:cnode1
+$rpc_py -p 5261 construct_nvme_bdev -b "Nvme0" -t "rdma" -f "ipv4" -a 192.168.100.8 -s $NVMF_PORT -n nqn.2016-06.io.spdk:cnode1
 $rpc_py -p 5261 construct_target_node Target1 Target1_alias 'Nvme0n1:0' '1:1' 64 1 0 0 0
 sleep 1
 

test/iscsi_tgt/rpc_config/rpc_config.py

@@ -399,7 +399,7 @@ def verify_add_nvme_bdev_rpc_methods(rpc_py):
     output = check_output(["lspci", "-mm", "-nn"])
     addrs = re.findall('^([0-9]{2}:[0-9]{2}.[0-9]) "Non-Volatile memory controller \[0108\]".*-p02', output, re.MULTILINE)
     for addr in addrs:
-        ctrlr_address = "-t pcie -a 0000:{}".format(addr)
+        ctrlr_address = "-b Nvme0 -t pcie -a 0000:{}".format(addr)
         rpc.construct_nvme_bdev(ctrlr_address)
         print "add nvme device passed first time"
         test_pass = 0