nvmf: Add a channel map to the poll group

As of this patch, it is unused.

Change-Id: I15f42b8cc43e3792ae69c91f3911b7853d5b5664
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/388292
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/nvmf/nvmf.c

@@ -33,6 +33,7 @@
 
 #include "spdk/stdinc.h"
 
+#include "spdk/bdev.h"
 #include "spdk/conf.h"
 #include "spdk/io_channel.h"
 #include "spdk/nvmf.h"
@@ -82,6 +83,7 @@ spdk_nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf)
 	struct spdk_nvmf_tgt *tgt = io_device;
 	struct spdk_nvmf_poll_group *group = ctx_buf;
 	struct spdk_nvmf_transport *transport;
+	uint32_t sid;
 
 	TAILQ_INIT(&group->tgroups);
 
@@ -89,6 +91,23 @@ spdk_nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf)
 		spdk_nvmf_poll_group_add_transport(group, transport);
 	}
 
+	group->num_sgroups = tgt->max_sid;
+	group->sgroups = calloc(group->num_sgroups, sizeof(struct spdk_nvmf_subsystem_poll_group));
+	if (!group->sgroups) {
+		return -1;
+	}
+
+	for (sid = 0; sid < group->num_sgroups; sid++) {
+		struct spdk_nvmf_subsystem *subsystem;
+
+		subsystem = tgt->subsystems[sid];
+		if (!subsystem) {
+			continue;
+		}
+
+		spdk_nvmf_poll_group_add_subsystem(group, subsystem);
+	}
+
 	group->poller = spdk_poller_register(spdk_nvmf_poll_group_poll, group, 0);
 
 	return 0;
@@ -99,6 +118,8 @@ spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf)
 {
 	struct spdk_nvmf_poll_group *group = ctx_buf;
 	struct spdk_nvmf_transport_poll_group *tgroup, *tmp;
+	struct spdk_nvmf_subsystem_poll_group *sgroup;
+	uint32_t sid, nsid;
 
 	spdk_poller_unregister(&group->poller);
 
@@ -106,6 +127,21 @@ spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf)
 		TAILQ_REMOVE(&group->tgroups, tgroup, link);
 		spdk_nvmf_transport_poll_group_destroy(tgroup);
 	}
+
+	for (sid = 0; sid < group->num_sgroups; sid++) {
+		sgroup = &group->sgroups[sid];
+
+		for (nsid = 0; nsid < sgroup->num_channels; nsid++) {
+			if (sgroup->channels[nsid]) {
+				spdk_put_io_channel(sgroup->channels[nsid]);
+				sgroup->channels[nsid] = NULL;
+			}
+		}
+
+		free(sgroup->channels);
+	}
+
+	free(group->sgroups);
 }
 
 struct spdk_nvmf_tgt *
@@ -369,6 +405,69 @@ spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group,
 	return 0;
 }
 
+int
+spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
+				   struct spdk_nvmf_subsystem *subsystem)
+{
+	struct spdk_nvmf_subsystem_poll_group *sgroup;
+	struct spdk_nvmf_ns *ns;
+	uint32_t i;
+
+	if (subsystem->id >= group->num_sgroups) {
+		void *buf;
+
+		group->num_sgroups = subsystem->id + 1;
+		buf = realloc(group->sgroups, group->num_sgroups * sizeof(*sgroup));
+		if (!buf) {
+			return -ENOMEM;
+		}
+		group->sgroups = buf;
+	}
+
+	sgroup = &group->sgroups[subsystem->id];
+
+	sgroup->num_channels = subsystem->max_nsid;
+	sgroup->channels = calloc(sgroup->num_channels, sizeof(struct spdk_io_channel *));
+	if (!sgroup->channels) {
+		return -1;
+	}
+
+	for (i = 0; i < sgroup->num_channels; i++) {
+		ns = &subsystem->ns[i];
+		if (ns->allocated && sgroup->channels[i] == NULL) {
+			sgroup->channels[i] = spdk_bdev_get_io_channel(ns->desc);
+			if (sgroup->channels[i] == NULL) {
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+int
+spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
+				      struct spdk_nvmf_subsystem *subsystem)
+{
+	struct spdk_nvmf_subsystem_poll_group *sgroup;
+	uint32_t nsid;
+
+	sgroup = &group->sgroups[subsystem->id];
+
+	for (nsid = 0; nsid < sgroup->num_channels; nsid++) {
+		if (sgroup->channels[nsid]) {
+			spdk_put_io_channel(sgroup->channels[nsid]);
+			sgroup->channels[nsid] = NULL;
+		}
+	}
+
+	sgroup->num_channels = 0;
+	free(sgroup->channels);
+	sgroup->channels = NULL;
+
+	return 0;
+}
+
 SPDK_TRACE_REGISTER_FN(nvmf_trace)
 {
 	spdk_trace_register_object(OBJECT_NVMF_IO, 'r');

lib/nvmf/nvmf_internal.h

@@ -77,9 +77,21 @@ struct spdk_nvmf_transport_poll_group {
 	TAILQ_ENTRY(spdk_nvmf_transport_poll_group)	link;
 };
 
+struct spdk_nvmf_subsystem_poll_group {
+	/* Array of channels for each namespace indexed by nsid - 1 */
+	struct spdk_io_channel	**channels;
+	uint32_t		num_channels;
+};
+
 struct spdk_nvmf_poll_group {
 	struct spdk_poller				*poller;
+
 	TAILQ_HEAD(, spdk_nvmf_transport_poll_group)	tgroups;
+
+	/* Array of poll groups indexed by subsystem id (sid) */
+	struct spdk_nvmf_subsystem_poll_group		*sgroups;
+	uint32_t					num_sgroups;
+
 };
 
 typedef enum _spdk_nvmf_request_exec_status {
@@ -207,6 +219,10 @@ int spdk_nvmf_poll_group_remove(struct spdk_nvmf_poll_group *group,
 				struct spdk_nvmf_qpair *qpair);
 int spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group,
 				       struct spdk_nvmf_transport *transport);
+int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
+				       struct spdk_nvmf_subsystem *subsystem);
+int spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
+		struct spdk_nvmf_subsystem *subsystem);
 
 void spdk_nvmf_request_exec(struct spdk_nvmf_request *req);
 int spdk_nvmf_request_complete(struct spdk_nvmf_request *req);

lib/nvmf/subsystem.c

@@ -82,6 +82,24 @@ spdk_nvmf_valid_nqn(const char *nqn)
 	return true;
 }
 
+static void
+spdk_nvmf_subsystem_create_done(void *io_device, void *ctx, int status)
+{
+}
+
+static int
+spdk_nvmf_subsystem_add_to_poll_group(void *io_device,
+				      struct spdk_io_channel *ch,
+				      void *ctx)
+{
+	struct spdk_nvmf_poll_group *group;
+	struct spdk_nvmf_subsystem *subsystem = ctx;
+
+	group = spdk_io_channel_get_ctx(ch);
+
+	return spdk_nvmf_poll_group_add_subsystem(group, subsystem);
+}
+
 struct spdk_nvmf_subsystem *
 spdk_nvmf_create_subsystem(struct spdk_nvmf_tgt *tgt,
 			   const char *nqn,
@@ -146,9 +164,47 @@ spdk_nvmf_create_subsystem(struct spdk_nvmf_tgt *tgt,
 	tgt->subsystems[sid] = subsystem;
 	tgt->discovery_genctr++;
 
+	/* Send a message to each poll group to notify it that a new subsystem
+	 * is available.
+	 * TODO: This call does not currently allow the user to wait for these
+	 * messages to propagate. It also does not protect against two calls
+	 * to this function overlapping
+	 */
+	spdk_for_each_channel(tgt,
+			      spdk_nvmf_subsystem_add_to_poll_group,
+			      subsystem,
+			      spdk_nvmf_subsystem_create_done);
+
 	return subsystem;
 }
 
+static void
+spdk_nvmf_subsystem_delete_done(void *io_device, void *ctx, int status)
+{
+	struct spdk_nvmf_tgt *tgt = io_device;
+	struct spdk_nvmf_subsystem *subsystem = ctx;
+
+	free(subsystem->ns);
+
+	tgt->subsystems[subsystem->id] = NULL;
+	tgt->discovery_genctr++;
+
+	free(subsystem);
+}
+
+static int
+spdk_nvmf_subsystem_remove_from_poll_group(void *io_device,
+		struct spdk_io_channel *ch,
+		void *ctx)
+{
+	struct spdk_nvmf_poll_group *group;
+	struct spdk_nvmf_subsystem *subsystem = ctx;
+
+	group = spdk_io_channel_get_ctx(ch);
+
+	return spdk_nvmf_poll_group_remove_subsystem(group, subsystem);
+}
+
 void
 spdk_nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem)
 {
@@ -177,12 +233,16 @@ spdk_nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem)
 		spdk_nvmf_ctrlr_destruct(ctrlr);
 	}
 
-	free(subsystem->ns);
+	/* Send a message to each poll group to notify it that a subsystem
-
+	 * is no longer available.
-	subsystem->tgt->subsystems[subsystem->id] = NULL;
+	 * TODO: This call does not currently allow the user to wait for these
-	subsystem->tgt->discovery_genctr++;
+	 * messages to propagate. It also does not protect against two calls
-
+	 * to this function overlapping
-	free(subsystem);
+	 */
+	spdk_for_each_channel(subsystem->tgt,
+			      spdk_nvmf_subsystem_remove_from_poll_group,
+			      subsystem,
+			      spdk_nvmf_subsystem_delete_done);
 }
 
 

test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c

@@ -122,6 +122,20 @@ spdk_nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr)
 {
 }
 
+int
+spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
+				   struct spdk_nvmf_subsystem *subsystem)
+{
+	return 0;
+}
+
+int
+spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
+				      struct spdk_nvmf_subsystem *subsystem)
+{
+	return 0;
+}
+
 int
 spdk_nvmf_subsystem_bdev_attach(struct spdk_nvmf_subsystem *subsystem)
 {

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

@@ -95,6 +95,20 @@ spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, enum spdk_nvme_transport_
 	return NULL;
 }
 
+int
+spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
+				   struct spdk_nvmf_subsystem *subsystem)
+{
+	return 0;
+}
+
+int
+spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
+				      struct spdk_nvmf_subsystem *subsystem)
+{
+	return 0;
+}
+
 int
 spdk_nvme_transport_id_parse_trtype(enum spdk_nvme_transport_type *trtype, const char *str)
 {