nvmf_tgt: Detect completion of poll group destroy at shutdown

Previously NVMe-oF target had requested each thread to destroy
its poll group, but had moved to the next state without waiting
for completion.

To create and destroy poll group threads dynamically, NVMe-oF
target have to know the completion of poll group destroy.

The reason is as follows.

spdk_thread_exit() requires that each poll group thread to unregister
its pollers, but poller unregistration is done asynchronously, and
releaes its I/O channels, but I/O channel release is done
asynchronously.

This patch does the following.

To avoid using mutex, add the init thread.

spdk_for_each_thread() requires completion callback but the
completion callback is usable only if each message handler is
synchronous. spdk_nvmf_poll_group_destroy() is asynchronous now.
So replace spdk_for_each_thread() by parsing g_poll_groups and
sending message to each corresonding thread.

Then add comletion callback nvmf_tgt_destroy_poll_group_done() to
spdk_nvmf_poll_group_destroy().

nvmf_tgt_destroy_poll_group_done() sends message to the init
thread.

The init thread executes _nvmf_tgt_destroy_poll_group_done().
_nvmf_tgt_destroy_poll_group_done() decrements g_num_poll_groups
and move to the next state if it becomes zero.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I97eb90b9dbff29d7702b20cf7ac233d39d7216b6
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/496
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>

module/event/subsystems/nvmf/nvmf_tgt.c

@@ -77,6 +77,8 @@ struct spdk_nvmf_tgt *g_spdk_nvmf_tgt = NULL;
 
 static enum nvmf_tgt_state g_tgt_state;
 
+static struct spdk_thread *g_tgt_fini_thread = NULL;
+
 /* Round-Robin/IP-based tracking of threads to poll group assignment */
 static struct nvmf_tgt_poll_group *g_next_poll_group = NULL;
 
@@ -306,30 +308,45 @@ acceptor_poll(void *arg)
 }
 
 static void
-nvmf_tgt_destroy_poll_group_done(void *ctx)
+_nvmf_tgt_destroy_poll_group_done(void *ctx)
 {
+	assert(g_num_poll_groups > 0);
+
+	if (--g_num_poll_groups == 0) {
 		g_tgt_state = NVMF_TGT_FINI_STOP_ACCEPTOR;
-	assert(g_num_poll_groups == 0);
 		nvmf_tgt_advance_state();
 	}
+}
+
+static void
+nvmf_tgt_destroy_poll_group_done(void *cb_arg, int status)
+{
+	struct nvmf_tgt_poll_group *pg = cb_arg;
+
+	free(pg);
+
+	spdk_thread_send_msg(g_tgt_fini_thread, _nvmf_tgt_destroy_poll_group_done, NULL);
+}
 
 static void
 nvmf_tgt_destroy_poll_group(void *ctx)
 {
-	struct nvmf_tgt_poll_group *pg, *tpg;
+	struct nvmf_tgt_poll_group *pg = ctx;
-	struct spdk_thread *thread;
 
-	thread = spdk_get_thread();
+	spdk_nvmf_poll_group_destroy(pg->group, nvmf_tgt_destroy_poll_group_done, pg);
+}
+
+static void
+nvmf_tgt_destroy_poll_groups(void)
+{
+	struct nvmf_tgt_poll_group *pg, *tpg;
+
+	g_tgt_fini_thread = spdk_get_thread();
+	assert(g_tgt_fini_thread != NULL);
 
 	TAILQ_FOREACH_SAFE(pg, &g_poll_groups, link, tpg) {
-		if (pg->thread == thread) {
 		TAILQ_REMOVE(&g_poll_groups, pg, link);
-			spdk_nvmf_poll_group_destroy(pg->group, NULL, NULL);
+		spdk_thread_send_msg(pg->thread, nvmf_tgt_destroy_poll_group, pg);
-			free(pg);
-			assert(g_num_poll_groups > 0);
-			g_num_poll_groups--;
-			return;
-		}
 	}
 }
 
@@ -567,9 +584,7 @@ nvmf_tgt_advance_state(void)
 		}
 		case NVMF_TGT_FINI_DESTROY_POLL_GROUPS:
 			/* Send a message to each thread and destroy the poll group */
-			spdk_for_each_thread(nvmf_tgt_destroy_poll_group,
+			nvmf_tgt_destroy_poll_groups();
-					     NULL,
-					     nvmf_tgt_destroy_poll_group_done);
 			break;
 		case NVMF_TGT_FINI_STOP_ACCEPTOR:
 			spdk_poller_unregister(&g_acceptor_poller);