event/subsystem/bdev: asynchronous SPDK finish

First this change moves spdk_subsystem_fini() to trigger on
spdk_app_stop(). This ensures that spdk_subsystem_fini() is called
before reactors are stopped in spdk_reactors_stop().

Finish paths for subsystems, bdevs and copy engine is now
asynchronous.
Each of those three mentioned have to make sure they are
asynchronous as well.

Only bdev that currently has requirement for asynchronous finish
are logical volume.
Thus the change in vbdev_lvol.c making it move to next bdev module
only after all lvol stores were unloaded.

Fio_plugin finish of bdev and copy_engine was removed for now.
Next patch in series adds it back with async support.

Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Change-Id: I80ee2d084f3d82c50bf1329e08996604ae61b1b3
Reviewed-on: https://review.gerrithub.io/381536
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>

examples/bdev/fio_plugin/fio_plugin.c

@@ -648,7 +648,5 @@ static void fio_init spdk_fio_register(void)
 
 static void fio_exit spdk_fio_unregister(void)
 {
-	spdk_bdev_finish();
-	spdk_copy_engine_finish();
 	unregister_ioengine(&ioengine);
 }

include/spdk/bdev.h

@@ -115,6 +115,7 @@ struct spdk_bdev_io_stat {
 struct spdk_bdev_poller;
 
 typedef void (*spdk_bdev_init_cb)(void *cb_arg, int rc);
+typedef void (*spdk_bdev_fini_cb)(void *cb_arg);
 
 typedef void (*spdk_bdev_poller_fn)(void *arg);
 typedef void (*spdk_bdev_poller_start_cb)(struct spdk_bdev_poller **ppoller,
@@ -126,7 +127,7 @@ typedef void (*spdk_bdev_poller_stop_cb)(struct spdk_bdev_poller **ppoller);
 void spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg,
 			  spdk_bdev_poller_start_cb start_poller_fn,
 			  spdk_bdev_poller_stop_cb stop_poller_fn);
-void spdk_bdev_finish(void);
+void spdk_bdev_finish(spdk_bdev_fini_cb cb_fn, void *cb_arg);
 void spdk_bdev_config_text(FILE *fp);
 
 struct spdk_bdev *spdk_bdev_get_by_name(const char *bdev_name);

include/spdk/copy_engine.h

@@ -41,13 +41,15 @@
 #include "spdk/stdinc.h"
 
 typedef void (*spdk_copy_completion_cb)(void *ref, int status);
+typedef void (*spdk_copy_fini_cb)(void *cb_arg);
 
 struct spdk_io_channel;
 
 struct spdk_copy_task;
 
 int spdk_copy_engine_initialize(void);
-void spdk_copy_engine_finish(void);
+void spdk_copy_engine_finish(spdk_copy_fini_cb cb_fn, void *cb_arg);
+void spdk_copy_engine_module_finish(void);
 
 struct spdk_io_channel *spdk_copy_engine_get_io_channel(void);
 int spdk_copy_submit(struct spdk_copy_task *copy_req, struct spdk_io_channel *ch, void *dst,

include/spdk_internal/bdev.h

@@ -121,6 +121,11 @@ struct spdk_bdev_module_if {
 	 */
 	uint32_t action_in_progress;
 
+	/**
+	 * Denotes if the module_fini function may complete asynchronously.
+	 */
+	bool async_fini;
+
 	TAILQ_ENTRY(spdk_bdev_module_if) tailq;
 };
 
@@ -380,6 +385,7 @@ void spdk_vbdev_unregister(struct spdk_bdev *vbdev, spdk_bdev_unregister_cb cb_f
 
 void spdk_bdev_module_examine_done(struct spdk_bdev_module_if *module);
 void spdk_bdev_module_init_done(struct spdk_bdev_module_if *module);
+void spdk_bdev_module_finish_done(void);
 int spdk_bdev_module_claim_bdev(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
 				struct spdk_bdev_module_if *module);
 void spdk_bdev_module_release_bdev(struct spdk_bdev *bdev);
@@ -536,7 +542,7 @@ void spdk_bdev_part_submit_request(struct spdk_bdev_part_channel *ch, struct spd
 
 /*
  * Set module initialization to be asynchronous. After using this macro, the module
- * initialization has to be explicitly finished by calling spdk_bdev_module_init_done().
+ * initialization has to be explicitly completed by calling spdk_bdev_module_init_done().
  */
 #define SPDK_BDEV_MODULE_ASYNC_INIT(name)							\
 	__attribute__((constructor)) static void name ## _async_init(void)			\
@@ -544,6 +550,16 @@ void spdk_bdev_part_submit_request(struct spdk_bdev_part_channel *ch, struct spd
 		SPDK_GET_BDEV_MODULE(name)->action_in_progress = 1;				\
 	}
 
+/*
+ * Set module finish to be asynchronous. After using this macro, the module
+ * finishing has to be explicitly completed by calling spdk_bdev_module_fini_done().
+ */
+#define SPDK_BDEV_MODULE_ASYNC_FINI(name)							\
+	__attribute__((constructor)) static void name ## _async_fini(void)			\
+	{											\
+		SPDK_GET_BDEV_MODULE(name)->async_fini = true;					\
+	}
+
 /*
  * Modules are not required to use this macro.  It allows modules to reference the module with
  * SPDK_GET_BDEV_MODULE() before it is defined by SPDK_BDEV_MODULE_REGISTER.

include/spdk_internal/copy_engine.h

@@ -65,7 +65,7 @@ struct spdk_copy_module_if {
 	 *
 	 *  Modules are not required to define this function.
 	 */
-	void	(*module_fini)(void);
+	void	(*module_fini)(void *ctx);
 
 	/** Function called to return a text string representing the
 	 *   module's configuration options for inclusion in an

include/spdk_internal/event.h

@@ -49,13 +49,13 @@ int spdk_reactors_init(unsigned int max_delay_us);
 void spdk_reactors_fini(void);
 
 void spdk_reactors_start(void);
-void spdk_reactors_stop(void);
+void spdk_reactors_stop(void *arg1, void *arg2);
 
 struct spdk_subsystem {
 	const char *name;
 	/* User must call spdk_subsystem_init_next() when they are done with their initialization. */
 	void (*init)(void);
-	void (*fini)(void);
+	void (*fini)(void *arg1, void *arg2);
 	void (*config)(FILE *fp);
 	TAILQ_ENTRY(spdk_subsystem) tailq;
 };
@@ -71,8 +71,9 @@ void spdk_add_subsystem(struct spdk_subsystem *subsystem);
 void spdk_add_subsystem_depend(struct spdk_subsystem_depend *depend);
 
 void spdk_subsystem_init(struct spdk_event *app_start_event);
-void spdk_subsystem_fini(void);
+void spdk_subsystem_fini(struct spdk_event *app_finish_event);
 void spdk_subsystem_init_next(int rc);
+void spdk_subsystem_fini_next(void);
 void spdk_subsystem_config(FILE *fp);
 
 /**

lib/bdev/bdev.c

@@ -37,6 +37,7 @@
 #include "spdk/bdev.h"
 
 #include "spdk/env.h"
+#include "spdk/event.h"
 #include "spdk/io_channel.h"
 #include "spdk/likely.h"
 #include "spdk/queue.h"
@@ -94,8 +95,13 @@ static struct spdk_bdev_mgr g_bdev_mgr = {
 	.module_init_complete = false,
 };
 
-static spdk_bdev_init_cb	g_cb_fn = NULL;
-static void			*g_cb_arg = NULL;
+static spdk_bdev_init_cb	g_init_cb_fn = NULL;
+static void			*g_init_cb_arg = NULL;
+
+static spdk_bdev_fini_cb	g_fini_cb_fn = NULL;
+static void			*g_fini_cb_arg = NULL;
+struct spdk_bdev_module_if	*g_bdev_module = NULL;
+struct spdk_thread		*g_fini_thread = NULL;
 
 
 struct spdk_bdev_mgmt_channel {
@@ -372,12 +378,12 @@ spdk_bdev_mgmt_channel_destroy(void *io_device, void *ctx_buf)
 static void
 spdk_bdev_init_complete(int rc)
 {
-	spdk_bdev_init_cb cb_fn = g_cb_fn;
-	void *cb_arg = g_cb_arg;
+	spdk_bdev_init_cb cb_fn = g_init_cb_fn;
+	void *cb_arg = g_init_cb_arg;
 
 	g_bdev_mgr.init_complete = true;
-	g_cb_fn = NULL;
-	g_cb_arg = NULL;
+	g_init_cb_fn = NULL;
+	g_init_cb_arg = NULL;
 
 	cb_fn(cb_arg, rc);
 }
@@ -478,8 +484,8 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg,
 
 	assert(cb_fn != NULL);
 
-	g_cb_fn = cb_fn;
-	g_cb_arg = cb_arg;
+	g_init_cb_fn = cb_fn;
+	g_init_cb_arg = cb_arg;
 
 	g_bdev_mgr.start_poller_fn = start_poller_fn;
 	g_bdev_mgr.stop_poller_fn = stop_poller_fn;
@@ -558,17 +564,19 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg,
 	spdk_bdev_module_action_complete();
 }
 
-void
-spdk_bdev_finish(void)
+static void
+spdk_bdev_module_finish_cb(void *io_device)
 {
-	struct spdk_bdev_module_if *bdev_module;
+	spdk_bdev_fini_cb cb_fn = g_fini_cb_fn;
 
-	TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, tailq) {
-		if (bdev_module->module_fini) {
-			bdev_module->module_fini();
-		}
-	}
+	cb_fn(g_fini_cb_arg);
+	g_fini_cb_fn = NULL;
+	g_fini_cb_arg = NULL;
+}
 
+static void
+spdk_bdev_module_finish_complete(void)
+{
 	if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != SPDK_BDEV_IO_POOL_SIZE) {
 		SPDK_ERRLOG("bdev IO pool count is %zu but should be %u\n",
 			    spdk_mempool_count(g_bdev_mgr.bdev_io_pool),
@@ -594,7 +602,55 @@ spdk_bdev_finish(void)
 	spdk_mempool_free(g_bdev_mgr.buf_large_pool);
 	spdk_dma_free(g_bdev_mgr.zero_buffer);
 
-	spdk_io_device_unregister(&g_bdev_mgr, NULL);
+	spdk_io_device_unregister(&g_bdev_mgr, spdk_bdev_module_finish_cb);
+}
+
+static void
+_call_next_module_fini(void *arg)
+{
+	struct spdk_bdev_module_if *module = arg;
+
+	module->module_fini();
+}
+
+void
+spdk_bdev_module_finish_done(void)
+{
+	if (spdk_get_thread() != g_fini_thread) {
+		SPDK_ERRLOG("%s changed threads\n", g_bdev_module->name);
+	}
+
+	if (!g_bdev_module) {
+		g_bdev_module = TAILQ_FIRST(&g_bdev_mgr.bdev_modules);
+	} else {
+		g_bdev_module = TAILQ_NEXT(g_bdev_module, tailq);
+	}
+
+	if (!g_bdev_module) {
+		spdk_bdev_module_finish_complete();
+		return;
+	}
+
+	if (g_bdev_module->module_fini) {
+		spdk_thread_send_msg(g_fini_thread, _call_next_module_fini, g_bdev_module);
+	}
+
+	if (!g_bdev_module->async_fini) {
+		spdk_bdev_module_finish_done();
+	}
+}
+
+void
+spdk_bdev_finish(spdk_bdev_fini_cb cb_fn, void *cb_arg)
+{
+	assert(cb_fn != NULL);
+
+	g_fini_thread = spdk_get_thread();
+
+	g_fini_cb_fn = cb_fn;
+	g_fini_cb_arg = cb_arg;
+
+	spdk_bdev_module_finish_done();
 }
 
 struct spdk_bdev_io *

lib/bdev/lvol/vbdev_lvol.c

@@ -715,13 +715,26 @@ vbdev_lvs_init(void)
 	return 0;
 }
 
+static void
+vbdev_lvs_finished(void *cb_arg, int lvserrno)
+{
+	if (TAILQ_EMPTY(&g_spdk_lvol_pairs)) {
+		spdk_bdev_module_finish_done();
+	}
+}
+
 static void
 vbdev_lvs_fini(void)
 {
 	struct lvol_store_bdev *lvs_bdev, *tmp;
 
+	if (TAILQ_EMPTY(&g_spdk_lvol_pairs)) {
+		spdk_bdev_module_finish_done();
+		return;
+	}
+
 	TAILQ_FOREACH_SAFE(lvs_bdev, &g_spdk_lvol_pairs, lvol_stores, tmp) {
-		vbdev_lvs_unload(lvs_bdev->lvs, NULL, NULL);
+		vbdev_lvs_unload(lvs_bdev->lvs, vbdev_lvs_finished, NULL);
 	}
 }
 
@@ -844,4 +857,5 @@ vbdev_lvs_examine(struct spdk_bdev *bdev)
 }
 SPDK_BDEV_MODULE_REGISTER(lvol, vbdev_lvs_init, vbdev_lvs_fini, NULL, vbdev_lvs_get_ctx_size,
 			  vbdev_lvs_examine)
+SPDK_BDEV_MODULE_ASYNC_FINI(lvol);
 SPDK_LOG_REGISTER_TRACE_FLAG("vbdev_lvol", SPDK_TRACE_VBDEV_LVOL);

lib/bdev/pmem/bdev_pmem.c

@@ -64,6 +64,7 @@ bdev_pmem_get_ctx_size(void)
 
 SPDK_BDEV_MODULE_REGISTER(pmem, bdev_pmem_initialize, bdev_pmem_finish,
 			  NULL, bdev_pmem_get_ctx_size, NULL)
+SPDK_BDEV_MODULE_ASYNC_FINI(pmem);
 
 
 typedef int(*spdk_bdev_pmem_io_request)(PMEMblkpool *pbp, void *buf, long long blockno);
@@ -376,6 +377,12 @@ bdev_pmem_initialize(void)
 
 }
 
+static void
+bdev_pmem_finish_done(void *io_device)
+{
+	spdk_bdev_module_finish_done();
+}
+
 static void
 bdev_pmem_finish(void)
 {
@@ -385,7 +392,7 @@ bdev_pmem_finish(void)
 		bdev_pmem_destruct(pdisk);
 	}
 
-	spdk_io_device_unregister(&g_pmem_disks, NULL);
+	spdk_io_device_unregister(&g_pmem_disks, bdev_pmem_finish_done);
 }
 
 SPDK_LOG_REGISTER_TRACE_FLAG("bdev_pmem", SPDK_TRACE_BDEV_PMEM)

lib/copy/copy_engine.c

@@ -36,6 +36,7 @@
 #include "spdk_internal/copy_engine.h"
 
 #include "spdk/env.h"
+#include "spdk/event.h"
 #include "spdk/log.h"
 #include "spdk/io_channel.h"
 
@@ -53,6 +54,10 @@ struct copy_io_channel {
 	struct spdk_io_channel	*ch;
 };
 
+struct spdk_copy_module_if *g_copy_engine_module = NULL;
+spdk_copy_fini_cb g_fini_cb_fn = NULL;
+void *g_fini_cb_arg = NULL;
+
 void
 spdk_copy_engine_register(struct spdk_copy_engine *copy_engine)
 {
@@ -224,17 +229,6 @@ spdk_copy_engine_module_initialize(void)
 	}
 }
 
-static void
-spdk_copy_engine_module_finish(void)
-{
-	struct spdk_copy_module_if *copy_engine_module;
-
-	TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) {
-		if (copy_engine_module->module_fini)
-			copy_engine_module->module_fini();
-	}
-}
-
 int
 spdk_copy_engine_initialize(void)
 {
@@ -249,9 +243,45 @@ spdk_copy_engine_initialize(void)
 	return 0;
 }
 
-void
-spdk_copy_engine_finish(void)
+static void
+spdk_copy_engine_module_finish_cb(void)
 {
+	spdk_copy_fini_cb cb_fn = g_fini_cb_fn;
+
+	cb_fn(g_fini_cb_arg);
+	g_fini_cb_fn = NULL;
+	g_fini_cb_arg = NULL;
+}
+
+void
+spdk_copy_engine_module_finish(void)
+{
+	if (!g_copy_engine_module) {
+		g_copy_engine_module = TAILQ_FIRST(&spdk_copy_module_list);
+	} else {
+		g_copy_engine_module = TAILQ_NEXT(g_copy_engine_module, tailq);
+	}
+
+	if (!g_copy_engine_module) {
+		spdk_copy_engine_module_finish_cb();
+		return;
+	}
+
+	if (g_copy_engine_module->module_fini) {
+		spdk_thread_send_msg(spdk_get_thread(), g_copy_engine_module->module_fini, NULL);
+	} else {
+		spdk_copy_engine_module_finish();
+	}
+}
+
+void
+spdk_copy_engine_finish(spdk_copy_fini_cb cb_fn, void *cb_arg)
+{
+	assert(cb_fn != NULL);
+
+	g_fini_cb_fn = cb_fn;
+	g_fini_cb_arg = cb_arg;
+
 	spdk_copy_engine_module_finish();
 }
 

lib/copy/ioat/copy_engine_ioat.c

@@ -106,7 +106,7 @@ struct ioat_task {
 };
 
 static int copy_engine_ioat_init(void);
-static void copy_engine_ioat_exit(void);
+static void copy_engine_ioat_exit(void *ctx);
 
 static size_t
 copy_engine_ioat_get_ctx_size(void)
@@ -118,7 +118,7 @@ SPDK_COPY_MODULE_REGISTER(copy_engine_ioat_init, copy_engine_ioat_exit, NULL,
 			  copy_engine_ioat_get_ctx_size)
 
 static void
-copy_engine_ioat_exit(void)
+copy_engine_ioat_exit(void *ctx)
 {
 	struct ioat_device *dev;
 
@@ -129,7 +129,7 @@ copy_engine_ioat_exit(void)
 		ioat_free_device(dev);
 		spdk_dma_free(dev);
 	}
-	return;
+	spdk_copy_engine_module_finish();
 }
 
 static void

lib/event/app.c

@@ -56,6 +56,7 @@ struct spdk_app {
 
 static struct spdk_app g_spdk_app;
 static struct spdk_event *g_shutdown_event = NULL;
+static int g_init_lcore;
 
 int
 spdk_app_get_shm_id(void)
@@ -376,8 +377,8 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_event_fn start_fn,
 	}
 
 	g_spdk_app.rc = 0;
-	app_start_event = spdk_event_allocate(spdk_env_get_current_core(), start_fn,
-					      arg1, arg2);
+	g_init_lcore = spdk_env_get_current_core();
+	app_start_event = spdk_event_allocate(g_init_lcore, start_fn, arg1, arg2);
 
 	spdk_subsystem_init(app_start_event);
 
@@ -390,16 +391,28 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_event_fn start_fn,
 void
 spdk_app_fini(void)
 {
-	spdk_subsystem_fini();
 	spdk_trace_cleanup();
 	spdk_reactors_fini();
 	spdk_conf_free(g_spdk_app.config);
 	spdk_log_close();
 }
 
+static void
+_spdk_app_stop(void *arg1, void *arg2)
+{
+	struct spdk_event *app_stop_event;
+
+	app_stop_event = spdk_event_allocate(spdk_env_get_current_core(), spdk_reactors_stop, NULL, NULL);
+	spdk_subsystem_fini(app_stop_event);
+}
+
 void
 spdk_app_stop(int rc)
 {
-	spdk_reactors_stop();
 	g_spdk_app.rc = rc;
+	/*
+	 * We want to run spdk_subsystem_fini() from the same lcore where spdk_subsystem_init()
+	 * was called.
+	 */
+	spdk_event_call(spdk_event_allocate(g_init_lcore, _spdk_app_stop, NULL, NULL));
 }

lib/event/reactor.c

@@ -580,7 +580,8 @@ spdk_reactors_start(void)
 	g_reactor_state = SPDK_REACTOR_STATE_SHUTDOWN;
 }
 
-void spdk_reactors_stop(void)
+void
+spdk_reactors_stop(void *arg1, void *arg2)
 {
 	g_reactor_state = SPDK_REACTOR_STATE_EXITING;
 }

lib/event/subsystem.c

@@ -44,6 +44,8 @@ static TAILQ_HEAD(subsystem_depend, spdk_subsystem_depend) g_depends =
 	TAILQ_HEAD_INITIALIZER(g_depends);
 static struct spdk_subsystem *g_next_subsystem;
 static struct spdk_event *g_app_start_event;
+static struct spdk_event *g_app_stop_event;
+static int g_fini_core;
 
 void
 spdk_add_subsystem(struct spdk_subsystem *subsystem)
@@ -176,18 +178,53 @@ spdk_subsystem_init(struct spdk_event *app_start_event)
 }
 
 void
-spdk_subsystem_fini(void)
+spdk_subsystem_fini_next(void)
 {
-	struct spdk_subsystem *cur;
+	struct spdk_event *next_fini_event;
 
-	cur = TAILQ_LAST(&g_subsystems, spdk_subsystem_list);
+	assert(g_app_stop_event->lcore == spdk_env_get_current_core());
 
-	while (cur) {
-		if (cur->fini) {
-			cur->fini();
-		}
-		cur = TAILQ_PREV(cur, spdk_subsystem_list, tailq);
+	if (!g_next_subsystem) {
+		g_next_subsystem = TAILQ_LAST(&g_subsystems, spdk_subsystem_list);
+	} else {
+		g_next_subsystem = TAILQ_PREV(g_next_subsystem, spdk_subsystem_list, tailq);
 	}
+
+	if (!g_next_subsystem) {
+		spdk_event_call(g_app_stop_event);
+		return;
+	}
+
+	if (g_next_subsystem->fini) {
+		next_fini_event = spdk_event_allocate(g_fini_core, g_next_subsystem->fini, NULL,
+						      NULL);
+		spdk_event_call(next_fini_event);
+	} else {
+		spdk_subsystem_fini_next();
+	}
+}
+
+static void
+spdk_subsystem_fini_schedule(void *arg1, void *arg2)
+{
+	spdk_subsystem_fini_next();
+}
+
+void
+spdk_subsystem_fini(struct spdk_event *app_stop_event)
+{
+	struct spdk_event *fini_event;
+
+	assert(g_next_subsystem == NULL);
+
+	g_app_stop_event = app_stop_event;
+
+	/* There is assumption that whole fini path is done on one core. */
+	assert(g_app_stop_event->lcore == spdk_env_get_current_core());
+
+	g_fini_core = spdk_env_get_current_core();
+	fini_event = spdk_event_allocate(g_fini_core, spdk_subsystem_fini_schedule, NULL, NULL);
+	spdk_event_call(fini_event);
 }
 
 void

lib/event/subsystems/bdev/bdev.c

@@ -37,6 +37,7 @@
 #include "spdk/env.h"
 
 #include "spdk_internal/event.h"
+#include "spdk/env.h"
 
 static void
 spdk_bdev_initialize_complete(void *cb_arg, int rc)
@@ -72,9 +73,15 @@ spdk_bdev_subsystem_initialize(void)
 }
 
 static void
-spdk_bdev_subsystem_finish(void)
+spdk_bdev_subsystem_finish_done(void *cb_arg)
 {
-	spdk_bdev_finish();
+	spdk_subsystem_fini_next();
+}
+
+static void
+spdk_bdev_subsystem_finish(void *arg1, void *arg2)
+{
+	spdk_bdev_finish(spdk_bdev_subsystem_finish_done, NULL);
 }
 
 SPDK_SUBSYSTEM_REGISTER(bdev, spdk_bdev_subsystem_initialize,

lib/event/subsystems/copy/copy.c

@@ -36,6 +36,7 @@
 #include "spdk/copy_engine.h"
 
 #include "spdk_internal/event.h"
+#include "spdk/env.h"
 
 static void
 spdk_copy_engine_subsystem_initialize(void)
@@ -48,9 +49,15 @@ spdk_copy_engine_subsystem_initialize(void)
 }
 
 static void
-spdk_copy_engine_subsystem_finish(void)
+spdk_copy_engine_subsystem_finish_done(void *cb_arg)
 {
-	spdk_copy_engine_finish();
+	spdk_subsystem_fini_next();
+}
+
+static void
+spdk_copy_engine_subsystem_finish(void *arg1, void *arg2)
+{
+	spdk_copy_engine_finish(spdk_copy_engine_subsystem_finish_done, NULL);
 }
 
 SPDK_SUBSYSTEM_REGISTER(copy, spdk_copy_engine_subsystem_initialize,

lib/event/subsystems/iscsi/iscsi.c

@@ -48,9 +48,10 @@ spdk_iscsi_subsystem_init(void)
 }
 
 static void
-spdk_iscsi_subsystem_fini(void)
+spdk_iscsi_subsystem_fini(void *arg1, void *arg2)
 {
 	spdk_iscsi_fini();
+	spdk_subsystem_fini_next();
 }
 
 SPDK_SUBSYSTEM_REGISTER(iscsi, spdk_iscsi_subsystem_init, spdk_iscsi_subsystem_fini,

lib/event/subsystems/net/net.c

@@ -48,9 +48,10 @@ spdk_interface_subsystem_init(void)
 }
 
 static void
-spdk_interface_subsystem_destroy(void)
+spdk_interface_subsystem_destroy(void *arg1, void *arg2)
 {
 	spdk_interface_destroy();
+	spdk_subsystem_fini_next();
 }
 
 SPDK_SUBSYSTEM_REGISTER(interface, spdk_interface_subsystem_init,
@@ -67,9 +68,10 @@ spdk_net_subsystem_start(void)
 }
 
 static void
-spdk_net_subsystem_fini(void)
+spdk_net_subsystem_fini(void *arg1, void *arg2)
 {
 	spdk_net_framework_fini();
+	spdk_subsystem_fini_next();
 }
 
 SPDK_SUBSYSTEM_REGISTER(net_framework, spdk_net_subsystem_start,

lib/event/subsystems/rpc/rpc.c

@@ -128,10 +128,11 @@ spdk_rpc_subsystem_initialize(void)
 }
 
 static void
-spdk_rpc_subsystem_finish(void)
+spdk_rpc_subsystem_finish(void *arg1, void *arg2)
 {
 	spdk_rpc_close();
 	spdk_poller_unregister(&g_rpc_poller, NULL);
+	spdk_subsystem_fini_next();
 }
 
 static void

lib/event/subsystems/scsi/scsi.c

@@ -48,9 +48,10 @@ spdk_scsi_subsystem_init(void)
 }
 
 static void
-spdk_scsi_subsystem_fini(void)
+spdk_scsi_subsystem_fini(void *arg1, void *arg2)
 {
 	spdk_scsi_fini();
+	spdk_subsystem_fini_next();
 }
 
 SPDK_SUBSYSTEM_REGISTER(scsi, spdk_scsi_subsystem_init, spdk_scsi_subsystem_fini, NULL)

lib/event/subsystems/vhost/vhost.c

@@ -48,9 +48,10 @@ spdk_vhost_subsystem_init(void)
 }
 
 static void
-spdk_vhost_subsystem_fini(void)
+spdk_vhost_subsystem_fini(void *arg1, void *arg2)
 {
 	spdk_vhost_fini();
+	spdk_subsystem_fini_next();
 }
 
 SPDK_SUBSYSTEM_REGISTER(vhost, spdk_vhost_subsystem_init, spdk_vhost_subsystem_fini, NULL)

test/lib/event/event_perf/event_perf.c

@@ -56,7 +56,9 @@ submit_new_event(void *arg1, void *arg2)
 	static __thread uint32_t next_lcore = RTE_MAX_LCORE;
 
 	if (spdk_get_ticks() > g_tsc_end) {
-		spdk_app_stop(0);
+		if (rte_lcore_id() == rte_get_master_lcore()) {
+			spdk_app_stop(0);
+		}
 		return;
 	}
 

test/unit/lib/bdev/bdev.c/bdev_ut.c

@@ -42,51 +42,6 @@
 
 SPDK_DECLARE_BDEV_MODULE(vbdev_ut);
 
-void *
-spdk_io_channel_get_ctx(struct spdk_io_channel *ch)
-{
-	return NULL;
-}
-
-void
-spdk_io_device_register(void *io_device, spdk_io_channel_create_cb create_cb,
-			spdk_io_channel_destroy_cb destroy_cb, uint32_t ctx_size)
-{
-}
-
-void
-spdk_io_device_unregister(void *io_device, spdk_io_device_unregister_cb unregister_cb)
-{
-}
-
-void
-spdk_thread_send_msg(const struct spdk_thread *thread, spdk_thread_fn fn, void *ctx)
-{
-}
-
-struct spdk_io_channel *
-spdk_get_io_channel(void *io_device)
-{
-	return NULL;
-}
-
-void
-spdk_put_io_channel(struct spdk_io_channel *ch)
-{
-}
-
-void
-spdk_for_each_channel(void *io_device, spdk_channel_msg fn, void *ctx,
-		      spdk_channel_for_each_cpl cpl)
-{
-}
-
-struct spdk_thread *
-spdk_io_channel_get_thread(struct spdk_io_channel *ch)
-{
-	return NULL;
-}
-
 void
 spdk_scsi_nvme_translate(const struct spdk_bdev_io *bdev_io,
 			 int *sc, int *sk, int *asc, int *ascq)

test/unit/lib/bdev/mt/bdev.c/bdev_ut.c

@@ -59,6 +59,7 @@ struct ut_bdev_channel {
 
 struct ut_bdev g_bdev;
 struct spdk_bdev_desc *g_desc;
+bool g_teardown_done = false;
 
 static int
 stub_create_ch(void *io_device, void *ctx_buf)
@@ -206,13 +207,23 @@ setup_test(void)
 	spdk_bdev_open(&g_bdev.bdev, true, NULL, NULL, &g_desc);
 }
 
+static void
+finish_cb(void *cb_arg)
+{
+	g_teardown_done = true;
+}
+
 static void
 teardown_test(void)
 {
+	g_teardown_done = false;
 	spdk_bdev_close(g_desc);
 	g_desc = NULL;
 	unregister_bdev();
-	spdk_bdev_finish();
+	spdk_bdev_finish(finish_cb, NULL);
+	poll_threads();
+	CU_ASSERT(g_teardown_done == true);
+	g_teardown_done = false;
 	free_threads();
 }
 

test/unit/lib/bdev/pmem/bdev_pmem_ut.c

@@ -266,6 +266,11 @@ spdk_bdev_register(struct spdk_bdev *bdev)
 	g_bdev = bdev;
 }
 
+void
+spdk_bdev_module_finish_done(void)
+{
+}
+
 static void
 ut_bdev_pmem_destruct(struct spdk_bdev *bdev)
 {

test/unit/lib/bdev/vbdev_lvol.c/vbdev_lvol_ut.c

@@ -115,6 +115,12 @@ spdk_vbdev_unregister(struct spdk_bdev *vbdev, spdk_bdev_unregister_cb cb_fn, vo
 	vbdev->fn_table->destruct(vbdev->ctxt);
 }
 
+void
+spdk_bdev_module_finish_done(void)
+{
+	return;
+}
+
 uint64_t
 spdk_bs_get_page_size(struct spdk_blob_store *bs)
 {