bdev: make module init synchronous again

Module initializaiton was made asynchronous recently
to support bdev modules like gpt which need to do
asynchronous I/O.  But all modules now do any
asynchronous I/O in their examine() routines, and
init functions only do very basic operations to be
ready to handle examine() callbacks.

So simplify the bdev code and modules to go back to
a synchronous init procedure.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: Idb16156796ad7511d00f465d7a2db9acda6315b6

Reviewed-on: https://review.gerrithub.io/369485
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>

include/spdk_internal/bdev.h

@@ -83,11 +83,10 @@ struct spdk_bdev_module_if {
 	/**
 	 * Initialization function for the module.  Called by the spdk
 	 * application during startup.
-	 * User must call spdk_bdev_module_init_next() with return code inside this func.
 	 *
 	 * Modules are required to define this function.
 	 */
-	void (*module_init)(void);
+	int (*module_init)(void);
 
 	/**
 	 * Finish function for the module.  Called by the spdk application
@@ -430,8 +429,6 @@ void spdk_scsi_nvme_translate(const struct spdk_bdev_io *bdev_io,
 
 void spdk_bdev_module_list_add(struct spdk_bdev_module_if *bdev_module);
 void spdk_vbdev_module_list_add(struct spdk_bdev_module_if *vbdev_module);
-void spdk_bdev_module_init_next(int rc);
-void spdk_vbdev_module_init_next(int rc);
 
 static inline struct spdk_bdev_io *
 spdk_bdev_io_from_ctx(void *ctx)

lib/bdev/aio/bdev_aio.c

@@ -43,7 +43,7 @@
 
 #include "spdk_internal/log.h"
 
-static void bdev_aio_initialize(void);
+static int bdev_aio_initialize(void);
 static void aio_free_disk(struct file_disk *fdisk);
 
 static int
@@ -382,7 +382,7 @@ error_return:
 	return NULL;
 }
 
-static void
+static int
 bdev_aio_initialize(void)
 {
 	size_t i;
@@ -391,7 +391,7 @@ bdev_aio_initialize(void)
 
 	sp = spdk_conf_find_section(NULL, "AIO");
 	if (!sp) {
-		goto end;
+		return 0;
 	}
 
 	i = 0;
@@ -421,8 +421,7 @@ bdev_aio_initialize(void)
 		i++;
 	}
 
-end:
+	return 0;
-	spdk_bdev_module_init_next(0);
 }
 
 SPDK_LOG_REGISTER_TRACE_FLAG("aio", SPDK_TRACE_AIO)

lib/bdev/bdev.c

@@ -73,7 +73,6 @@ struct spdk_bdev_mgr {
 
 	bool init_complete;
 	bool module_init_complete;
-	int module_init_rc;
 
 #ifdef SPDK_CONFIG_VTUNE
 	__itt_domain	*domain;
@@ -88,11 +87,8 @@ static struct spdk_bdev_mgr g_bdev_mgr = {
 	.stop_poller_fn = NULL,
 	.init_complete = false,
 	.module_init_complete = false,
-	.module_init_rc = 0,
 };
 
-static struct spdk_bdev_module_if *g_next_bdev_module;
-static struct spdk_bdev_module_if *g_next_vbdev_module;
 static spdk_bdev_init_cb	g_cb_fn = NULL;
 static void			*g_cb_arg = NULL;
 
@@ -381,7 +377,10 @@ spdk_bdev_module_init_complete(int rc)
 	struct spdk_bdev_module_if *m;
 
 	g_bdev_mgr.module_init_complete = true;
-	g_bdev_mgr.module_init_rc = rc;
+
+	if (rc != 0) {
+		spdk_bdev_init_complete(rc);
+	}
 
 	/*
 	 * Check all vbdev modules for an examinations in progress.  If any
@@ -394,53 +393,39 @@ spdk_bdev_module_init_complete(int rc)
 		}
 	}
 
-	spdk_bdev_init_complete(rc);
+	spdk_bdev_init_complete(0);
 }
 
-void
+static int
-spdk_bdev_module_init_next(int rc)
+spdk_bdev_modules_init(void)
 {
-	if (rc) {
+	struct spdk_bdev_module_if *module;
-		assert(g_next_bdev_module != NULL);
+	int rc;
-		SPDK_ERRLOG("Failed to init bdev module: %s\n", g_next_bdev_module->name);
-		spdk_bdev_module_init_complete(rc);
-		return;
-	}
 
-	if (!g_next_bdev_module) {
+	TAILQ_FOREACH(module, &g_bdev_mgr.bdev_modules, tailq) {
-		g_next_bdev_module = TAILQ_FIRST(&g_bdev_mgr.bdev_modules);
+		rc = module->module_init();
-	} else {
+		if (rc != 0) {
-		g_next_bdev_module = TAILQ_NEXT(g_next_bdev_module, tailq);
+			return rc;
-	}
-
-	if (g_next_bdev_module) {
-		g_next_bdev_module->module_init();
-	} else {
-		spdk_bdev_module_init_complete(rc);
 		}
 	}
 
-void
+	return 0;
-spdk_vbdev_module_init_next(int rc)
+}
+
+static int
+spdk_vbdev_modules_init(void)
 {
-	if (rc) {
+	struct spdk_bdev_module_if *module;
-		assert(g_next_vbdev_module != NULL);
+	int rc;
-		SPDK_ERRLOG("Failed to init vbdev module: %s\n", g_next_vbdev_module->name);
+
-		spdk_bdev_module_init_complete(rc);
+	TAILQ_FOREACH(module, &g_bdev_mgr.vbdev_modules, tailq) {
-		return;
+		rc = module->module_init();
+		if (rc != 0) {
+			return rc;
+		}
 	}
 
-	if (!g_next_vbdev_module) {
+	return 0;
-		g_next_vbdev_module = TAILQ_FIRST(&g_bdev_mgr.vbdev_modules);
-	} else {
-		g_next_vbdev_module = TAILQ_NEXT(g_next_vbdev_module, tailq);
-	}
-
-	if (g_next_vbdev_module) {
-		g_next_vbdev_module->module_init();
-	} else {
-		spdk_bdev_module_init_next(0);
-	}
 }
 
 void
@@ -484,8 +469,8 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg,
 
 	if (g_bdev_mgr.bdev_io_pool == NULL) {
 		SPDK_ERRLOG("could not allocate spdk_bdev_io pool");
-		rc = -1;
+		spdk_bdev_module_init_complete(-1);
-		goto end;
+		return;
 	}
 
 	/**
@@ -501,8 +486,8 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg,
 				    SPDK_ENV_SOCKET_ID_ANY);
 	if (!g_bdev_mgr.buf_small_pool) {
 		SPDK_ERRLOG("create rbuf small pool failed\n");
-		rc = -1;
+		spdk_bdev_module_init_complete(-1);
-		goto end;
+		return;
 	}
 
 	cache_size = BUF_LARGE_POOL_SIZE / (2 * spdk_env_get_core_count());
@@ -513,8 +498,8 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg,
 				    SPDK_ENV_SOCKET_ID_ANY);
 	if (!g_bdev_mgr.buf_large_pool) {
 		SPDK_ERRLOG("create rbuf large pool failed\n");
-		rc = -1;
+		spdk_bdev_module_init_complete(-1);
-		goto end;
+		return;
 	}
 
 #ifdef SPDK_CONFIG_VTUNE
@@ -525,8 +510,14 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg,
 				spdk_bdev_mgmt_channel_destroy,
 				sizeof(struct spdk_bdev_mgmt_channel));
 
-end:
+	rc = spdk_vbdev_modules_init();
-	spdk_vbdev_module_init_next(rc);
+	if (rc != 0) {
+		spdk_bdev_module_init_complete(rc);
+		return;
+	}
+
+	rc = spdk_bdev_modules_init();
+	spdk_bdev_module_init_complete(rc);
 }
 
 int
@@ -1511,7 +1502,7 @@ spdk_vbdev_module_examine_done(struct spdk_bdev_module_if *module)
 		 * the vbdevs have finished their asynchronous I/O processing,
 		 * the entire bdev layer can be marked as complete.
 		 */
-		spdk_bdev_init_complete(g_bdev_mgr.module_init_rc);
+		spdk_bdev_init_complete(0);
 	}
 }
 

lib/bdev/error/vbdev_error.c

@@ -290,10 +290,10 @@ cleanup:
 	return rc;
 }
 
-static void
+static int
 vbdev_error_init(void)
 {
-	spdk_vbdev_module_init_next(0);
+	return 0;
 }
 
 static void

lib/bdev/gpt/vbdev_gpt.c

@@ -497,7 +497,7 @@ vbdev_gpt_read_gpt(struct spdk_bdev *bdev)
 	return 0;
 }
 
-static void
+static int
 vbdev_gpt_init(void)
 {
 	struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Gpt");
@@ -507,7 +507,7 @@ vbdev_gpt_init(void)
 		g_gpt_disabled = true;
 	}
 
-	spdk_vbdev_module_init_next(0);
+	return 0;
 }
 
 static void

lib/bdev/malloc/bdev_malloc.c

@@ -89,7 +89,7 @@ static struct malloc_disk *g_malloc_disk_head = NULL;
 
 int malloc_disk_count = 0;
 
-static void bdev_malloc_initialize(void);
+static int bdev_malloc_initialize(void);
 static void bdev_malloc_finish(void);
 static void bdev_malloc_get_spdk_running_config(FILE *fp);
 
@@ -439,7 +439,7 @@ static void free_malloc_disk(struct malloc_disk *mdisk)
 	spdk_dma_free(mdisk);
 }
 
-static void bdev_malloc_initialize(void)
+static int bdev_malloc_initialize(void)
 {
 	struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Malloc");
 	int NumberOfLuns, LunSizeInMB, BlockSize, i, rc = 0;
@@ -471,7 +471,7 @@ static void bdev_malloc_initialize(void)
 	}
 
 end:
-	spdk_bdev_module_init_next(rc);
+	return rc;
 }
 
 static void bdev_malloc_finish(void)

lib/bdev/null/bdev_null.c

@@ -178,7 +178,7 @@ null_bdev_destroy_cb(void *io_device, void *ctx_buf)
 {
 }
 
-static void
+static int
 bdev_null_initialize(void)
 {
 	struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Null");
@@ -257,7 +257,7 @@ bdev_null_initialize(void)
 	}
 
 end:
-	spdk_bdev_module_init_next(rc);
+	return rc;
 }
 
 static void

lib/bdev/nvme/bdev_nvme.c

@@ -131,7 +131,7 @@ static TAILQ_HEAD(, nvme_ctrlr)	g_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_ct
 static TAILQ_HEAD(, nvme_bdev) g_nvme_bdevs = TAILQ_HEAD_INITIALIZER(g_nvme_bdevs);
 
 static void nvme_ctrlr_create_bdevs(struct nvme_ctrlr *nvme_ctrlr);
-static void bdev_nvme_library_init(void);
+static int bdev_nvme_library_init(void);
 static void bdev_nvme_library_fini(void);
 static int bdev_nvme_queue_cmd(struct nvme_bdev *bdev, struct spdk_nvme_qpair *qpair,
 			       struct nvme_bdev_io *bio,
@@ -850,7 +850,7 @@ spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
 	return 0;
 }
 
-static void
+static int
 bdev_nvme_library_init(void)
 {
 	struct spdk_conf_section *sp;
@@ -991,7 +991,7 @@ bdev_nvme_library_init(void)
 
 end:
 	free(probe_ctx);
-	spdk_bdev_module_init_next(rc);
+	return rc;
 }
 
 static void

lib/bdev/rbd/bdev_rbd.c

@@ -201,7 +201,7 @@ bdev_rbd_start_aio(rbd_image_t image, struct bdev_rbd_io *cmd,
 	return 0;
 }
 
-static void bdev_rbd_library_init(void);
+static int bdev_rbd_library_init(void);
 static void bdev_rbd_library_fini(void);
 
 static int
@@ -549,7 +549,7 @@ spdk_bdev_rbd_create(const char *pool_name, const char *rbd_name, uint32_t block
 	return &rbd->disk;
 }
 
-static void
+static int
 bdev_rbd_library_init(void)
 {
 	int i, rc = 0;
@@ -609,5 +609,5 @@ bdev_rbd_library_init(void)
 	}
 
 end:
-	spdk_bdev_module_init_next(rc);
+	return rc;
 }

lib/bdev/split/vbdev_split.c

@@ -341,10 +341,10 @@ cleanup:
 	return rc;
 }
 
-static void
+static int
 vbdev_split_init(void)
 {
-	spdk_vbdev_module_init_next(0);
+	return 0;
 }
 
 static void