lib/event: enable repeated spdk_app_start/stop

With this patch, spdk_app_start/stop can be repeatedly
called by users based on their upper level application's
requirement.
Changes are:
* Add reinit ability inside spdk_env_init and related functions
* Clear g_shutdown_sig_received in spdk_app_setup_signal_handlers
* Clear malloc_disk_count in bdev_malloc_initialize

Change-Id: I2d7be52b0e4aac2cb6734cc1237ce72d33b6de0c
Signed-off-by: Xiaodong Liu <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2260
Community-CI: Mellanox Build Bot
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

include/spdk/env.h

@@ -159,8 +159,11 @@ void spdk_free(void *buf);
 void spdk_env_opts_init(struct spdk_env_opts *opts);
 
 /**
- * Initialize the environment library. This must be called prior to using
+ * Initialize or reinitialize the environment library.
- * any other functions in this library.
+ * For initialization, this must be called prior to using any other functions
+ * in this library. For reinitialization, the parameter `opts` must be set to
+ * NULL and this must be called after the environment library was finished by
+ * spdk_env_fini() within the same process.
  *
  * \param opts Environment initialization options.
  * \return 0 on success, or negative errno on failure.
@@ -168,10 +171,11 @@ void spdk_env_opts_init(struct spdk_env_opts *opts);
 int spdk_env_init(const struct spdk_env_opts *opts);
 
 /**
- * Release any resources of the environment library that were alllocated with
+ * Release any resources of the environment library that were allocated with
  * spdk_env_init(). After this call, no SPDK env function calls may be made.
  * It is expected that common usage of this function is to call it just before
- * terminating the process.
+ * terminating the process or before reinitializing the environment library
+ * within the same process.
  */
 void spdk_env_fini(void);
 

lib/env_dpdk/env_internal.h

@@ -77,6 +77,7 @@ int pci_device_init(struct rte_pci_driver *driver, struct rte_pci_device *device
 int pci_device_fini(struct rte_pci_device *device);
 
 void pci_env_init(void);
+void pci_env_reinit(void);
 void pci_env_fini(void);
 int mem_map_init(bool legacy_mem);
 int vtophys_init(void);

lib/env_dpdk/init.c

@@ -128,6 +128,10 @@ free_args(char **args, int argcount)
 {
 	int i;
 
+	if (args == NULL) {
+		return;
+	}
+
 	for (i = 0; i < argcount; i++) {
 		free(args[i]);
 	}
@@ -501,6 +505,8 @@ spdk_env_dpdk_post_fini(void)
 	pci_env_fini();
 
 	free_args(g_eal_cmdline, g_eal_cmdline_argcount);
+	g_eal_cmdline = NULL;
+	g_eal_cmdline_argcount = 0;
 }
 
 int
@@ -511,6 +517,26 @@ spdk_env_init(const struct spdk_env_opts *opts)
 	int orig_optind;
 	bool legacy_mem;
 
+	/* If SPDK env has been initialized before, then only pci env requires
+	 * reinitialization.
+	 */
+	if (g_external_init == false) {
+		if (opts != NULL) {
+			fprintf(stderr, "Invalid arguments to reinitialize SPDK env\n");
+			return -EINVAL;
+		}
+
+		printf("Starting %s / %s reinitialization...\n", SPDK_VERSION_STRING, rte_version());
+		pci_env_reinit();
+
+		return 0;
+	}
+
+	if (opts == NULL) {
+		fprintf(stderr, "NULL arguments to initialize DPDK\n");
+		return -EINVAL;
+	}
+
 	rc = build_eal_cmdline(opts);
 	if (rc < 0) {
 		fprintf(stderr, "Invalid arguments to initialize DPDK\n");

lib/env_dpdk/pci.c

@@ -239,15 +239,9 @@ cleanup_pci_devices(void)
 
 static int scan_pci_bus(bool delay_init);
 
-void
+static inline void
-pci_env_init(void)
+_pci_env_init(void)
 {
-	struct spdk_pci_driver *driver;
-
-	TAILQ_FOREACH(driver, &g_pci_drivers, tailq) {
-		rte_pci_register(&driver->driver);
-	}
-
 	/* We assume devices were present on the bus for more than 2 seconds
 	 * before initializing SPDK and there's no need to wait more. We scan
 	 * the bus, but we don't blacklist any devices.
@@ -260,6 +254,28 @@ pci_env_init(void)
 	}
 }
 
+void
+pci_env_init(void)
+{
+	struct spdk_pci_driver *driver;
+
+	TAILQ_FOREACH(driver, &g_pci_drivers, tailq) {
+		rte_pci_register(&driver->driver);
+	}
+
+	_pci_env_init();
+}
+
+void
+pci_env_reinit(void)
+{
+	/* There is no need to register pci drivers again, since they were
+	 * already pre-registered in pci_env_init.
+	 */
+
+	_pci_env_init();
+}
+
 void
 pci_env_fini(void)
 {

lib/event/app.c

@@ -315,8 +315,8 @@ app_setup_signal_handlers(struct spdk_app_opts *opts)
 	}
 
 	/* Install the same handler for SIGINT and SIGTERM */
+	g_shutdown_sig_received = false;
 	sigact.sa_handler = __shutdown_signal;
-
 	rc = sigaction(SIGINT, &sigact, NULL);
 	if (rc < 0) {
 		SPDK_ERRLOG("sigaction(SIGINT) failed\n");
@@ -497,6 +497,16 @@ app_setup_env(struct spdk_app_opts *opts)
 	struct spdk_env_opts env_opts = {};
 	int rc;
 
+	if (opts == NULL) {
+		rc = spdk_env_init(NULL);
+		if (rc != 0) {
+			SPDK_ERRLOG("Unable to reinitialize SPDK env\n");
+		}
+
+		return rc;
+	}
+
+
 	spdk_env_opts_init(&env_opts);
 
 	env_opts.name = opts->name;
@@ -512,7 +522,6 @@ app_setup_env(struct spdk_app_opts *opts)
 	env_opts.num_pci_addr = opts->num_pci_addr;
 	env_opts.pci_blacklist = opts->pci_blacklist;
 	env_opts.pci_whitelist = opts->pci_whitelist;
-	env_opts.base_virtaddr = opts->base_virtaddr;
 	env_opts.env_context = opts->env_context;
 	env_opts.iova_mode = opts->iova_mode;
 
@@ -520,6 +529,7 @@ app_setup_env(struct spdk_app_opts *opts)
 	free(env_opts.pci_blacklist);
 	free(env_opts.pci_whitelist);
 
+
 	if (rc < 0) {
 		SPDK_ERRLOG("Unable to initialize SPDK env\n");
 	}
@@ -589,6 +599,7 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn,
 	int			rc;
 	char			*tty;
 	struct spdk_cpuset	tmp_cpumask = {};
+	static bool		g_env_was_setup = false;
 
 	if (!opts) {
 		SPDK_ERRLOG("opts should not be NULL\n");
@@ -644,7 +655,10 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn,
 
 	spdk_log_set_level(SPDK_APP_DEFAULT_LOG_LEVEL);
 
-	if (app_setup_env(opts) < 0) {
+	/* Pass NULL to app_setup_env if SPDK app has been set up, in order to
+	 * indicate that this is a reinitialization.
+	 */
+	if (app_setup_env(g_env_was_setup ? NULL : opts) < 0) {
 		return 1;
 	}
 
@@ -695,6 +709,8 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn,
 	/* This blocks until spdk_app_stop is called */
 	spdk_reactors_start();
 
+	g_env_was_setup = true;
+
 	return g_spdk_app.rc;
 }
 

module/bdev/malloc/bdev_malloc.c

@@ -481,6 +481,8 @@ static int bdev_malloc_initialize(void)
 	uint64_t size;
 	struct spdk_bdev *bdev;
 
+	malloc_disk_count = 0;
+
 	if (sp != NULL) {
 		NumberOfLuns = spdk_conf_section_get_intval(sp, "NumberOfLuns");
 		LunSizeInMB = spdk_conf_section_get_intval(sp, "LunSizeInMB");