From ef7c128a8a7f2b8b36ed14412d4e1b1628a6ca1f Mon Sep 17 00:00:00 2001 From: Xiaodong Liu Date: Fri, 8 May 2020 02:03:22 -0400 Subject: [PATCH] lib/event: enable repeated spdk_app_start/stop With this patch, spdk_app_start/stop can be repeatedly called by users based on their upper level application's requirement. Changes are: * Add reinit ability inside spdk_env_init and related functions * Clear g_shutdown_sig_received in spdk_app_setup_signal_handlers * Clear malloc_disk_count in bdev_malloc_initialize Change-Id: I2d7be52b0e4aac2cb6734cc1237ce72d33b6de0c Signed-off-by: Xiaodong Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2260 Community-CI: Mellanox Build Bot Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins --- include/spdk/env.h | 12 ++++++++---- lib/env_dpdk/env_internal.h | 1 + lib/env_dpdk/init.c | 26 ++++++++++++++++++++++++++ lib/env_dpdk/pci.c | 32 ++++++++++++++++++++++++-------- lib/event/app.c | 22 +++++++++++++++++++--- module/bdev/malloc/bdev_malloc.c | 2 ++ 6 files changed, 80 insertions(+), 15 deletions(-) diff --git a/include/spdk/env.h b/include/spdk/env.h index c02232275..f8cb058dd 100644 --- a/include/spdk/env.h +++ b/include/spdk/env.h @@ -159,8 +159,11 @@ void spdk_free(void *buf); void spdk_env_opts_init(struct spdk_env_opts *opts); /** - * Initialize the environment library. This must be called prior to using - * any other functions in this library. + * Initialize or reinitialize the environment library. + * For initialization, this must be called prior to using any other functions + * in this library. For reinitialization, the parameter `opts` must be set to + * NULL and this must be called after the environment library was finished by + * spdk_env_fini() within the same process. * * \param opts Environment initialization options. * \return 0 on success, or negative errno on failure. @@ -168,10 +171,11 @@ void spdk_env_opts_init(struct spdk_env_opts *opts); int spdk_env_init(const struct spdk_env_opts *opts); /** - * Release any resources of the environment library that were alllocated with + * Release any resources of the environment library that were allocated with * spdk_env_init(). After this call, no SPDK env function calls may be made. * It is expected that common usage of this function is to call it just before - * terminating the process. + * terminating the process or before reinitializing the environment library + * within the same process. */ void spdk_env_fini(void); diff --git a/lib/env_dpdk/env_internal.h b/lib/env_dpdk/env_internal.h index 3643aa316..2f2b2f947 100644 --- a/lib/env_dpdk/env_internal.h +++ b/lib/env_dpdk/env_internal.h @@ -77,6 +77,7 @@ int pci_device_init(struct rte_pci_driver *driver, struct rte_pci_device *device int pci_device_fini(struct rte_pci_device *device); void pci_env_init(void); +void pci_env_reinit(void); void pci_env_fini(void); int mem_map_init(bool legacy_mem); int vtophys_init(void); diff --git a/lib/env_dpdk/init.c b/lib/env_dpdk/init.c index 814159824..0376dbe7b 100644 --- a/lib/env_dpdk/init.c +++ b/lib/env_dpdk/init.c @@ -128,6 +128,10 @@ free_args(char **args, int argcount) { int i; + if (args == NULL) { + return; + } + for (i = 0; i < argcount; i++) { free(args[i]); } @@ -501,6 +505,8 @@ spdk_env_dpdk_post_fini(void) pci_env_fini(); free_args(g_eal_cmdline, g_eal_cmdline_argcount); + g_eal_cmdline = NULL; + g_eal_cmdline_argcount = 0; } int @@ -511,6 +517,26 @@ spdk_env_init(const struct spdk_env_opts *opts) int orig_optind; bool legacy_mem; + /* If SPDK env has been initialized before, then only pci env requires + * reinitialization. + */ + if (g_external_init == false) { + if (opts != NULL) { + fprintf(stderr, "Invalid arguments to reinitialize SPDK env\n"); + return -EINVAL; + } + + printf("Starting %s / %s reinitialization...\n", SPDK_VERSION_STRING, rte_version()); + pci_env_reinit(); + + return 0; + } + + if (opts == NULL) { + fprintf(stderr, "NULL arguments to initialize DPDK\n"); + return -EINVAL; + } + rc = build_eal_cmdline(opts); if (rc < 0) { fprintf(stderr, "Invalid arguments to initialize DPDK\n"); diff --git a/lib/env_dpdk/pci.c b/lib/env_dpdk/pci.c index d729d7435..80bb19246 100644 --- a/lib/env_dpdk/pci.c +++ b/lib/env_dpdk/pci.c @@ -239,15 +239,9 @@ cleanup_pci_devices(void) static int scan_pci_bus(bool delay_init); -void -pci_env_init(void) +static inline void +_pci_env_init(void) { - struct spdk_pci_driver *driver; - - TAILQ_FOREACH(driver, &g_pci_drivers, tailq) { - rte_pci_register(&driver->driver); - } - /* We assume devices were present on the bus for more than 2 seconds * before initializing SPDK and there's no need to wait more. We scan * the bus, but we don't blacklist any devices. @@ -260,6 +254,28 @@ pci_env_init(void) } } +void +pci_env_init(void) +{ + struct spdk_pci_driver *driver; + + TAILQ_FOREACH(driver, &g_pci_drivers, tailq) { + rte_pci_register(&driver->driver); + } + + _pci_env_init(); +} + +void +pci_env_reinit(void) +{ + /* There is no need to register pci drivers again, since they were + * already pre-registered in pci_env_init. + */ + + _pci_env_init(); +} + void pci_env_fini(void) { diff --git a/lib/event/app.c b/lib/event/app.c index 9ec47d153..b6cab05a3 100644 --- a/lib/event/app.c +++ b/lib/event/app.c @@ -315,8 +315,8 @@ app_setup_signal_handlers(struct spdk_app_opts *opts) } /* Install the same handler for SIGINT and SIGTERM */ + g_shutdown_sig_received = false; sigact.sa_handler = __shutdown_signal; - rc = sigaction(SIGINT, &sigact, NULL); if (rc < 0) { SPDK_ERRLOG("sigaction(SIGINT) failed\n"); @@ -497,6 +497,16 @@ app_setup_env(struct spdk_app_opts *opts) struct spdk_env_opts env_opts = {}; int rc; + if (opts == NULL) { + rc = spdk_env_init(NULL); + if (rc != 0) { + SPDK_ERRLOG("Unable to reinitialize SPDK env\n"); + } + + return rc; + } + + spdk_env_opts_init(&env_opts); env_opts.name = opts->name; @@ -512,7 +522,6 @@ app_setup_env(struct spdk_app_opts *opts) env_opts.num_pci_addr = opts->num_pci_addr; env_opts.pci_blacklist = opts->pci_blacklist; env_opts.pci_whitelist = opts->pci_whitelist; - env_opts.base_virtaddr = opts->base_virtaddr; env_opts.env_context = opts->env_context; env_opts.iova_mode = opts->iova_mode; @@ -520,6 +529,7 @@ app_setup_env(struct spdk_app_opts *opts) free(env_opts.pci_blacklist); free(env_opts.pci_whitelist); + if (rc < 0) { SPDK_ERRLOG("Unable to initialize SPDK env\n"); } @@ -589,6 +599,7 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn, int rc; char *tty; struct spdk_cpuset tmp_cpumask = {}; + static bool g_env_was_setup = false; if (!opts) { SPDK_ERRLOG("opts should not be NULL\n"); @@ -644,7 +655,10 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn, spdk_log_set_level(SPDK_APP_DEFAULT_LOG_LEVEL); - if (app_setup_env(opts) < 0) { + /* Pass NULL to app_setup_env if SPDK app has been set up, in order to + * indicate that this is a reinitialization. + */ + if (app_setup_env(g_env_was_setup ? NULL : opts) < 0) { return 1; } @@ -695,6 +709,8 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_msg_fn start_fn, /* This blocks until spdk_app_stop is called */ spdk_reactors_start(); + g_env_was_setup = true; + return g_spdk_app.rc; } diff --git a/module/bdev/malloc/bdev_malloc.c b/module/bdev/malloc/bdev_malloc.c index 53156dc3a..ed7e94afc 100644 --- a/module/bdev/malloc/bdev_malloc.c +++ b/module/bdev/malloc/bdev_malloc.c @@ -481,6 +481,8 @@ static int bdev_malloc_initialize(void) uint64_t size; struct spdk_bdev *bdev; + malloc_disk_count = 0; + if (sp != NULL) { NumberOfLuns = spdk_conf_section_get_intval(sp, "NumberOfLuns"); LunSizeInMB = spdk_conf_section_get_intval(sp, "LunSizeInMB");