From 396c445cb13727680cf762a9ab5425a20b1465fe Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Fri, 6 Dec 2019 16:26:49 -0700 Subject: [PATCH] env_dpdk: tell spdk_mem_map_init whether legacy_mem was specified We will use this in a future patch to determine whether it's safe to use DPDK allocated memory when allocating new 1gb page entries. We could use it in this patch to decide whether or not to register the memory hotplug handler, but there's really no harm registering it even when it's not needed. Ideally DPDK would provide some kind of API to query how DPDK was configured. In the normal case we know whether legacy-mem was specified, but if users initialize DPDK themselves and then call spdk_env_dpdk_post_init(), we won't know if legacy-mem was specified. So in that case, we will just assume that it wasn't specified. Signed-off-by: Jim Harris Change-Id: Ied0e5ff777c8ee651043f46a37ce62e44bfcc5fe Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/477086 Tested-by: SPDK CI Jenkins Reviewed-by: Changpeng Liu Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker Community-CI: SPDK CI Jenkins --- include/spdk/env_dpdk.h | 4 +++- lib/env_dpdk/env_internal.h | 2 +- lib/env_dpdk/init.c | 12 +++++++++--- lib/env_dpdk/memory.c | 6 +++++- test/env/env_dpdk_post_init/env_dpdk_post_init.c | 2 +- test/env/memory/memory_ut.c | 2 +- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/include/spdk/env_dpdk.h b/include/spdk/env_dpdk.h index 9729ad560..2240558ad 100644 --- a/include/spdk/env_dpdk.h +++ b/include/spdk/env_dpdk.h @@ -48,9 +48,11 @@ extern "C" { * instead of spdk_env_init, prior to using any other functions in SPDK * env library. * + * \param legacy_mem Indicates whether DPDK was initialized with --legacy-mem + * eal parameter. * \return 0 on success, or negative errno on failure. */ -int spdk_env_dpdk_post_init(void); +int spdk_env_dpdk_post_init(bool legacy_mem); /** * Release any resources of the environment library that were alllocated with diff --git a/lib/env_dpdk/env_internal.h b/lib/env_dpdk/env_internal.h index 39ba35371..2a525a3ae 100644 --- a/lib/env_dpdk/env_internal.h +++ b/lib/env_dpdk/env_internal.h @@ -79,7 +79,7 @@ int spdk_pci_device_fini(struct rte_pci_device *device); void spdk_pci_init(void); void spdk_pci_fini(void); -int spdk_mem_map_init(void); +int spdk_mem_map_init(bool legacy_mem); int spdk_vtophys_init(void); /** diff --git a/lib/env_dpdk/init.c b/lib/env_dpdk/init.c index 9fed7c00f..ab2b9ac24 100644 --- a/lib/env_dpdk/init.c +++ b/lib/env_dpdk/init.c @@ -480,13 +480,13 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts) } int -spdk_env_dpdk_post_init(void) +spdk_env_dpdk_post_init(bool legacy_mem) { int rc; spdk_pci_init(); - rc = spdk_mem_map_init(); + rc = spdk_mem_map_init(legacy_mem); if (rc < 0) { fprintf(stderr, "Failed to allocate mem_map\n"); return rc; @@ -515,6 +515,7 @@ spdk_env_init(const struct spdk_env_opts *opts) char **dpdk_args = NULL; int i, rc; int orig_optind; + bool legacy_mem; g_external_init = false; @@ -570,7 +571,12 @@ spdk_env_init(const struct spdk_env_opts *opts) spdk_env_unlink_shared_files(); } - return spdk_env_dpdk_post_init(); + legacy_mem = false; + if (opts->env_context && strstr(opts->env_context, "--legacy-mem") != NULL) { + legacy_mem = true; + } + + return spdk_env_dpdk_post_init(legacy_mem); } void diff --git a/lib/env_dpdk/memory.c b/lib/env_dpdk/memory.c index 215967048..5099517d2 100644 --- a/lib/env_dpdk/memory.c +++ b/lib/env_dpdk/memory.c @@ -146,6 +146,8 @@ static struct spdk_mem_map *g_mem_reg_map; static TAILQ_HEAD(, spdk_mem_map) g_spdk_mem_maps = TAILQ_HEAD_INITIALIZER(g_spdk_mem_maps); static pthread_mutex_t g_spdk_mem_map_mutex = PTHREAD_MUTEX_INITIALIZER; +static bool g_legacy_mem; + /* * Walk the currently registered memory via the main memory registration map * and call the new map's notify callback for each virtually contiguous region. @@ -700,8 +702,10 @@ memory_iter_cb(const struct rte_memseg_list *msl, #endif int -spdk_mem_map_init(void) +spdk_mem_map_init(bool legacy_mem) { + g_legacy_mem = legacy_mem; + g_mem_reg_map = spdk_mem_map_alloc(0, NULL, NULL); if (g_mem_reg_map == NULL) { DEBUG_PRINT("memory registration map allocation failed\n"); diff --git a/test/env/env_dpdk_post_init/env_dpdk_post_init.c b/test/env/env_dpdk_post_init/env_dpdk_post_init.c index 94a5685ae..1b3897ea8 100644 --- a/test/env/env_dpdk_post_init/env_dpdk_post_init.c +++ b/test/env/env_dpdk_post_init/env_dpdk_post_init.c @@ -104,7 +104,7 @@ main(int argc, char **argv) } printf("Starting SPDK post initialization...\n"); - ret = spdk_env_dpdk_post_init(); + ret = spdk_env_dpdk_post_init(false); if (ret < 0) { fprintf(stderr, "Failed to initialize SPDK\n"); return -1; diff --git a/test/env/memory/memory_ut.c b/test/env/memory/memory_ut.c index 0f7e036df..8122e73ba 100644 --- a/test/env/memory/memory_ut.c +++ b/test/env/memory/memory_ut.c @@ -489,7 +489,7 @@ main(int argc, char **argv) g_page_array = spdk_bit_array_create(PAGE_ARRAY_SIZE); /* Initialize the memory map */ - if (spdk_mem_map_init() < 0) { + if (spdk_mem_map_init(false) < 0) { return CUE_NOMEMORY; }