diff --git a/app/nvmf_tgt/nvmf_tgt.c b/app/nvmf_tgt/nvmf_tgt.c index 0ee94f6a9..624bfde5b 100644 --- a/app/nvmf_tgt/nvmf_tgt.c +++ b/app/nvmf_tgt/nvmf_tgt.c @@ -55,8 +55,6 @@ #include "spdk/nvme.h" #include "spdk/io_channel.h" -struct rte_mempool *request_mempool; - #define SPDK_NVMF_BUILD_ETC "/usr/local/etc/nvmf" #define SPDK_NVMF_DEFAULT_CONFIG SPDK_NVMF_BUILD_ETC "/nvmf.conf" @@ -70,11 +68,7 @@ static bool g_subsystems_shutdown; static void shutdown_complete(void) { - int rc; - - rc = spdk_nvmf_check_pools(); - - spdk_app_stop(rc); + spdk_app_stop(0); } static void @@ -380,7 +374,6 @@ This is the main file. nvmf=>nvmf [label="spdk_event_allocate()"]; nvmf=>nvmf [label="spdk_app_start()"]; nvmf=>nvmf [label="spdk_app_fini()"]; - nvmf=>nvmf [label="spdk_nvmf_check_pools()"]; c_runtime<done = true; } -size_t -spdk_nvme_request_size(void) -{ - return sizeof(struct nvme_request); -} - struct nvme_request * nvme_allocate_request(const struct nvme_payload *payload, uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, void *cb_arg) { struct nvme_request *req = NULL; - nvme_alloc_request(&req); - + nvme_mempool_get(g_spdk_nvme_driver->request_mempool, (void **)&req); if (req == NULL) { return req; } @@ -214,7 +208,7 @@ nvme_free_request(struct nvme_request *req) assert(req != NULL); assert(req->num_children == 0); - nvme_dealloc_request(req); + nvme_mempool_put(g_spdk_nvme_driver->request_mempool, req); } int @@ -284,6 +278,16 @@ spdk_nvme_probe(void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb a pthread_mutex_lock(&g_spdk_nvme_driver->lock); + if (g_spdk_nvme_driver->request_mempool == NULL) { + g_spdk_nvme_driver->request_mempool = nvme_mempool_create("nvme_request", 8192, + sizeof(struct nvme_request), 128); + if (g_spdk_nvme_driver->request_mempool == NULL) { + nvme_printf(NULL, "Unable to allocate pool of requests\n"); + pthread_mutex_unlock(&g_spdk_nvme_driver->lock); + return -1; + } + } + enum_ctx.probe_cb = probe_cb; enum_ctx.cb_ctx = cb_ctx; diff --git a/lib/nvme/nvme_impl.h b/lib/nvme/nvme_impl.h index 8c96d289a..acfdb0ab5 100644 --- a/lib/nvme/nvme_impl.h +++ b/lib/nvme/nvme_impl.h @@ -83,9 +83,7 @@ /** * Allocate a pinned, physically contiguous memory buffer with the * given size and alignment. - * Note: these calls are only made during driver initialization. Per - * I/O allocations during driver operation use the nvme_alloc_request - * callback. + * Note: these calls are only made during driver initialization. */ static inline void * nvme_malloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr) @@ -173,18 +171,41 @@ nvme_process_is_primary(void) #define nvme_vtophys(buf) spdk_vtophys(buf) #define NVME_VTOPHYS_ERROR SPDK_VTOPHYS_ERROR -extern struct rte_mempool *request_mempool; +typedef struct rte_mempool nvme_mempool_t; /** - * Return a buffer for an nvme_request object. These objects are allocated - * for each I/O. They do not need to be pinned nor physically contiguous. + * Create a mempool with the given configuration. + * Return a pointer to the allocated memory address. If the allocation + * cannot be done, return NULL. */ -#define nvme_alloc_request(bufp) rte_mempool_get(request_mempool, (void **)(bufp)); +static inline nvme_mempool_t * +nvme_mempool_create(const char *name, unsigned n, unsigned elt_size, + unsigned cache_size) +{ + struct rte_mempool *mp; -/** - * Free a buffer previously allocated with nvme_alloc_request(). - */ -#define nvme_dealloc_request(buf) rte_mempool_put(request_mempool, buf) + mp = rte_mempool_create(name, n, elt_size, cache_size, + 0, NULL, NULL, NULL, NULL, + SOCKET_ID_ANY, 0); + + if (mp == NULL) { + return NULL; + } + + return (nvme_mempool_t *)mp; +} + +static inline void +nvme_mempool_get(nvme_mempool_t *mp, void **buf) +{ + rte_mempool_get(mp, buf); +} + +static inline void +nvme_mempool_put(nvme_mempool_t *mp, void *buf) +{ + rte_mempool_put(mp, buf); +} /** * Get a monotonic timestamp counter (used for measuring timeouts during initialization). diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index f8aaf882a..6f1eee98c 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -464,6 +464,7 @@ struct nvme_driver { pthread_mutex_t lock; TAILQ_HEAD(, spdk_nvme_ctrlr) init_ctrlrs; TAILQ_HEAD(, spdk_nvme_ctrlr) attached_ctrlrs; + nvme_mempool_t *request_mempool; }; struct pci_id { diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 5fd111d2d..108d88ca0 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -49,77 +49,10 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF) struct spdk_nvmf_globals g_nvmf_tgt; -extern struct rte_mempool *request_mempool; -static unsigned g_num_requests; - -static int -spdk_nvmf_initialize_pools(void) -{ - SPDK_NOTICELOG("\n*** NVMf Pool Creation ***\n"); - - g_num_requests = MAX_SUBSYSTEMS * g_nvmf_tgt.max_queues_per_session * g_nvmf_tgt.max_queue_depth; - - /* create NVMe backend request pool */ - request_mempool = rte_mempool_create("NVMe_Pool", - g_num_requests, - spdk_nvme_request_size(), - 128, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - if (!request_mempool) { - SPDK_ERRLOG("create NVMe request pool failed\n"); - return -1; - } - - SPDK_TRACELOG(SPDK_TRACE_DEBUG, "NVMe request_mempool %p, size %" PRIu64 " bytes\n", - request_mempool, - (uint64_t)g_num_requests * spdk_nvme_request_size()); - - return 0; -} - -/* - * Wrapper to provide rte_mempool_avail_count() on older DPDK versions. - * Drop this if the minimum DPDK version is raised to at least 16.07. - */ -#if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1) -static unsigned rte_mempool_avail_count(const struct rte_mempool *pool) -{ - return rte_mempool_count(pool); -} -#endif - -static int spdk_nvmf_check_pool(struct rte_mempool *pool, uint32_t count) -{ - if (rte_mempool_avail_count(pool) != count) { - SPDK_ERRLOG("rte_mempool_avail_count(%s) == %d, should be %d\n", - pool->name, rte_mempool_avail_count(pool), count); - return -1; - } else { - return 0; - } -} - -int -spdk_nvmf_check_pools(void) -{ - int rc = 0; - - rc += spdk_nvmf_check_pool(request_mempool, g_num_requests); - - if (rc == 0) { - return 0; - } else { - return -1; - } -} - int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, uint32_t in_capsule_data_size, uint32_t max_io_size) { - int rc; - g_nvmf_tgt.max_queues_per_session = max_queues_per_sess; g_nvmf_tgt.max_queue_depth = max_queue_depth; g_nvmf_tgt.in_capsule_data_size = in_capsule_data_size; @@ -130,12 +63,6 @@ nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max In Capsule Data: %d bytes\n", in_capsule_data_size); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max I/O Size: %d bytes\n", max_io_size); - rc = spdk_nvmf_initialize_pools(); - if (rc != 0) { - SPDK_ERRLOG("spdk_nvmf_initialize_pools() failed\n"); - return rc; - } - return 0; } diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index a22fd8691..078b2a4bc 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -73,8 +73,6 @@ struct spdk_nvmf_globals { int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess, uint32_t in_capsule_data_size, uint32_t max_io_size); -int spdk_nvmf_check_pools(void); - static inline uint32_t nvmf_u32log2(uint32_t x) { diff --git a/test/lib/nvme/aer/aer.c b/test/lib/nvme/aer/aer.c index f1e619018..d6f696a53 100644 --- a/test/lib/nvme/aer/aer.c +++ b/test/lib/nvme/aer/aer.c @@ -41,8 +41,6 @@ #include "spdk/nvme.h" #include "spdk/pci.h" -struct rte_mempool *request_mempool; - #define MAX_DEVS 64 struct dev { @@ -256,16 +254,6 @@ int main(int argc, char **argv) exit(1); } - request_mempool = rte_mempool_create("nvme_request", 8192, - spdk_nvme_request_size(), 128, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - - if (request_mempool == NULL) { - fprintf(stderr, "could not initialize request mempool\n"); - exit(1); - } - if (spdk_nvme_probe(NULL, probe_cb, attach_cb, NULL) != 0) { fprintf(stderr, "spdk_nvme_probe() failed\n"); return 1; diff --git a/test/lib/nvme/e2edp/nvme_dp.c b/test/lib/nvme/e2edp/nvme_dp.c index 95062e2a6..01f65958d 100644 --- a/test/lib/nvme/e2edp/nvme_dp.c +++ b/test/lib/nvme/e2edp/nvme_dp.c @@ -68,8 +68,6 @@ static uint16_t swap16(uint16_t value) return result; } -struct rte_mempool *request_mempool; - #define MAX_DEVS 64 #define DATA_PATTERN 0x5A @@ -584,16 +582,6 @@ int main(int argc, char **argv) exit(1); } - request_mempool = rte_mempool_create("nvme_request", 8192, - spdk_nvme_request_size(), 128, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - - if (request_mempool == NULL) { - fprintf(stderr, "could not initialize request mempool\n"); - exit(1); - } - if (spdk_nvme_probe(NULL, probe_cb, attach_cb, NULL) != 0) { fprintf(stderr, "nvme_probe() failed\n"); exit(1); diff --git a/test/lib/nvme/overhead/overhead.c b/test/lib/nvme/overhead/overhead.c index 91ab23464..750a86a8f 100644 --- a/test/lib/nvme/overhead/overhead.c +++ b/test/lib/nvme/overhead/overhead.c @@ -98,8 +98,6 @@ struct perf_task { #endif }; -struct rte_mempool *request_mempool; - static struct ctrlr_entry *g_ctrlr = NULL; static struct ns_entry *g_ns = NULL; @@ -614,16 +612,6 @@ int main(int argc, char **argv) return 1; } - request_mempool = rte_mempool_create("nvme_request", 8192, - spdk_nvme_request_size(), 128, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - - if (request_mempool == NULL) { - fprintf(stderr, "could not initialize request mempool\n"); - return 1; - } - g_task = rte_zmalloc("task", sizeof(struct perf_task), 0); if (g_task == NULL) { fprintf(stderr, "g_task alloc failed\n"); diff --git a/test/lib/nvme/reset/reset.c b/test/lib/nvme/reset/reset.c index b969d2e1c..4210c40ef 100644 --- a/test/lib/nvme/reset/reset.c +++ b/test/lib/nvme/reset/reset.c @@ -84,7 +84,6 @@ struct worker_thread { unsigned lcore; }; -struct rte_mempool *request_mempool; static struct rte_mempool *task_pool; static struct ctrlr_entry *g_controllers = NULL; @@ -648,16 +647,6 @@ int main(int argc, char **argv) return 1; } - request_mempool = rte_mempool_create("nvme_request", 8192, - spdk_nvme_request_size(), 128, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - - if (request_mempool == NULL) { - fprintf(stderr, "could not initialize request mempool\n"); - return 1; - } - task_pool = rte_mempool_create("task_pool", 8192, sizeof(struct reset_task), 64, 0, NULL, NULL, task_ctor, NULL, diff --git a/test/lib/nvme/sgl/nvme_sgl.c b/test/lib/nvme/sgl/nvme_sgl.c index 3cc7a46ac..3decc5bd5 100644 --- a/test/lib/nvme/sgl/nvme_sgl.c +++ b/test/lib/nvme/sgl/nvme_sgl.c @@ -43,8 +43,6 @@ #include "spdk/nvme.h" #include "spdk/pci.h" -struct rte_mempool *request_mempool; - #define MAX_DEVS 64 #define MAX_IOVS 128 @@ -431,16 +429,6 @@ int main(int argc, char **argv) exit(1); } - request_mempool = rte_mempool_create("nvme_request", 8192, - spdk_nvme_request_size(), 128, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - - if (request_mempool == NULL) { - fprintf(stderr, "could not initialize request mempool\n"); - exit(1); - } - if (spdk_nvme_probe(NULL, probe_cb, attach_cb, NULL) != 0) { fprintf(stderr, "nvme_probe() failed\n"); exit(1); diff --git a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c index 7036bca43..190c03f89 100644 --- a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c +++ b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c @@ -37,6 +37,7 @@ struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, + .request_mempool = NULL, }; static uint16_t g_pci_vendor_id; @@ -119,7 +120,7 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re * Free the request here so it does not leak. * For the purposes of this unit test, we don't need to bother emulating request submission. */ - nvme_dealloc_request(req); + nvme_mempool_put(_g_nvme_driver.request_mempool, req); return 0; } @@ -286,7 +287,7 @@ nvme_allocate_request(const struct nvme_payload *payload, uint32_t payload_size, void *cb_arg) { struct nvme_request *req = NULL; - nvme_alloc_request(&req); + nvme_mempool_get(_g_nvme_driver.request_mempool, (void **)&req); if (req != NULL) { memset(req, 0, offsetof(struct nvme_request, children)); diff --git a/test/lib/nvme/unit/nvme_impl.h b/test/lib/nvme/unit/nvme_impl.h index 83dafa47f..dc7d731f8 100644 --- a/test/lib/nvme/unit/nvme_impl.h +++ b/test/lib/nvme/unit/nvme_impl.h @@ -66,17 +66,6 @@ extern char outbuf[OUTBUF_SIZE]; uint64_t nvme_vtophys(void *buf); #define NVME_VTOPHYS_ERROR (0xFFFFFFFFFFFFFFFFULL) -#define nvme_alloc_request(bufp) \ -do \ - { \ - if (posix_memalign((void **)(bufp), 64, sizeof(struct nvme_request))) { \ - *(bufp) = NULL; \ - } \ - } \ - while (0) - -#define nvme_dealloc_request(buf) free(buf) - extern uint64_t g_ut_tsc; #define nvme_get_tsc() (g_ut_tsc) #define nvme_get_tsc_hz() (1000000) @@ -146,4 +135,29 @@ nvme_process_is_primary(void) return true; } +#define NVME_SOCKET_ID_ANY -1 + +typedef unsigned nvme_mempool_t; + +static inline nvme_mempool_t * +nvme_mempool_create(const char *name, unsigned n, unsigned elt_size, + unsigned cache_size) +{ + static int mp; + + return ∓ +} + +static inline void +nvme_mempool_get(nvme_mempool_t *mp, void **buf) +{ + posix_memalign(buf, 64, 0x1000); +} + +static inline void +nvme_mempool_put(nvme_mempool_t *mp, void *buf) +{ + free(buf); +} + #endif /* __NVME_IMPL_H__ */ diff --git a/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c index 4955815f7..d815537b3 100644 --- a/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c +++ b/test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c @@ -40,6 +40,7 @@ struct nvme_driver _g_nvme_driver = { .lock = PTHREAD_MUTEX_INITIALIZER, + .request_mempool = NULL, }; int32_t spdk_nvme_retry_count = 1; @@ -129,7 +130,7 @@ nvme_allocate_request(const struct nvme_payload *payload, uint32_t payload_size, { struct nvme_request *req = NULL; - nvme_alloc_request(&req); + nvme_mempool_get(_g_nvme_driver.request_mempool, (void **)&req); if (req == NULL) { return req; @@ -173,7 +174,7 @@ nvme_allocate_request_null(spdk_nvme_cmd_cb cb_fn, void *cb_arg) void nvme_free_request(struct nvme_request *req) { - nvme_dealloc_request(req); + nvme_mempool_put(_g_nvme_driver.request_mempool, req); } void @@ -263,7 +264,7 @@ ut_insert_cq_entry(struct spdk_nvme_qpair *qpair, uint32_t slot) struct nvme_tracker *tr; struct spdk_nvme_cpl *cpl; - nvme_alloc_request(&req); + nvme_mempool_get(_g_nvme_driver.request_mempool, (void **)&req); SPDK_CU_ASSERT_FATAL(req != NULL); memset(req, 0, sizeof(*req)); diff --git a/test/lib/nvmf/nvmf/nvmf_ut.c b/test/lib/nvmf/nvmf/nvmf_ut.c index 44d000822..568c3d271 100644 --- a/test/lib/nvmf/nvmf/nvmf_ut.c +++ b/test/lib/nvmf/nvmf/nvmf_ut.c @@ -81,8 +81,6 @@ struct spdk_nvme_ctrlr { static int controller_checked[20]; -struct rte_mempool *request_mempool; - int spdk_nvmf_parse_conf(void) { return 0; @@ -218,12 +216,6 @@ spdk_nvme_ns_get_data(struct spdk_nvme_ns *ns) return nsdata; } -size_t -spdk_nvme_request_size(void) -{ - return 0; -} - int spdk_nvme_detach(struct spdk_nvme_ctrlr *ctrlr) { @@ -316,14 +308,8 @@ nvmf_test_init(void) struct spdk_nvme_ctrlr *ctrlr; uint32_t i; - request_mempool = NULL; - /* test that NVMf library will trap if mempool not created */ - rc = nvmf_initialize(); - CU_ASSERT(rc < 0); - request_mempool = malloc(sizeof(struct rte_mempool)); rc = nvmf_initialize(); CU_ASSERT(rc == 0); - free(request_mempool); /* create faked controller */ ctrlr = malloc(sizeof(struct spdk_nvme_ctrlr)); SPDK_CU_ASSERT_FATAL(ctrlr != NULL);