From 31d2a116c8bd347cbc4289f96cd9b6ea793591d0 Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Thu, 23 Jun 2016 15:43:41 -0700 Subject: [PATCH] nvmf: remove rx_desc and tx_desc mempools The RDMA tx_desc and rx_desc pools were only used at startup; all descriptors are immediately allocated and put into a queue, and the mempool functionality was never used at runtime. Change-Id: I2882274962550191a555c8483b8f7be2854b32ec Signed-off-by: Daniel Verkamp --- lib/nvmf/nvmf.c | 39 --------------------------------------- lib/nvmf/nvmf_internal.h | 2 -- lib/nvmf/rdma.c | 23 +++++++++++------------ 3 files changed, 11 insertions(+), 53 deletions(-) diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 61941505c..8808e9342 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -65,9 +65,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF) * * SPDK_NVMF_DESC_POOL_SIZE: The total number of RDMA descriptors * needed for all possible admin and I/O queue requests. - * - * SPDK_NVMF_TX_DESC_POOL_SIZE: The total number of RDMA descriptors - * needed for all possible admin and I/O queue tx requests. */ #define SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) (MAX_SUBSYSTEMS * \ (spdk->MaxSessionsPerSubsystem) * \ @@ -81,10 +78,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF) #define SPDK_NVMF_DESC_POOL_SIZE(spdk) (SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \ SPDK_NVMF_IOQ_POOL_SIZE(spdk)) -/* Per each connection, cq depth is double of sq depth. Double the memory accordingly. */ -#define SPDK_NVMF_TX_DESC_POOL_SIZE(spdk) ((SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \ - SPDK_NVMF_IOQ_POOL_SIZE(spdk)) * 2) - #define SPDK_NVMF_MAX_CONNECTIONS(spdk) (MAX_SUBSYSTEMS * \ ((spdk)->MaxSessionsPerSubsystem) * \ ((spdk)->MaxConnectionsPerSession)) @@ -145,36 +138,6 @@ spdk_nvmf_initialize_pools(struct spdk_nvmf_globals *spdk_nvmf) spdk_nvmf->bb_large_pool, (SPDK_NVMF_IOQ_POOL_SIZE(spdk_nvmf) * LARGE_BB_MAX_SIZE)); - spdk_nvmf->rx_desc_pool = - rte_mempool_create("RDMA RX Desc Pool", - SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf), - sizeof(struct nvme_qp_rx_desc), - 0, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - if (!spdk_nvmf->rx_desc_pool) { - SPDK_ERRLOG("create RX Desc pool failed\n"); - return -1; - } - SPDK_TRACELOG(SPDK_TRACE_DEBUG, "RDMA Rx descriptor pool %p, size 0x%lx bytes\n", - spdk_nvmf->rx_desc_pool, - (SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf) * sizeof(struct nvme_qp_rx_desc))); - - spdk_nvmf->tx_desc_pool = - rte_mempool_create("RDMA TX Desc Pool", - SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf), - sizeof(struct nvme_qp_tx_desc), - 0, 0, - NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); - if (!spdk_nvmf->tx_desc_pool) { - SPDK_ERRLOG("create TX Desc pool failed\n"); - return -1; - } - SPDK_TRACELOG(SPDK_TRACE_DEBUG, "RDMA Tx descriptor pool %p, size 0x%lx bytes\n", - spdk_nvmf->tx_desc_pool, - (SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf) * sizeof(struct nvme_qp_tx_desc))); - return 0; } @@ -196,8 +159,6 @@ spdk_nvmf_check_pools(void) int rc = 0; rc += spdk_nvmf_check_pool(spdk_nvmf->nvme_request_pool, SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf)); - rc += spdk_nvmf_check_pool(spdk_nvmf->rx_desc_pool, SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf)); - rc += spdk_nvmf_check_pool(spdk_nvmf->tx_desc_pool, SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf)); rc += spdk_nvmf_check_pool(spdk_nvmf->bb_small_pool, SPDK_NVMF_ADMINQ_POOL_SIZE(spdk_nvmf)); rc += spdk_nvmf_check_pool(spdk_nvmf->bb_large_pool, SPDK_NVMF_IOQ_POOL_SIZE(spdk_nvmf)); diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index a67b35ad6..cb7e9c89d 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -113,8 +113,6 @@ struct spdk_nvmf_globals { int MaxConnectionsPerSession; int MaxRecvDataSegmentLength; - struct rte_mempool *rx_desc_pool; - struct rte_mempool *tx_desc_pool; struct rte_mempool *nvme_request_pool; struct rte_mempool *bb_small_pool; struct rte_mempool *bb_large_pool; diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index a714f265c..6c1508315 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include "conn.h" @@ -186,7 +187,7 @@ free_qp_desc(struct spdk_nvmf_conn *conn) SPDK_ERRLOG("Unable to de-register rx mr\n"); } - rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)tmp_rx); + rte_free(tmp_rx); } STAILQ_FOREACH(tmp_tx, &conn->rdma.qp_tx_desc, link) { @@ -197,7 +198,7 @@ free_qp_desc(struct spdk_nvmf_conn *conn) SPDK_ERRLOG("Unable to de-register tx mr\n"); } - rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tmp_tx); + rte_free(tmp_tx); } } @@ -910,9 +911,8 @@ alloc_qp_rx_desc(struct spdk_nvmf_conn *conn) /* Allocate buffer for rx descriptors (RX WQE + Msg Buffer) */ for (i = 0; i < conn->sq_depth; i++) { - rx_desc = NULL; - rc = rte_mempool_get(g_nvmf_tgt.rx_desc_pool, (void **)&rx_desc); - if ((rc < 0) || !rx_desc) { + rx_desc = rte_zmalloc("nvmf_rx_desc", sizeof(*rx_desc), 0); + if (!rx_desc) { SPDK_ERRLOG("Unable to get rx desc object\n"); goto fail; } @@ -998,7 +998,7 @@ fail: } } - rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)rx_desc); + rte_free(rx_desc); } STAILQ_FOREACH(tmp, &conn->rdma.qp_rx_desc, link) { @@ -1020,7 +1020,7 @@ fail: SPDK_ERRLOG("Unable to de-register rx mr\n"); } - rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)tmp); + rte_free(tmp); } return -ENOMEM; @@ -1036,9 +1036,8 @@ alloc_qp_tx_desc(struct spdk_nvmf_conn *conn) /* Initialize the tx descriptors */ for (i = 0; i < conn->cq_depth; i++) { - tx_desc = NULL; - rc = rte_mempool_get(g_nvmf_tgt.tx_desc_pool, (void **)&tx_desc); - if ((rc < 0) || !tx_desc) { + tx_desc = rte_zmalloc("nvmf_tx_desc", sizeof(*tx_desc), 0); + if (!tx_desc) { SPDK_ERRLOG("Unable to get tx desc object\n"); goto fail; } @@ -1079,7 +1078,7 @@ fail: } } - rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tx_desc); + rte_free(tx_desc); } STAILQ_FOREACH(tmp, &conn->rdma.qp_tx_desc, link) { @@ -1090,7 +1089,7 @@ fail: SPDK_ERRLOG("Unable to de-register tx mr\n"); } - rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tmp); + rte_free(tmp); } return -ENOMEM;