From 1d9be84bfdf80072422b5bc64a498714af04ca16 Mon Sep 17 00:00:00 2001 From: Seth Howell Date: Wed, 10 Oct 2018 15:19:31 -0700 Subject: [PATCH] nvmf/rdma: change the default buffer size. Having the buffers be the same size as the maximum xfer size doesn't do us any favors. Make these buffers a ratio of the maximum transfer size and the number of supported nvmf SGLs. Also configure the number of nvmf request iovs to correspond with this new ratio. Change-Id: I3147dcd86b599c74521ebfdf3bcdbcdee8871a3a Signed-off-by: Seth Howell Reviewed-on: https://review.gerrithub.io/428747 Chandler-Test-Pool: SPDK Automated Test System Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris Reviewed-by: Changpeng Liu Reviewed-by: Shuhei Matsumoto --- lib/nvmf/nvmf_internal.h | 2 +- lib/nvmf/rdma.c | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index e86c00588..07c8a316b 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -151,7 +151,7 @@ struct spdk_nvmf_request { void *data; union nvmf_h2c_msg *cmd; union nvmf_c2h_msg *rsp; - struct iovec iov[SPDK_NVMF_MAX_SGL_ENTRIES]; + struct iovec iov[SPDK_NVMF_MAX_SGL_ENTRIES * 2]; uint32_t iovcnt; struct spdk_bdev_io_wait_entry bdev_io_wait; diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 0765bcbbe..0a4d8809d 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -1499,7 +1499,8 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, #define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 64 #define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 #define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072 -#define SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE 131072 +#define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE 4096 +#define SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES) static void spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) @@ -1508,7 +1509,8 @@ spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR; opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE; opts->max_io_size = SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE; - opts->io_unit_size = SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE; + opts->io_unit_size = spdk_max(SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE, + SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE); opts->max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH; } @@ -1589,9 +1591,10 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts) return NULL; } + /* The maximum number of buffers we will need for a given request is equal to just less than double the number of SGL elements */ rtransport->data_buf_pool = spdk_mempool_create("spdk_nvmf_rdma", - opts->max_queue_depth * 4, /* The 4 is arbitrarily chosen. Needs to be configurable. */ - opts->max_io_size + NVMF_DATA_BUFFER_ALIGNMENT, + opts->max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4, + opts->io_unit_size + NVMF_DATA_BUFFER_ALIGNMENT, SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, SPDK_ENV_SOCKET_ID_ANY); if (!rtransport->data_buf_pool) { @@ -1740,10 +1743,10 @@ spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport) if (rtransport->data_buf_pool != NULL) { if (spdk_mempool_count(rtransport->data_buf_pool) != - (transport->opts.max_queue_depth * 4)) { + (transport->opts.max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4)) { SPDK_ERRLOG("transport buffer pool count is %zu but should be %u\n", spdk_mempool_count(rtransport->data_buf_pool), - transport->opts.max_queue_depth * 4); + transport->opts.max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4); } }