ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf/rdma: change the default buffer size.

Browse Source 
Having the buffers be the same size as the maximum xfer size doesn't do
us any favors. Make these buffers a ratio of the maximum transfer size
and the number of supported nvmf SGLs.

Also configure the number of nvmf request iovs to correspond with this
new ratio.

Change-Id: I3147dcd86b599c74521ebfdf3bcdbcdee8871a3a
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/428747
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Seth Howell 2018-10-10 15:19:31 -07:00 committed by Changpeng Liu
parent 6134d778d4
commit 1d9be84bfd
2 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/nvmf/nvmf_internal.h
View File

@ -151,7 +151,7 @@ struct spdk_nvmf_request {
void *data; void *data;
union nvmf_h2c_msg *cmd; union nvmf_h2c_msg *cmd;
union nvmf_c2h_msg *rsp; union nvmf_c2h_msg *rsp;
struct iovec iov[SPDK_NVMF_MAX_SGL_ENTRIES]; struct iovec iov[SPDK_NVMF_MAX_SGL_ENTRIES * 2];
uint32_t iovcnt; uint32_t iovcnt;
struct spdk_bdev_io_wait_entry bdev_io_wait; struct spdk_bdev_io_wait_entry bdev_io_wait;

15
lib/nvmf/rdma.c
View File

@ -1499,7 +1499,8 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport,
#define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 64 #define SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR 64
#define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 #define SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE 4096
#define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072 #define SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE 131072
#define SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE 131072 #define SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE 4096
#define SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE (SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE / SPDK_NVMF_MAX_SGL_ENTRIES)
static void static void
spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts) spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts)
@ -1508,7 +1509,8 @@ spdk_nvmf_rdma_opts_init(struct spdk_nvmf_transport_opts *opts)
opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR; opts->max_qpairs_per_ctrlr = SPDK_NVMF_RDMA_DEFAULT_MAX_QPAIRS_PER_CTRLR;
opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE; opts->in_capsule_data_size = SPDK_NVMF_RDMA_DEFAULT_IN_CAPSULE_DATA_SIZE;
opts->max_io_size = SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE; opts->max_io_size = SPDK_NVMF_RDMA_DEFAULT_MAX_IO_SIZE;
opts->io_unit_size = SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE; opts->io_unit_size = spdk_max(SPDK_NVMF_RDMA_DEFAULT_IO_BUFFER_SIZE,
SPDK_NVMF_RDMA_MIN_IO_BUFFER_SIZE);
opts->max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH; opts->max_aq_depth = SPDK_NVMF_RDMA_DEFAULT_AQ_DEPTH;
} }
@ -1589,9 +1591,10 @@ spdk_nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts)
return NULL; return NULL;
} }
/* The maximum number of buffers we will need for a given request is equal to just less than double the number of SGL elements */
rtransport->data_buf_pool = spdk_mempool_create("spdk_nvmf_rdma", rtransport->data_buf_pool = spdk_mempool_create("spdk_nvmf_rdma",
opts->max_queue_depth * 4, /* The 4 is arbitrarily chosen. Needs to be configurable. */ opts->max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4,
opts->max_io_size + NVMF_DATA_BUFFER_ALIGNMENT, opts->io_unit_size + NVMF_DATA_BUFFER_ALIGNMENT,
SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
SPDK_ENV_SOCKET_ID_ANY); SPDK_ENV_SOCKET_ID_ANY);
if (!rtransport->data_buf_pool) { if (!rtransport->data_buf_pool) {
@ -1740,10 +1743,10 @@ spdk_nvmf_rdma_destroy(struct spdk_nvmf_transport *transport)
if (rtransport->data_buf_pool != NULL) { if (rtransport->data_buf_pool != NULL) {
if (spdk_mempool_count(rtransport->data_buf_pool) != if (spdk_mempool_count(rtransport->data_buf_pool) !=
(transport->opts.max_queue_depth * 4)) { (transport->opts.max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4)) {
SPDK_ERRLOG("transport buffer pool count is %zu but should be %u\n", SPDK_ERRLOG("transport buffer pool count is %zu but should be %u\n",
spdk_mempool_count(rtransport->data_buf_pool), spdk_mempool_count(rtransport->data_buf_pool),
transport->opts.max_queue_depth * 4); transport->opts.max_queue_depth * (SPDK_NVMF_MAX_SGL_ENTRIES * 2) * 4);
} }
} }