ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme: Set appropriate value to max_xfer_size and max_sge

Browse Source 
SPDK NVMe-oF initiator driver could not transfer IO whose size is
more than 128KiB even if NVMe-oF target allows IO whose size is
more than 128KiB both for RDMA and TCP transport.

Some use cases need to transfer IO larger than 128KiB.

For RDMA transport, max_mr_size by ibv_query_device of RDMA devices
indicates the maximum size of a single memory region and is independent
from the actual I/O size, and is very likely to be larger than 2 MiB
which is the granularity we currently register memory regions.

Actually some RDMA NICs return UINT64_MAX for max_mr_size by ibv_query_device.

Hence use UINT32_MAX and let the generic layer use the controller data
to moderate this value.

On the other hand, for TCP transport, there is no limit for maximum IO
size and hence use UINT32_MAX.

Besides, for RDMA transport, max_sges should be the minimum of
max_sge got by querying RDMA devices and NVME_RDMA_MAX_SGL_DESCRIPTORS.
Hence do this change together in this patch.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: Idc813afd3e525bf5f370c0fcd2623f9c146a5528
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/459218
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Seth Howell <seth.howell5141@gmail.com>
Reviewed-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
This commit is contained in:
Shuhei Matsumoto 2019-06-27 15:15:56 +09:00 committed by Changpeng Liu
parent cf3c54bc03
commit 8b539eb553
2 changed files with 42 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
lib/nvme/nvme_rdma.c
View File

@ -86,6 +86,8 @@ struct nvme_rdma_ctrlr {
struct spdk_nvme_ctrlr ctrlr;
struct ibv_pd *pd;
uint16_t max_sge;
};
/* NVMe RDMA qpair extensions for spdk_nvme_qpair */
@ -1461,7 +1463,9 @@ struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_transpo
struct nvme_rdma_ctrlr *rctrlr;
union spdk_nvme_cap_register cap;
union spdk_nvme_vs_register vs;
int rc;
struct ibv_context **contexts;
struct ibv_device_attr dev_attr;
int i, rc;
rctrlr = calloc(1, sizeof(struct nvme_rdma_ctrlr));
if (rctrlr == NULL) {
@ -1473,6 +1477,30 @@ struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(const struct spdk_nvme_transpo
rctrlr->ctrlr.opts = *opts;
memcpy(&rctrlr->ctrlr.trid, trid, sizeof(rctrlr->ctrlr.trid));
contexts = rdma_get_devices(NULL);
if (contexts == NULL) {
SPDK_ERRLOG("rdma_get_devices() failed: %s (%d)\n", spdk_strerror(errno), errno);
free(rctrlr);
return NULL;
}
i = 0;
rctrlr->max_sge = NVME_RDMA_MAX_SGL_DESCRIPTORS;
while (contexts[i] != NULL) {
rc = ibv_query_device(contexts[i], &dev_attr);
if (rc < 0) {
SPDK_ERRLOG("Failed to query RDMA device attributes.\n");
rdma_free_devices(contexts);
free(rctrlr);
return NULL;
}
rctrlr->max_sge = spdk_min(rctrlr->max_sge, (uint16_t)dev_attr.max_sge);
i++;
}
rdma_free_devices(contexts);
rc = nvme_ctrlr_construct(&rctrlr->ctrlr);
if (rc != 0) {
free(rctrlr);
@ -1763,14 +1791,22 @@ nvme_rdma_qpair_process_completions(struct spdk_nvme_qpair *qpair,
uint32_t
nvme_rdma_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
{
/* Todo, which should get from the NVMF target */
return NVME_RDMA_RW_BUFFER_SIZE;
/* max_mr_size by ibv_query_device indicates the largest value that we can
* set for a registered memory region. It is independent from the actual
* I/O size and is very likely to be larger than 2 MiB which is the
* granularity we currently register memory regions. Hence return
* UINT32_MAX here and let the generic layer use the controller data to
* moderate this value.
*/
return UINT32_MAX;
}
uint16_t
nvme_rdma_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr)
{
return NVME_RDMA_MAX_SGL_DESCRIPTORS;
struct nvme_rdma_ctrlr *rctrlr = nvme_rdma_ctrlr(ctrlr);
return rctrlr->max_sge;
}
volatile struct spdk_nvme_registers *

3
lib/nvme/nvme_tcp.c
View File

@ -1836,7 +1836,8 @@ struct spdk_nvme_ctrlr *nvme_tcp_ctrlr_construct(const struct spdk_nvme_transpor
uint32_t
nvme_tcp_ctrlr_get_max_xfer_size(struct spdk_nvme_ctrlr *ctrlr)
{
return NVME_TCP_RW_BUFFER_SIZE;
/* TCP transport doens't limit maximum IO transfer size. */
return UINT32_MAX;
}
uint16_t