From 5c80b1e5ab0503efdd4eb17204853b03c1e10c84 Mon Sep 17 00:00:00 2001 From: Evgeniy Kochetov Date: Fri, 18 Feb 2022 12:03:30 +0200 Subject: [PATCH] nvme/rdma: Limit max_sges by command capsule size According to NVMe over Fabrics spec number of SGLs supported by the controller is reported in MSDBD. But it is also implicitly limited by command capsule size (IOCCSZ) since SGL are passed in capsule. This patch adjusts max_sges to capsule size if required. Adjustment to MSDBD is also moved to transport layer because it is fabrics specific parameter and is not valid for PCIe transport. Signed-off-by: Evgeniy Kochetov Change-Id: I44918eb949345c61242ca50a524d21d04b6ac058 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11669 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Reviewed-by: Aleksey Marchuk Reviewed-by: Shuhei Matsumoto Reviewed-by: Jim Harris --- lib/nvme/nvme_ctrlr.c | 10 +------ lib/nvme/nvme_rdma.c | 15 +++++++++- test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c | 29 +++++++++++++++++++ 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index 38a1de024..04958338b 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -1986,16 +1986,8 @@ nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl) if (ctrlr->cdata.sgls.supported == 0x2) { ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT; } - /* - * Use MSDBD to ensure our max_sges doesn't exceed what the - * controller supports. - */ + ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr); - if (ctrlr->cdata.nvmf_specific.msdbd != 0) { - ctrlr->max_sges = spdk_min(ctrlr->cdata.nvmf_specific.msdbd, ctrlr->max_sges); - } else { - /* A value 0 indicates no limit. */ - } NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_sges %u\n", ctrlr->max_sges); } diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index 5e7f468bd..6fcea32c0 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -2434,8 +2434,21 @@ static uint16_t nvme_rdma_ctrlr_get_max_sges(struct spdk_nvme_ctrlr *ctrlr) { struct nvme_rdma_ctrlr *rctrlr = nvme_rdma_ctrlr(ctrlr); + uint32_t max_sge = rctrlr->max_sge; + uint32_t max_in_capsule_sge = (ctrlr->cdata.nvmf_specific.ioccsz * 16 - + sizeof(struct spdk_nvme_cmd)) / + sizeof(struct spdk_nvme_sgl_descriptor); - return rctrlr->max_sge; + /* Max SGE is limited by capsule size */ + max_sge = spdk_min(max_sge, max_in_capsule_sge); + /* Max SGE may be limited by MSDBD */ + if (ctrlr->cdata.nvmf_specific.msdbd != 0) { + max_sge = spdk_min(max_sge, ctrlr->cdata.nvmf_specific.msdbd); + } + + /* Max SGE can't be less than 1 */ + max_sge = spdk_max(1, max_sge); + return max_sge; } static int diff --git a/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c b/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c index e5179ae1b..25c671c8a 100644 --- a/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c +++ b/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c @@ -1370,6 +1370,34 @@ test_nvme_rdma_poll_group_get_qpair_by_id(void) STAILQ_REMOVE_HEAD(&group.destroyed_qpairs, link); } +static void +test_nvme_rdma_ctrlr_get_max_sges(void) +{ + struct nvme_rdma_ctrlr rctrlr = {}; + + rctrlr.ctrlr.trid.trtype = SPDK_NVME_TRANSPORT_RDMA; + rctrlr.max_sge = NVME_RDMA_MAX_SGL_DESCRIPTORS; + rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16; + rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096; + CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 16); + + rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 32; + rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096; + CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 16); + + rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 8; + rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4096; + CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 8); + + rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16; + rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 4; + CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 1); + + rctrlr.ctrlr.cdata.nvmf_specific.msdbd = 16; + rctrlr.ctrlr.cdata.nvmf_specific.ioccsz = 6; + CU_ASSERT(nvme_rdma_ctrlr_get_max_sges(&rctrlr.ctrlr) == 2); +} + int main(int argc, char **argv) { CU_pSuite suite = NULL; @@ -1402,6 +1430,7 @@ int main(int argc, char **argv) CU_ADD_TEST(suite, test_rdma_ctrlr_get_memory_domains); CU_ADD_TEST(suite, test_rdma_get_memory_translation); CU_ADD_TEST(suite, test_nvme_rdma_poll_group_get_qpair_by_id); + CU_ADD_TEST(suite, test_nvme_rdma_ctrlr_get_max_sges); CU_basic_set_mode(CU_BRM_VERBOSE); CU_basic_run_tests();