From aa212405745d40080b4cb20712e233df0b558c74 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Thu, 14 Apr 2022 14:06:42 +0200 Subject: [PATCH] nvme/pcie: increase min admin queue size to 256 Now that IO qpairs can be created asynchronously, we need to make sure that all the create IO CQ/SQ commands can be executed simultaneously. It is pretty common to create multiple IO qpairs at the same time, e.g. adding an NVMe bdev to an nvmf subsystem will create an IO qpair on each poll group. In that case, if the number of cores exceed the size of the admin queue (actually it can be even lower due to outstanding AERs), we might run out nvme_requests on the admin queue. The chosen minimum value for the admin queue size, 256, should be enough to cover most cases. Fixes #2465 Signed-off-by: Konrad Sztyber Change-Id: I55c59aef64f3fdb33f7b4824d3e9beb403602633 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12270 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Tomasz Zawadzki --- lib/nvme/nvme_pcie.c | 2 ++ lib/nvme/nvme_pcie_internal.h | 3 +++ lib/nvme/nvme_vfio_user.c | 2 ++ 3 files changed, 7 insertions(+) diff --git a/lib/nvme/nvme_pcie.c b/lib/nvme/nvme_pcie.c index 5711e7b2d..14f563532 100644 --- a/lib/nvme/nvme_pcie.c +++ b/lib/nvme/nvme_pcie.c @@ -932,6 +932,8 @@ static struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(const struct spdk_nvme_ pctrlr->devhandle = devhandle; pctrlr->ctrlr.opts = *opts; pctrlr->ctrlr.trid = *trid; + pctrlr->ctrlr.opts.admin_queue_size = spdk_max(pctrlr->ctrlr.opts.admin_queue_size, + NVME_PCIE_MIN_ADMIN_QUEUE_SIZE); rc = nvme_ctrlr_construct(&pctrlr->ctrlr); if (rc != 0) { diff --git a/lib/nvme/nvme_pcie_internal.h b/lib/nvme/nvme_pcie_internal.h index d8f4f29a4..c7aaeaf67 100644 --- a/lib/nvme/nvme_pcie_internal.h +++ b/lib/nvme/nvme_pcie_internal.h @@ -49,6 +49,9 @@ #define NVME_MAX_PRP_LIST_ENTRIES (503) +/* Minimum admin queue size */ +#define NVME_PCIE_MIN_ADMIN_QUEUE_SIZE (256) + /* PCIe transport extensions for spdk_nvme_ctrlr */ struct nvme_pcie_ctrlr { struct spdk_nvme_ctrlr ctrlr; diff --git a/lib/nvme/nvme_vfio_user.c b/lib/nvme/nvme_vfio_user.c index a5a3df407..c52b8d4ef 100644 --- a/lib/nvme/nvme_vfio_user.c +++ b/lib/nvme/nvme_vfio_user.c @@ -204,6 +204,8 @@ static struct spdk_nvme_ctrlr * pctrlr->ctrlr.opts = *opts; pctrlr->ctrlr.trid = *trid; pctrlr->ctrlr.opts.use_cmb_sqs = false; + pctrlr->ctrlr.opts.admin_queue_size = spdk_max(pctrlr->ctrlr.opts.admin_queue_size, + NVME_PCIE_MIN_ADMIN_QUEUE_SIZE); ret = nvme_ctrlr_construct(&pctrlr->ctrlr); if (ret != 0) {