From a4002614db150a728c50cbbfd33039145b76e49a Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Thu, 14 Apr 2022 14:06:42 +0200 Subject: [PATCH] nvme/pcie: increase min admin queue size to 256 Now that IO qpairs can be created asynchronously, we need to make sure that all the create IO CQ/SQ commands can be executed simultaneously. It is pretty common to create multiple IO qpairs at the same time, e.g. adding an NVMe bdev to an nvmf subsystem will create an IO qpair on each poll group. In that case, if the number of cores exceed the size of the admin queue (actually it can be even lower due to outstanding AERs), we might run out nvme_requests on the admin queue. The chosen minimum value for the admin queue size, 256, should be enough to cover most cases. Fixes #2465 Signed-off-by: Konrad Sztyber Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12270 (master) (cherry picked from commit aa212405745d40080b4cb20712e233df0b558c74) Change-Id: I55c59aef64f3fdb33f7b4824d3e9beb403602633 Signed-off-by: Krzysztof Karas Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12485 Reviewed-by: Tomasz Zawadzki Reviewed-by: Konrad Sztyber Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins --- lib/nvme/nvme_pcie.c | 2 ++ lib/nvme/nvme_pcie_internal.h | 3 +++ lib/nvme/nvme_vfio_user.c | 2 ++ 3 files changed, 7 insertions(+) diff --git a/lib/nvme/nvme_pcie.c b/lib/nvme/nvme_pcie.c index 5711e7b2d..14f563532 100644 --- a/lib/nvme/nvme_pcie.c +++ b/lib/nvme/nvme_pcie.c @@ -932,6 +932,8 @@ static struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(const struct spdk_nvme_ pctrlr->devhandle = devhandle; pctrlr->ctrlr.opts = *opts; pctrlr->ctrlr.trid = *trid; + pctrlr->ctrlr.opts.admin_queue_size = spdk_max(pctrlr->ctrlr.opts.admin_queue_size, + NVME_PCIE_MIN_ADMIN_QUEUE_SIZE); rc = nvme_ctrlr_construct(&pctrlr->ctrlr); if (rc != 0) { diff --git a/lib/nvme/nvme_pcie_internal.h b/lib/nvme/nvme_pcie_internal.h index 663cccf82..e3fb862fd 100644 --- a/lib/nvme/nvme_pcie_internal.h +++ b/lib/nvme/nvme_pcie_internal.h @@ -49,6 +49,9 @@ #define NVME_MAX_PRP_LIST_ENTRIES (503) +/* Minimum admin queue size */ +#define NVME_PCIE_MIN_ADMIN_QUEUE_SIZE (256) + /* PCIe transport extensions for spdk_nvme_ctrlr */ struct nvme_pcie_ctrlr { struct spdk_nvme_ctrlr ctrlr; diff --git a/lib/nvme/nvme_vfio_user.c b/lib/nvme/nvme_vfio_user.c index bdf52c534..721913413 100644 --- a/lib/nvme/nvme_vfio_user.c +++ b/lib/nvme/nvme_vfio_user.c @@ -239,6 +239,8 @@ static struct spdk_nvme_ctrlr * pctrlr->ctrlr.opts = *opts; pctrlr->ctrlr.trid = *trid; pctrlr->ctrlr.opts.use_cmb_sqs = false; + pctrlr->ctrlr.opts.admin_queue_size = spdk_max(pctrlr->ctrlr.opts.admin_queue_size, + NVME_PCIE_MIN_ADMIN_QUEUE_SIZE); ret = nvme_ctrlr_construct(&pctrlr->ctrlr); if (ret != 0) {