From bad452d25e4250ff8f9c170865b4de08ccc8493d Mon Sep 17 00:00:00 2001 From: Thanos Makatos Date: Mon, 19 Sep 2022 12:53:09 +0000 Subject: [PATCH] nvmf/vfio-user: calculate doorbells based on number of queue pairs It doesn't make sense to have the size of the doorbells fixed and then calculate the maximum number of queue pairs based on it, do it the other way round. Also, add some sanity checks based on the spec. Signed-off-by: Thanos Makatos Change-Id: I17e3509fb0a011128ca089ce78b7a296262e6f8e Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14932 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- include/spdk/nvme_spec.h | 2 ++ lib/nvmf/vfio_user.c | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/include/spdk/nvme_spec.h b/include/spdk/nvme_spec.h index 68bd9615d..d23a6cfc4 100644 --- a/include/spdk/nvme_spec.h +++ b/include/spdk/nvme_spec.h @@ -56,6 +56,8 @@ extern "C" { */ #define SPDK_NVME_MAX_CHANGED_NAMESPACES 1024 +#define SPDK_NVME_DOORBELL_REGISTER_SIZE 4 + union spdk_nvme_cap_register { uint64_t raw; struct { diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index af3fa3104..e87637312 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -38,25 +38,39 @@ #define NVMF_VFIO_USER_DEFAULT_IO_UNIT_SIZE NVMF_VFIO_USER_DEFAULT_MAX_IO_SIZE #define NVME_DOORBELLS_OFFSET 0x1000 -#define NVMF_VFIO_USER_DOORBELLS_SIZE 0x1000 #define NVMF_VFIO_USER_SHADOW_DOORBELLS_BUFFER_COUNT 2 #define NVMF_VFIO_USER_SET_EVENTIDX_MAX_ATTEMPTS 3 #define NVMF_VFIO_USER_EVENTIDX_POLL UINT32_MAX +#define NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR 512 +#define NVMF_VFIO_USER_DEFAULT_MAX_QPAIRS_PER_CTRLR (NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR / 4) + +/* NVMe spec 1.4, section 5.21.1.7 */ +SPDK_STATIC_ASSERT(NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR >= 2 && + NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR <= 65535, + "bad number of queues"); + /* * NVMe driver reads 4096 bytes, which is the extended PCI configuration space * available on PCI-X 2.0 and PCI Express buses */ #define NVME_REG_CFG_SIZE 0x1000 -#define NVME_REG_BAR0_SIZE (NVME_DOORBELLS_OFFSET + NVMF_VFIO_USER_DOORBELLS_SIZE) -#define NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR ((NVMF_VFIO_USER_DOORBELLS_SIZE) / 8) #define NVME_IRQ_MSIX_NUM NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR /* MSIX Table Size */ #define NVME_BAR4_SIZE SPDK_ALIGN_CEIL((NVME_IRQ_MSIX_NUM * 16), 0x1000) /* MSIX Pending Bit Array Size */ #define NVME_BAR5_SIZE SPDK_ALIGN_CEIL((NVME_IRQ_MSIX_NUM / 8), 0x1000) -#define NVMF_VFIO_USER_DEFAULT_MAX_QPAIRS_PER_CTRLR (NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR / 4) +/* + * Doorbells must be page aligned so that they can memory mapped. + * + * TODO does the NVMe spec also require this? Document it. + */ +#define NVMF_VFIO_USER_DOORBELLS_SIZE \ + SPDK_ALIGN_CEIL( \ + (NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR * 2 * SPDK_NVME_DOORBELL_REGISTER_SIZE), \ + 0x1000) +#define NVME_REG_BAR0_SIZE (NVME_DOORBELLS_OFFSET + NVMF_VFIO_USER_DOORBELLS_SIZE) struct nvmf_vfio_user_req;