From 07dc433a2f0bb3cbfc051b046890a931367fb57e Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Fri, 19 Feb 2021 14:33:58 +0100 Subject: [PATCH] external_code/nvme: queue pair initialization This patch adds NVMe submission/completion queue pair definitions. These definitions are required to keep track of outstanding NVMe requests. The admin queue pair is now instantiated with the minumum number of entries (2). Signed-off-by: Konrad Sztyber Change-Id: I2ced3ce7d210408d66cc17de1e66d86b1a1dbf79 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6671 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- test/external_code/nvme/Makefile | 5 +- test/external_code/nvme/nvme.c | 120 +++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/test/external_code/nvme/Makefile b/test/external_code/nvme/Makefile index 81ec22ce2..63ed067da 100644 --- a/test/external_code/nvme/Makefile +++ b/test/external_code/nvme/Makefile @@ -33,7 +33,8 @@ PKG_CONFIG_PATH = $(SPDK_LIB_DIR)/pkgconfig -DEPLIBS := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk) +DEPLIBS := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk spdk_util) +SYS_LIB := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs --static spdk_syslibs) shared: $(CC) $(COMMON_CFLAGS) -c -fPIC nvme.c -o nvme.o @@ -43,4 +44,4 @@ shared: static: $(CC) $(COMMON_CFLAGS) nvme.c identify.c -o identify -Wl,--whole-archive \ - $(DEPLIBS) -Wl,--no-whole-archive + $(DEPLIBS) -Wl,--no-whole-archive $(SYS_LIB) diff --git a/test/external_code/nvme/nvme.c b/test/external_code/nvme/nvme.c index 2779565c7..6946e4d14 100644 --- a/test/external_code/nvme/nvme.c +++ b/test/external_code/nvme/nvme.c @@ -37,6 +37,40 @@ #include "spdk/stdinc.h" #include "nvme.h" +struct nvme_request { + /* Command identifier and position within qpair's requests array */ + uint16_t cid; + /* NVMe command */ + struct spdk_nvme_cmd cmd; + TAILQ_ENTRY(nvme_request) tailq; +}; + +struct nvme_qpair { + /* Submission queue */ + struct spdk_nvme_cmd *cmd; + /* Completion queue */ + struct spdk_nvme_cpl *cpl; + /* Physical address of the submission queue */ + uint64_t sq_paddr; + /* Physical address of the completion queue */ + uint64_t cq_paddr; + /* Submission queue tail doorbell */ + volatile uint32_t *sq_tdbl; + /* Completion queue head doorbell */ + volatile uint32_t *cq_hdbl; + /* Submission/completion queues pointers */ + uint16_t sq_head; + uint16_t sq_tail; + uint16_t cq_head; + /* Current phase tag value */ + uint8_t phase; + /* NVMe requests queue */ + TAILQ_HEAD(, nvme_request) free_requests; + struct nvme_request *requests; + /* Size of both queues */ + uint32_t num_entries; +}; + struct nvme_ctrlr { /* Underlying PCI device */ struct spdk_pci_device *pci_device; @@ -46,6 +80,8 @@ struct nvme_ctrlr { uint32_t doorbell_stride_u32; /* Controller's memory page size */ uint32_t page_size; + /* Admin queue pair */ + struct nvme_qpair *admin_qpair; TAILQ_ENTRY(nvme_ctrlr) tailq; }; @@ -168,6 +204,80 @@ nvme_ctrlr_set_aqa(struct nvme_ctrlr *ctrlr, const union spdk_nvme_aqa_register set_pcie_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, aqa.raw), aqa->raw); } +static void +free_qpair(struct nvme_qpair *qpair) +{ + spdk_free(qpair->cmd); + spdk_free(qpair->cpl); + free(qpair->requests); + free(qpair); +} + +static struct nvme_qpair * +init_qpair(struct nvme_ctrlr *ctrlr, uint16_t id, uint16_t num_entries) +{ + struct nvme_qpair *qpair; + size_t page_align = sysconf(_SC_PAGESIZE); + size_t queue_align, queue_len; + volatile uint32_t *doorbell_base; + uint16_t i; + + qpair = calloc(1, sizeof(*qpair)); + if (!qpair) { + SPDK_ERRLOG("Failed to allocate queue pair\n"); + return NULL; + } + + qpair->phase = 1; + qpair->num_entries = num_entries; + queue_len = num_entries * sizeof(struct spdk_nvme_cmd); + queue_align = spdk_max(spdk_align32pow2(queue_len), page_align); + qpair->cmd = spdk_zmalloc(queue_len, queue_align, NULL, + SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); + if (!qpair->cmd) { + SPDK_ERRLOG("Failed to allocate submission queue buffer\n"); + free_qpair(qpair); + return NULL; + } + + queue_len = num_entries * sizeof(struct spdk_nvme_cpl); + queue_align = spdk_max(spdk_align32pow2(queue_len), page_align); + qpair->cpl = spdk_zmalloc(queue_len, queue_align, NULL, + SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); + if (!qpair->cpl) { + SPDK_ERRLOG("Failed to allocate completion queue buffer\n"); + free_qpair(qpair); + return NULL; + } + + qpair->requests = calloc(num_entries - 1, sizeof(*qpair->requests)); + if (!qpair->requests) { + SPDK_ERRLOG("Failed to allocate NVMe request descriptors\n"); + free_qpair(qpair); + return NULL; + } + + TAILQ_INIT(&qpair->free_requests); + for (i = 0; i < num_entries - 1; ++i) { + qpair->requests[i].cid = i; + TAILQ_INSERT_TAIL(&qpair->free_requests, &qpair->requests[i], tailq); + } + + qpair->sq_paddr = spdk_vtophys(qpair->cmd, NULL); + qpair->cq_paddr = spdk_vtophys(qpair->cpl, NULL); + if (qpair->sq_paddr == SPDK_VTOPHYS_ERROR || qpair->cq_paddr == SPDK_VTOPHYS_ERROR) { + SPDK_ERRLOG("Failed to translate the sq/cq virtual address\n"); + free_qpair(qpair); + return NULL; + } + + doorbell_base = (volatile uint32_t *)&ctrlr->regs->doorbell[0]; + qpair->sq_tdbl = doorbell_base + (2 * id + 0) * ctrlr->doorbell_stride_u32; + qpair->cq_hdbl = doorbell_base + (2 * id + 1) * ctrlr->doorbell_stride_u32; + + return qpair; +} + static int pcie_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) { @@ -212,6 +322,15 @@ pcie_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) ctrlr->page_size = 1 << (12 + cap.bits.mpsmin); ctrlr->doorbell_stride_u32 = 1 << cap.bits.dstrd; + /* Initialize admin queue pair with minimum number of entries (2) */ + ctrlr->admin_qpair = init_qpair(ctrlr, 0, SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES); + if (!ctrlr->admin_qpair) { + SPDK_ERRLOG("Failed to initialize admin queue pair for controller: %s\n", addr); + spdk_pci_device_unclaim(pci_dev); + free(ctrlr); + return -1; + } + TAILQ_INSERT_TAIL(ctrlrs, ctrlr, tailq); return 0; @@ -223,6 +342,7 @@ free_ctrlr(struct nvme_ctrlr *ctrlr) spdk_pci_device_unmap_bar(ctrlr->pci_device, 0, (void *)ctrlr->regs); spdk_pci_device_unclaim(ctrlr->pci_device); spdk_pci_device_detach(ctrlr->pci_device); + free_qpair(ctrlr->admin_qpair); free(ctrlr); }