diff --git a/test/external_code/nvme/Makefile b/test/external_code/nvme/Makefile index bf1763cc1..81ec22ce2 100644 --- a/test/external_code/nvme/Makefile +++ b/test/external_code/nvme/Makefile @@ -33,7 +33,7 @@ PKG_CONFIG_PATH = $(SPDK_LIB_DIR)/pkgconfig -DEPLIBS := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_log) +DEPLIBS := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk) shared: $(CC) $(COMMON_CFLAGS) -c -fPIC nvme.c -o nvme.o diff --git a/test/external_code/nvme/identify.c b/test/external_code/nvme/identify.c index 0d9272cc2..5c36115fb 100644 --- a/test/external_code/nvme/identify.c +++ b/test/external_code/nvme/identify.c @@ -34,14 +34,37 @@ #include "spdk/stdinc.h" #include "nvme.h" +static void +attach_cb(void *cb_ctx, const struct spdk_pci_addr *addr, + struct nvme_ctrlr *ctrlr) +{ + char fmtaddr[32] = {}; + + (void)cb_ctx; + + spdk_pci_addr_fmt(fmtaddr, sizeof(fmtaddr), addr); + printf("Found NVMe controller at: %s\n", fmtaddr); + + nvme_detach(ctrlr); +} + int main(int argc, const char **argv) { + struct spdk_env_opts opts; int rc; (void)argc; - rc = nvme_probe(NULL, NULL); + spdk_env_opts_init(&opts); + opts.name = "identify"; + + if (spdk_env_init(&opts) != 0) { + fprintf(stderr, "%s: unable to initialize SPDK env\n", argv[0]); + return 1; + } + + rc = nvme_probe(attach_cb, NULL); if (rc != 0) { fprintf(stderr, "%s: nvme probe failed\n", argv[0]); return 1; diff --git a/test/external_code/nvme/nvme.c b/test/external_code/nvme/nvme.c index 15ec88566..7537d3282 100644 --- a/test/external_code/nvme/nvme.c +++ b/test/external_code/nvme/nvme.c @@ -35,19 +35,96 @@ #include "spdk/stdinc.h" #include "nvme.h" +struct nvme_ctrlr { + /* Underlying PCI device */ + struct spdk_pci_device *pci_device; + TAILQ_ENTRY(nvme_ctrlr) tailq; +}; + +static struct spdk_pci_id nvme_pci_driver_id[] = { + { + .class_id = SPDK_PCI_CLASS_NVME, + .vendor_id = SPDK_PCI_ANY_ID, + .device_id = SPDK_PCI_ANY_ID, + .subvendor_id = SPDK_PCI_ANY_ID, + .subdevice_id = SPDK_PCI_ANY_ID, + }, + { .vendor_id = 0, /* sentinel */ }, +}; + +SPDK_PCI_DRIVER_REGISTER(nvme_external, nvme_pci_driver_id, SPDK_PCI_DRIVER_NEED_MAPPING); +static TAILQ_HEAD(, nvme_ctrlr) g_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_ctrlrs); + +static int +pcie_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) +{ + struct nvme_ctrlr *ctrlr; + TAILQ_HEAD(, nvme_ctrlr) *ctrlrs = ctx; + char addr[32] = {}; + + spdk_pci_addr_fmt(addr, sizeof(addr), &pci_dev->addr); + + ctrlr = calloc(1, sizeof(*ctrlr)); + if (!ctrlr) { + SPDK_ERRLOG("Failed to allocate NVMe controller: %s\n", addr); + return -1; + } + + if (spdk_pci_device_claim(pci_dev)) { + SPDK_ERRLOG("Failed to claim PCI device: %s\n", addr); + free(ctrlr); + return -1; + } + + ctrlr->pci_device = pci_dev; + TAILQ_INSERT_TAIL(ctrlrs, ctrlr, tailq); + + return 0; +} + +static void +free_ctrlr(struct nvme_ctrlr *ctrlr) +{ + spdk_pci_device_unclaim(ctrlr->pci_device); + spdk_pci_device_detach(ctrlr->pci_device); + free(ctrlr); +} + int nvme_probe(nvme_attach_cb attach_cb, void *cb_ctx) { - (void)attach_cb; - (void)cb_ctx; + struct nvme_ctrlr *ctrlr, *tmp; + TAILQ_HEAD(, nvme_ctrlr) ctrlrs = TAILQ_HEAD_INITIALIZER(ctrlrs); + int rc; - return -ENOTSUP; + rc = spdk_pci_enumerate(spdk_pci_get_driver("nvme_external"), + pcie_enum_cb, &ctrlrs); + if (rc != 0) { + SPDK_ERRLOG("Failed to enumerate PCI devices\n"); + while (!TAILQ_EMPTY(&ctrlrs)) { + ctrlr = TAILQ_FIRST(&ctrlrs); + TAILQ_REMOVE(&ctrlrs, ctrlr, tailq); + free_ctrlr(ctrlr); + } + + return rc; + } + + TAILQ_FOREACH_SAFE(ctrlr, &ctrlrs, tailq, tmp) { + TAILQ_REMOVE(&ctrlrs, ctrlr, tailq); + TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, ctrlr, tailq); + + attach_cb(cb_ctx, &ctrlr->pci_device->addr, ctrlr); + } + + return 0; } void nvme_detach(struct nvme_ctrlr *ctrlr) { - (void) ctrlr; + TAILQ_REMOVE(&g_nvme_ctrlrs, ctrlr, tailq); + free_ctrlr(ctrlr); } SPDK_LOG_REGISTER_COMPONENT(nvme_external) diff --git a/test/external_code/nvme/nvme.h b/test/external_code/nvme/nvme.h index e3bfff7dc..e76de6b6b 100644 --- a/test/external_code/nvme/nvme.h +++ b/test/external_code/nvme/nvme.h @@ -42,8 +42,8 @@ struct nvme_ctrlr; * Callback for nvme_probe() to report a device that has been attached to * the userspace NVMe driver. * - * \param cb_ctx Opaque value passed to spdk_nvme_attach_cb(). - * \param addr NVMe PCI address. + * \param cb_ctx Opaque value passed to nvme_attach_cb(). + * \param addr The PCI address of the NVMe controller. * \param ctrlr Opaque handle to NVMe controller. */ typedef void (*nvme_attach_cb)(void *cb_ctx, const struct spdk_pci_addr *addr, @@ -52,6 +52,9 @@ typedef void (*nvme_attach_cb)(void *cb_ctx, const struct spdk_pci_addr *addr, /** * Enumerate PCIe bus and attach all NVMe devices found to the driver. * + * This function is not thread safe and should only be called from one thread at + * a time while no other threads are actively using any NVMe devices. + * * \param cb_ctx Opaque value which will be passed back in cb_ctx parameter of * the callbacks. * \param attach_cb will be called for each NVMe device found @@ -64,6 +67,9 @@ int nvme_probe(nvme_attach_cb attach_cb, void *ctx); * Detach specified device returned by nvme_probe()'s attach_cb. After returning * the nvme_ctrlr handle is no longer valid. * + * This function should be called from a single thread while no other threads + * are actively using the NVMe device. + * * \param ctrlr Opaque handle to NVMe controller. */ void nvme_detach(struct nvme_ctrlr *ctrlr);