From 7d5bcb48931fdab3dd15d2b91a935158f6c3d5ac Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Mon, 21 Nov 2016 16:32:23 -0700 Subject: [PATCH] nvme: remove spdk_nvme_transport_type from API Use the NVMe over Fabrics spec definitions for TRTYPE rather than the internal library transport type. Change-Id: Idead559a8f8d95274fc580d10e82033822e6eda8 Signed-off-by: Daniel Verkamp --- examples/nvme/identify/identify.c | 4 +-- include/spdk/nvme.h | 33 ++++++++++--------- lib/nvme/nvme.c | 20 +++++++---- lib/nvme/nvme_internal.h | 26 +++++++++++---- lib/nvme/nvme_pcie.c | 4 +-- lib/nvme/nvme_rdma.c | 20 +++++++---- lib/nvme/nvme_transport.c | 24 ++++++++++++-- test/lib/nvme/unit/nvme_c/nvme_ut.c | 10 ++++-- .../nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c | 2 +- .../nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c | 10 ++++-- 10 files changed, 106 insertions(+), 47 deletions(-) diff --git a/examples/nvme/identify/identify.c b/examples/nvme/identify/identify.c index 370fb4433..fb1c22887 100644 --- a/examples/nvme/identify/identify.c +++ b/examples/nvme/identify/identify.c @@ -925,7 +925,7 @@ parse_args(int argc, char **argv) return 0; } - info.type = SPDK_NVME_TRANSPORT_RDMA; + info.trtype = SPDK_NVMF_TRTYPE_RDMA; optind = 1; return 0; @@ -972,7 +972,7 @@ int main(int argc, char **argv) } rc = 0; - if (info.type == SPDK_NVME_TRANSPORT_RDMA) { + if (info.trtype == SPDK_NVMF_TRTYPE_RDMA) { if (spdk_nvme_discover(&info, NULL, probe_cb, attach_cb, NULL) != 0) { fprintf(stderr, "spdk_nvme_probe() failed\n"); } diff --git a/include/spdk/nvme.h b/include/spdk/nvme.h index 192cddaf1..88cb0ca82 100644 --- a/include/spdk/nvme.h +++ b/include/spdk/nvme.h @@ -102,21 +102,6 @@ struct spdk_nvme_ctrlr_opts { int queue_size; }; -/** - * NVMe transport type - */ -enum spdk_nvme_transport_type { - /** - * NVMe connected via local PCI Express - */ - SPDK_NVME_TRANSPORT_PCIE, - - /** - * NVMe over Fabrics with RDMA transport - */ - SPDK_NVME_TRANSPORT_RDMA, -}; - /** * NVMe over Fabrics discovery parameters. * @@ -126,7 +111,7 @@ struct spdk_nvme_discover_info { /** * NVMe over Fabrics transport type. */ - enum spdk_nvme_transport_type type; + enum spdk_nvmf_trtype trtype; /** * Subsystem NQN of the NVMe over Fabrics discovery service. @@ -171,6 +156,13 @@ struct spdk_nvme_probe_info { */ char nqn[SPDK_NVMF_NQN_MAX_LEN + 1]; + /** + * NVMe over Fabrics transport type. + * + * This field will be 0 if this is not an NVMe over Fabrics controller. + */ + enum spdk_nvmf_trtype trtype; + /** * Transport address of the NVMe over Fabrics target. For transports which use IP * addressing (e.g. RDMA), this will be an IP-based address. @@ -184,6 +176,15 @@ struct spdk_nvme_probe_info { char trsvcid[SPDK_NVMF_TRSVCID_MAX_LEN + 1]; }; +/** + * Determine whether the NVMe library can handle a specific NVMe over Fabrics transport type. + * + * \param trtype NVMe over Fabrics transport type to check. + * + * \return true if trtype is supported or false if it is not supported. + */ +bool spdk_nvme_transport_available(enum spdk_nvmf_trtype trtype); + /** * Callback for spdk_nvme_probe() enumeration. * diff --git a/lib/nvme/nvme.c b/lib/nvme/nvme.c index 7f159290c..767267065 100644 --- a/lib/nvme/nvme.c +++ b/lib/nvme/nvme.c @@ -47,7 +47,7 @@ struct nvme_driver *g_spdk_nvme_driver = &_g_nvme_driver; int32_t spdk_nvme_retry_count; struct spdk_nvme_ctrlr * -nvme_attach(enum spdk_nvme_transport_type transport, void *devhandle) +nvme_attach(enum spdk_nvme_transport transport, void *devhandle) { struct spdk_nvme_ctrlr *ctrlr; @@ -229,7 +229,7 @@ nvme_mutex_init_shared(pthread_mutex_t *mtx) } int -nvme_probe_one(enum spdk_nvme_transport_type type, spdk_nvme_probe_cb probe_cb, void *cb_ctx, +nvme_probe_one(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, struct spdk_nvme_probe_info *probe_info, void *devhandle) { struct spdk_nvme_ctrlr *ctrlr; @@ -238,7 +238,7 @@ nvme_probe_one(enum spdk_nvme_transport_type type, spdk_nvme_probe_cb probe_cb, spdk_nvme_ctrlr_opts_set_defaults(&opts); if (probe_cb(cb_ctx, probe_info, &opts)) { - ctrlr = nvme_attach(type, devhandle); + ctrlr = nvme_attach(transport, devhandle); if (ctrlr == NULL) { SPDK_ERRLOG("nvme_attach() failed\n"); return -1; @@ -260,7 +260,7 @@ _spdk_nvme_probe(const struct spdk_nvme_discover_info *info, void *cb_ctx, { int rc, start_rc; struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp; - enum spdk_nvme_transport_type type; + enum spdk_nvme_transport transport; if (!spdk_process_is_primary()) { while (g_spdk_nvme_driver->initialized == false) { @@ -281,12 +281,18 @@ _spdk_nvme_probe(const struct spdk_nvme_discover_info *info, void *cb_ctx, } if (!info) { - type = SPDK_NVME_TRANSPORT_PCIE; + transport = SPDK_NVME_TRANSPORT_PCIE; } else { - type = info->type; + if (!spdk_nvme_transport_available(info->trtype)) { + SPDK_ERRLOG("NVMe over Fabrics trtype %u not available\n", info->trtype); + pthread_mutex_unlock(&g_spdk_nvme_driver->lock); + return -1; + } + + transport = (uint8_t)info->trtype; } - rc = nvme_transport_ctrlr_scan(type, probe_cb, cb_ctx, (void *)info); + rc = nvme_transport_ctrlr_scan(transport, probe_cb, cb_ctx, (void *)info); /* * Keep going even if one or more nvme_attach() calls failed, diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index 4da63f015..0244705f5 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -220,6 +220,20 @@ struct nvme_request { void *user_buffer; }; +/* + * NVMe library transports + * + * NOTE: These are mapped directly to the NVMe over Fabrics TRTYPE values, except for PCIe, + * which is a special case since NVMe over Fabrics does not define a TRTYPE for local PCIe. + * + * Currently, this uses 0 for PCIe since it is reserved by NVMe-oF. If 0 is ever assigned as a + * valid TRTYPE, this would need to be changed. + */ +enum spdk_nvme_transport { + SPDK_NVME_TRANSPORT_PCIE = 0, + SPDK_NVME_TRANSPORT_RDMA = SPDK_NVMF_TRTYPE_RDMA, +}; + struct nvme_completion_poll_status { struct spdk_nvme_cpl cpl; bool done; @@ -234,7 +248,7 @@ struct nvme_async_event_request { struct spdk_nvme_qpair { STAILQ_HEAD(, nvme_request) queued_req; - enum spdk_nvme_transport_type transport; + enum spdk_nvme_transport transport; uint16_t id; @@ -323,7 +337,7 @@ struct spdk_nvme_ctrlr { /** Array of namespaces indexed by nsid - 1 */ struct spdk_nvme_ns *ns; - enum spdk_nvme_transport_type transport; + enum spdk_nvme_transport transport; uint32_t num_ns; @@ -480,7 +494,7 @@ void nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl); int nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle); void nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr); -int nvme_probe_one(enum spdk_nvme_transport_type type, spdk_nvme_probe_cb probe_cb, void *cb_ctx, +int nvme_probe_one(enum spdk_nvme_transport type, spdk_nvme_probe_cb probe_cb, void *cb_ctx, struct spdk_nvme_probe_info *probe_info, void *devhandle); int nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr); @@ -525,13 +539,13 @@ int nvme_mutex_init_recursive_shared(pthread_mutex_t *mtx); bool nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl); void nvme_qpair_print_command(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd); void nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cpl *cpl); -struct spdk_nvme_ctrlr *nvme_attach(enum spdk_nvme_transport_type transport, void *devhandle); +struct spdk_nvme_ctrlr *nvme_attach(enum spdk_nvme_transport transport, void *devhandle); /* Transport specific functions */ #define DECLARE_TRANSPORT(name) \ - struct spdk_nvme_ctrlr *nvme_ ## name ## _ctrlr_construct(enum spdk_nvme_transport_type transport, void *devhandle); \ + struct spdk_nvme_ctrlr *nvme_ ## name ## _ctrlr_construct(enum spdk_nvme_transport transport, void *devhandle); \ int nvme_ ## name ## _ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr); \ - int nvme_ ## name ## _ctrlr_scan(enum spdk_nvme_transport_type transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, void *devhandle); \ + int nvme_ ## name ## _ctrlr_scan(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, void *devhandle); \ int nvme_ ## name ## _ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr); \ int nvme_ ## name ## _ctrlr_get_pci_id(struct spdk_nvme_ctrlr *ctrlr, struct spdk_pci_id *pci_id); \ int nvme_ ## name ## _ctrlr_set_reg_4(struct spdk_nvme_ctrlr *ctrlr, uint32_t offset, uint32_t value); \ diff --git a/lib/nvme/nvme_pcie.c b/lib/nvme/nvme_pcie.c index 4946e7822..916623790 100644 --- a/lib/nvme/nvme_pcie.c +++ b/lib/nvme/nvme_pcie.c @@ -457,7 +457,7 @@ pcie_nvme_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) } int -nvme_pcie_ctrlr_scan(enum spdk_nvme_transport_type transport, +nvme_pcie_ctrlr_scan(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, void *devhandle) { struct nvme_pcie_enum_ctx enum_ctx; @@ -468,7 +468,7 @@ nvme_pcie_ctrlr_scan(enum spdk_nvme_transport_type transport, return spdk_pci_enumerate(SPDK_PCI_DEVICE_NVME, pcie_nvme_enum_cb, &enum_ctx); } -struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(enum spdk_nvme_transport_type transport, +struct spdk_nvme_ctrlr *nvme_pcie_ctrlr_construct(enum spdk_nvme_transport transport, void *devhandle) { struct spdk_pci_device *pci_dev = devhandle; diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index c8c0e83ac..96c977432 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -1136,7 +1136,7 @@ nvme_fabrics_get_log_discovery_page(struct spdk_nvme_ctrlr *ctrlr, /* This function must only be called while holding g_spdk_nvme_driver->lock */ int -nvme_rdma_ctrlr_scan(enum spdk_nvme_transport_type transport, +nvme_rdma_ctrlr_scan(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, void *devhandle) { struct spdk_nvme_discover_info *discover_info = devhandle; @@ -1147,12 +1147,13 @@ nvme_rdma_ctrlr_scan(enum spdk_nvme_transport_type transport, int rc; uint32_t i; + probe_info.trtype = (uint8_t)transport; snprintf(probe_info.nqn, sizeof(probe_info.nqn), "%s", discover_info->nqn); snprintf(probe_info.traddr, sizeof(probe_info.traddr), "%s", discover_info->traddr); snprintf(probe_info.trsvcid, sizeof(probe_info.trsvcid), "%s", discover_info->trsvcid); memset(buffer, 0x0, 4096); - discovery_ctrlr = nvme_attach(discover_info->type, &probe_info); + discovery_ctrlr = nvme_attach(transport, &probe_info); if (discovery_ctrlr == NULL) { return -1; } @@ -1170,6 +1171,13 @@ nvme_rdma_ctrlr_scan(enum spdk_nvme_transport_type transport, uint8_t *end; size_t len; + probe_info.trtype = entry->trtype; + if (!spdk_nvme_transport_available(probe_info.trtype)) { + SPDK_WARNLOG("NVMe transport type %u not available; skipping probe\n", + probe_info.trtype); + continue; + } + /* Ensure that subnqn is null terminated. */ end = memchr(entry->subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN); if (!end) { @@ -1188,17 +1196,17 @@ nvme_rdma_ctrlr_scan(enum spdk_nvme_transport_type transport, len = spdk_strlen_pad(entry->trsvcid, sizeof(entry->trsvcid), ' '); memcpy(probe_info.trsvcid, entry->trsvcid, len); - SPDK_NOTICELOG("nqn=%s, traddr=%s, trsvcid=%s\n", probe_info.nqn, - probe_info.traddr, probe_info.trsvcid); + SPDK_NOTICELOG("nqn=%s, trtype=%u, traddr=%s, trsvcid=%s\n", probe_info.nqn, + probe_info.trtype, probe_info.traddr, probe_info.trsvcid); /* Todo: need to differentiate the NVMe over fabrics to avoid duplicated connection */ - nvme_probe_one(discover_info->type, probe_cb, cb_ctx, &probe_info, &probe_info); + nvme_probe_one(entry->trtype, probe_cb, cb_ctx, &probe_info, &probe_info); } nvme_ctrlr_destruct(discovery_ctrlr); return 0; } -struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(enum spdk_nvme_transport_type transport, +struct spdk_nvme_ctrlr *nvme_rdma_ctrlr_construct(enum spdk_nvme_transport transport, void *devhandle) { struct nvme_rdma_ctrlr *rctrlr; diff --git a/lib/nvme/nvme_transport.c b/lib/nvme/nvme_transport.c index bec85dce7..4574375d8 100644 --- a/lib/nvme/nvme_transport.c +++ b/lib/nvme/nvme_transport.c @@ -39,7 +39,7 @@ #ifdef DEBUG static __attribute__((noreturn)) void -nvme_transport_unknown(enum spdk_nvme_transport_type transport) +nvme_transport_unknown(enum spdk_nvme_transport transport) { SPDK_ERRLOG("Unknown transport %d\n", (int)transport); abort(); @@ -52,8 +52,10 @@ nvme_transport_unknown(enum spdk_nvme_transport_type transport) #define TRANSPORT_PCIE(func_name, args) case SPDK_NVME_TRANSPORT_PCIE: return nvme_pcie_ ## func_name args; #ifdef SPDK_CONFIG_RDMA #define TRANSPORT_FABRICS_RDMA(func_name, args) case SPDK_NVME_TRANSPORT_RDMA: return nvme_rdma_ ## func_name args; +#define TRANSPORT_RDMA_AVAILABLE true #else #define TRANSPORT_FABRICS_RDMA(func_name, args) case SPDK_NVME_TRANSPORT_RDMA: SPDK_UNREACHABLE(); +#define TRANSPORT_RDMA_AVAILABLE false #endif #define NVME_TRANSPORT_CALL(transport, func_name, args) \ do { \ @@ -65,15 +67,31 @@ nvme_transport_unknown(enum spdk_nvme_transport_type transport) SPDK_UNREACHABLE(); \ } while (0) +bool +spdk_nvme_transport_available(enum spdk_nvmf_trtype trtype) +{ + switch (trtype) { + case SPDK_NVMF_TRTYPE_RDMA: + return TRANSPORT_RDMA_AVAILABLE; -struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport_type transport, + case SPDK_NVMF_TRTYPE_FC: + return false; + + case SPDK_NVMF_TRTYPE_INTRA_HOST: + return false; + } + + return false; +} + +struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport transport, void *devhandle) { NVME_TRANSPORT_CALL(transport, ctrlr_construct, (transport, devhandle)); } int -nvme_transport_ctrlr_scan(enum spdk_nvme_transport_type transport, +nvme_transport_ctrlr_scan(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, void *devhandle) { NVME_TRANSPORT_CALL(transport, ctrlr_scan, (transport, probe_cb, cb_ctx, devhandle)); diff --git a/test/lib/nvme/unit/nvme_c/nvme_ut.c b/test/lib/nvme/unit/nvme_c/nvme_ut.c index d7ffa3c3f..ed1654733 100644 --- a/test/lib/nvme/unit/nvme_c/nvme_ut.c +++ b/test/lib/nvme/unit/nvme_c/nvme_ut.c @@ -57,14 +57,20 @@ spdk_pci_device_get_id(struct spdk_pci_device *pci_dev) return pci_id; } -struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport_type transport, +bool +spdk_nvme_transport_available(enum spdk_nvmf_trtype trtype) +{ + return true; +} + +struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport transport, void *devhandle) { return NULL; } int -nvme_transport_ctrlr_scan(enum spdk_nvme_transport_type transport, +nvme_transport_ctrlr_scan(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, void *devhandle) { return 0; diff --git a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c index 152ae018b..763dd5b5f 100644 --- a/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c +++ b/test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut.c @@ -57,7 +57,7 @@ struct spdk_nvme_registers g_ut_nvme_regs = {}; __thread int nvme_thread_ioq_index = -1; -struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport_type transport, +struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport transport, void *devhandle) { return NULL; diff --git a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c index 23e06137c..e505e8945 100644 --- a/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c +++ b/test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut.c @@ -56,7 +56,13 @@ static int nvme_request_next_sge(void *cb_arg, void **address, uint32_t *length) return 0; } -struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport_type transport, +bool +spdk_nvme_transport_available(enum spdk_nvmf_trtype trtype) +{ + return true; +} + +struct spdk_nvme_ctrlr *nvme_transport_ctrlr_construct(enum spdk_nvme_transport transport, void *devhandle) { return NULL; @@ -155,7 +161,7 @@ nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr) } int -nvme_transport_ctrlr_scan(enum spdk_nvme_transport_type transport, +nvme_transport_ctrlr_scan(enum spdk_nvme_transport transport, spdk_nvme_probe_cb probe_cb, void *cb_ctx, void *devhandle) { return 0;