diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index 1a19126a4..0315d437c 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -1361,6 +1361,7 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, uint32_t prchk_flags) { struct nvme_bdev_ctrlr *nvme_bdev_ctrlr; + struct nvme_bdev_ctrlr_trid *trid_entry; uint32_t i; int rc; @@ -1370,22 +1371,25 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, return -ENOMEM; } - nvme_bdev_ctrlr->connected_trid = calloc(1, sizeof(*nvme_bdev_ctrlr->connected_trid)); - if (nvme_bdev_ctrlr->connected_trid == NULL) { - SPDK_ERRLOG("Failed to allocate device trid struct\n"); - free(nvme_bdev_ctrlr); - return -ENOMEM; - } - + TAILQ_INIT(&nvme_bdev_ctrlr->trids); nvme_bdev_ctrlr->num_ns = spdk_nvme_ctrlr_get_num_ns(ctrlr); nvme_bdev_ctrlr->namespaces = calloc(nvme_bdev_ctrlr->num_ns, sizeof(struct nvme_bdev_ns *)); if (!nvme_bdev_ctrlr->namespaces) { SPDK_ERRLOG("Failed to allocate block namespaces pointer\n"); - free(nvme_bdev_ctrlr->connected_trid); free(nvme_bdev_ctrlr); return -ENOMEM; } + trid_entry = calloc(1, sizeof(*trid_entry)); + if (trid_entry == NULL) { + SPDK_ERRLOG("Failed to allocate trid entry pointer\n"); + free(nvme_bdev_ctrlr->namespaces); + free(nvme_bdev_ctrlr); + return -ENOMEM; + } + + trid_entry->trid = *trid; + for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) { nvme_bdev_ctrlr->namespaces[i] = calloc(1, sizeof(struct nvme_bdev_ns)); if (nvme_bdev_ctrlr->namespaces[i] == NULL) { @@ -1393,8 +1397,8 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, for (; i > 0; i--) { free(nvme_bdev_ctrlr->namespaces[i - 1]); } + free(trid_entry); free(nvme_bdev_ctrlr->namespaces); - free(nvme_bdev_ctrlr->connected_trid); free(nvme_bdev_ctrlr); return -ENOMEM; } @@ -1404,11 +1408,11 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, nvme_bdev_ctrlr->adminq_timer_poller = NULL; nvme_bdev_ctrlr->ctrlr = ctrlr; nvme_bdev_ctrlr->ref = 0; - *nvme_bdev_ctrlr->connected_trid = *trid; + nvme_bdev_ctrlr->connected_trid = &trid_entry->trid; nvme_bdev_ctrlr->name = strdup(name); if (nvme_bdev_ctrlr->name == NULL) { + free(trid_entry); free(nvme_bdev_ctrlr->namespaces); - free(nvme_bdev_ctrlr->connected_trid); free(nvme_bdev_ctrlr); return -ENOMEM; } @@ -1417,9 +1421,9 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, rc = bdev_ocssd_init_ctrlr(nvme_bdev_ctrlr); if (spdk_unlikely(rc != 0)) { SPDK_ERRLOG("Unable to initialize OCSSD controller\n"); + free(trid_entry); free(nvme_bdev_ctrlr->name); free(nvme_bdev_ctrlr->namespaces); - free(nvme_bdev_ctrlr->connected_trid); free(nvme_bdev_ctrlr); return rc; } @@ -1450,6 +1454,8 @@ create_ctrlr(struct spdk_nvme_ctrlr *ctrlr, SPDK_ERRLOG("Failed to initialize Opal\n"); } } + + TAILQ_INSERT_HEAD(&nvme_bdev_ctrlr->trids, trid_entry, link); return 0; } @@ -1722,6 +1728,84 @@ bdev_nvme_async_poll(void *arg) return SPDK_POLLER_BUSY; } +int +bdev_nvme_add_trid(const char *name, struct spdk_nvme_transport_id *trid) +{ + struct nvme_bdev_ctrlr *nvme_bdev_ctrlr; + struct spdk_nvme_ctrlr *new_ctrlr; + struct spdk_nvme_ctrlr_opts opts; + uint32_t i; + struct spdk_nvme_ns *ns, *new_ns; + const struct spdk_nvme_ns_data *ns_data, *new_ns_data; + struct nvme_bdev_ctrlr_trid *new_trid; + int rc = 0; + + assert(name != NULL); + + nvme_bdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name); + if (nvme_bdev_ctrlr == NULL) { + SPDK_ERRLOG("Failed to find NVMe controller\n"); + return -ENODEV; + } + + /* Currently we only support failover to the same transport type. */ + if (nvme_bdev_ctrlr->connected_trid->trtype != trid->trtype) { + return -EINVAL; + } + + /* Currently we only support failover to the same NQN. */ + if (strncmp(trid->subnqn, nvme_bdev_ctrlr->connected_trid->subnqn, SPDK_NVMF_NQN_MAX_LEN)) { + return -EINVAL; + } + + /* Skip all the other checks if we've already registered this path. */ + TAILQ_FOREACH(new_trid, &nvme_bdev_ctrlr->trids, link) { + if (!spdk_nvme_transport_id_compare(&new_trid->trid, trid)) { + return -EEXIST; + } + } + + spdk_nvme_ctrlr_get_default_ctrlr_opts(&opts, sizeof(opts)); + opts.transport_retry_count = g_opts.retry_count; + + new_ctrlr = spdk_nvme_connect(trid, &opts, sizeof(opts)); + + if (new_ctrlr == NULL) { + return -ENODEV; + } + + if (spdk_nvme_ctrlr_get_num_ns(new_ctrlr) != nvme_bdev_ctrlr->num_ns) { + rc = -EINVAL; + goto out; + } + + for (i = 1; i <= nvme_bdev_ctrlr->num_ns; i++) { + ns = spdk_nvme_ctrlr_get_ns(nvme_bdev_ctrlr->ctrlr, i); + new_ns = spdk_nvme_ctrlr_get_ns(new_ctrlr, i); + assert(ns != NULL); + assert(new_ns != NULL); + + ns_data = spdk_nvme_ns_get_data(ns); + new_ns_data = spdk_nvme_ns_get_data(new_ns); + if (memcmp(ns_data->nguid, new_ns_data->nguid, sizeof(ns_data->nguid))) { + rc = -EINVAL; + goto out; + } + } + + new_trid = calloc(1, sizeof(*new_trid)); + if (new_trid == NULL) { + rc = -ENOMEM; + goto out; + } + new_trid->trid = *trid; + TAILQ_INSERT_TAIL(&nvme_bdev_ctrlr->trids, new_trid, link); + +out: + spdk_nvme_detach(new_ctrlr); + return rc; +} + int bdev_nvme_create(struct spdk_nvme_transport_id *trid, struct spdk_nvme_host_id *hostid, diff --git a/module/bdev/nvme/bdev_nvme.h b/module/bdev/nvme/bdev_nvme.h index 417c21cad..62c3f5265 100644 --- a/module/bdev/nvme/bdev_nvme.h +++ b/module/bdev/nvme/bdev_nvme.h @@ -66,6 +66,7 @@ struct spdk_nvme_qpair *bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ void bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts); int bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts); int bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx); +int bdev_nvme_add_trid(const char *name, struct spdk_nvme_transport_id *trid); int bdev_nvme_create(struct spdk_nvme_transport_id *trid, struct spdk_nvme_host_id *hostid, diff --git a/module/bdev/nvme/common.c b/module/bdev/nvme/common.c index 16686bbb1..571f2f8a7 100644 --- a/module/bdev/nvme/common.c +++ b/module/bdev/nvme/common.c @@ -116,6 +116,7 @@ static void nvme_bdev_unregister_cb(void *io_device) { struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = io_device; + struct nvme_bdev_ctrlr_trid *trid, *tmp_trid; uint32_t i; pthread_mutex_lock(&g_bdev_nvme_mutex); @@ -127,8 +128,13 @@ nvme_bdev_unregister_cb(void *io_device) for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) { free(nvme_bdev_ctrlr->namespaces[i]); } + + TAILQ_FOREACH_SAFE(trid, &nvme_bdev_ctrlr->trids, link, tmp_trid) { + TAILQ_REMOVE(&nvme_bdev_ctrlr->trids, trid, link); + free(trid); + } + free(nvme_bdev_ctrlr->namespaces); - free(nvme_bdev_ctrlr->connected_trid); free(nvme_bdev_ctrlr); pthread_mutex_lock(&g_bdev_nvme_mutex); diff --git a/module/bdev/nvme/common.h b/module/bdev/nvme/common.h index d59e7a012..e7f6567ca 100644 --- a/module/bdev/nvme/common.h +++ b/module/bdev/nvme/common.h @@ -69,38 +69,45 @@ struct nvme_bdev_ns { struct ocssd_bdev_ctrlr; +struct nvme_bdev_ctrlr_trid { + struct spdk_nvme_transport_id trid; + TAILQ_ENTRY(nvme_bdev_ctrlr_trid) link; +}; + struct nvme_bdev_ctrlr { /** * points to pinned, physically contiguous memory region; * contains 4KB IDENTIFY structure for controller which is * target for CONTROLLER IDENTIFY command during initialization */ - struct spdk_nvme_ctrlr *ctrlr; - struct spdk_nvme_transport_id *connected_trid; - char *name; - int ref; - bool resetting; - bool destruct; + struct spdk_nvme_ctrlr *ctrlr; + struct spdk_nvme_transport_id *connected_trid; + char *name; + int ref; + bool resetting; + bool destruct; /** * PI check flags. This flags is set to NVMe controllers created only * through bdev_nvme_attach_controller RPC or .INI config file. Hot added * NVMe controllers are not included. */ - uint32_t prchk_flags; - uint32_t num_ns; + uint32_t prchk_flags; + uint32_t num_ns; /** Array of pointers to namespaces indexed by nsid - 1 */ - struct nvme_bdev_ns **namespaces; + struct nvme_bdev_ns **namespaces; - struct spdk_opal_dev *opal_dev; + struct spdk_opal_dev *opal_dev; - struct spdk_poller *adminq_timer_poller; - struct spdk_poller *destruct_poller; - struct spdk_thread *thread; + struct spdk_poller *adminq_timer_poller; + struct spdk_poller *destruct_poller; + struct spdk_thread *thread; - struct ocssd_bdev_ctrlr *ocssd_ctrlr; + struct ocssd_bdev_ctrlr *ocssd_ctrlr; /** linked list pointer for device list */ - TAILQ_ENTRY(nvme_bdev_ctrlr) tailq; + TAILQ_ENTRY(nvme_bdev_ctrlr) tailq; + + TAILQ_HEAD(, nvme_bdev_ctrlr_trid) trids; }; struct nvme_bdev { diff --git a/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c b/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c index 4cfd0c8b7..59a99aa08 100644 --- a/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c +++ b/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c @@ -198,6 +198,7 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha { struct spdk_nvme_ctrlr *ctrlr; struct nvme_bdev_ctrlr *nvme_bdev_ctrlr; + struct nvme_bdev_ctrlr_trid *trid_entry; uint32_t nsid; ctrlr = find_controller(trid); @@ -211,15 +212,15 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha nvme_bdev_ctrlr->namespaces = calloc(ctrlr->ns_count, sizeof(struct nvme_bdev_ns *)); SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->namespaces != NULL); - nvme_bdev_ctrlr->connected_trid = calloc(1, sizeof(struct spdk_nvme_transport_id)); - SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->connected_trid != NULL); + trid_entry = calloc(1, sizeof(struct nvme_bdev_ctrlr_trid)); + SPDK_CU_ASSERT_FATAL(trid_entry != NULL); + trid_entry->trid = *trid; nvme_bdev_ctrlr->ctrlr = ctrlr; nvme_bdev_ctrlr->num_ns = ctrlr->ns_count; nvme_bdev_ctrlr->ref = 0; - *nvme_bdev_ctrlr->connected_trid = *trid; + nvme_bdev_ctrlr->connected_trid = &trid_entry->trid; nvme_bdev_ctrlr->name = strdup(name); - for (nsid = 0; nsid < ctrlr->ns_count; ++nsid) { nvme_bdev_ctrlr->namespaces[nsid] = calloc(1, sizeof(struct nvme_bdev_ns)); SPDK_CU_ASSERT_FATAL(nvme_bdev_ctrlr->namespaces[nsid] != NULL); @@ -239,6 +240,9 @@ create_nvme_bdev_controller(const struct spdk_nvme_transport_id *trid, const cha TAILQ_INSERT_TAIL(&g_nvme_bdev_ctrlrs, nvme_bdev_ctrlr, tailq); + TAILQ_INIT(&nvme_bdev_ctrlr->trids); + TAILQ_INSERT_HEAD(&nvme_bdev_ctrlr->trids, trid_entry, link); + return nvme_bdev_ctrlr; }