From c30609c9ea18a3c938326e8e4b508c7e4207f532 Mon Sep 17 00:00:00 2001 From: Maciej Szwed Date: Thu, 17 Oct 2019 13:33:25 +0200 Subject: [PATCH] bdev/nvme: Allow multiple bdevs per namespace Some future NVMe namespace types will be able to create multiple bdevs per one namespace. This patch makes it possible. Signed-off-by: Maciej Szwed Change-Id: I47b4c1fc545c59dcc3171ab0960f1835b6aa6d24 Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/471620 Community-CI: Broadcom SPDK FC-NVMe CI Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto --- module/bdev/nvme/bdev_nvme.c | 38 +++++++++++++++++++++++++---------- module/bdev/nvme/common.h | 5 +++-- module/bdev/nvme/vbdev_opal.c | 6 +++++- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index 786fb25c4..6ea630ebe 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -258,6 +258,7 @@ bdev_nvme_destruct(void *ctx) pthread_mutex_lock(&g_bdev_nvme_mutex); nvme_bdev_ctrlr->ref--; + TAILQ_REMOVE(&nvme_disk->nvme_ns->bdevs, nvme_disk, tailq); free(nvme_disk->disk.name); free(nvme_disk); if (nvme_bdev_ctrlr->ref == 0 && nvme_bdev_ctrlr->destruct) { @@ -801,7 +802,7 @@ nvme_ctrlr_create_bdev(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct nvme_bdev return rc; } - nvme_ns->bdev = bdev; + TAILQ_INSERT_TAIL(&nvme_ns->bdevs, bdev, tailq); return 0; } @@ -937,7 +938,12 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, static void nvme_ctrlr_deactivate_namespace(struct nvme_bdev_ns *ns) { - spdk_bdev_unregister(&ns->bdev->disk, NULL, NULL); + struct nvme_bdev *bdev, *tmp; + + TAILQ_FOREACH_SAFE(bdev, &ns->bdevs, tailq, tmp) { + spdk_bdev_unregister(&bdev->disk, NULL, NULL); + } + ns->active = false; } @@ -958,6 +964,8 @@ nvme_ctrlr_update_ns_bdevs(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr) ns->id = nsid; ns->ctrlr = nvme_bdev_ctrlr; + TAILQ_INIT(&ns->bdevs); + rc = nvme_ctrlr_create_bdev(nvme_bdev_ctrlr, ns); if (rc) { memset(ns, 0, sizeof(*ns)); @@ -1114,6 +1122,7 @@ remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) uint32_t i; struct nvme_bdev_ctrlr *nvme_bdev_ctrlr; struct nvme_bdev_ns *ns; + struct nvme_bdev *nvme_bdev, *tmp; pthread_mutex_lock(&g_bdev_nvme_mutex); TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) { @@ -1131,7 +1140,9 @@ remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr) if (ns->active) { assert(ns->id == nsid); ns->active = false; - spdk_bdev_unregister(&ns->bdev->disk, NULL, NULL); + TAILQ_FOREACH_SAFE(nvme_bdev, &ns->bdevs, tailq, tmp) { + spdk_bdev_unregister(&nvme_bdev->disk, NULL, NULL); + } } } @@ -1262,6 +1273,7 @@ bdev_nvme_create_bdevs(struct nvme_async_probe_ctx *ctx, spdk_bdev_create_nvme_f { struct nvme_bdev_ctrlr *nvme_bdev_ctrlr; struct nvme_bdev_ns *ns; + struct nvme_bdev *nvme_bdev, *tmp; uint32_t i, nsid; size_t j; @@ -1282,14 +1294,16 @@ bdev_nvme_create_bdevs(struct nvme_async_probe_ctx *ctx, spdk_bdev_create_nvme_f continue; } assert(ns->id == nsid); - if (j < ctx->count) { - ctx->names[j] = ns->bdev->disk.name; - j++; - } else { - SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", - ctx->count); - cb_fn(cb_arg, 0, -ERANGE); - return; + TAILQ_FOREACH_SAFE(nvme_bdev, &ns->bdevs, tailq, tmp) { + if (j < ctx->count) { + ctx->names[j] = nvme_bdev->disk.name; + j++; + } else { + SPDK_ERRLOG("Maximum number of namespaces supported per NVMe controller is %du. Unable to return all names of created bdevs\n", + ctx->count); + cb_fn(cb_arg, 0, -ERANGE); + return; + } } } @@ -1691,6 +1705,8 @@ nvme_ctrlr_create_bdevs(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr) ns->id = nsid; ns->ctrlr = nvme_bdev_ctrlr; + TAILQ_INIT(&ns->bdevs); + rc = nvme_ctrlr_create_bdev(nvme_bdev_ctrlr, ns); if (rc == 0) { ns->active = true; diff --git a/module/bdev/nvme/common.h b/module/bdev/nvme/common.h index 735f7d996..c0e800a59 100644 --- a/module/bdev/nvme/common.h +++ b/module/bdev/nvme/common.h @@ -49,7 +49,7 @@ struct nvme_bdev_ns { bool active; struct spdk_nvme_ns *ns; struct nvme_bdev_ctrlr *ctrlr; - struct nvme_bdev *bdev; + TAILQ_HEAD(, nvme_bdev) bdevs; }; struct nvme_bdev_ctrlr { @@ -84,8 +84,9 @@ struct nvme_bdev_ctrlr { struct nvme_bdev { struct spdk_bdev disk; - struct nvme_bdev_ctrlr *nvme_bdev_ctrlr; struct nvme_bdev_ns *nvme_ns; + struct nvme_bdev_ctrlr *nvme_bdev_ctrlr; + TAILQ_ENTRY(nvme_bdev) tailq; }; typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, size_t bdev_count, int rc); diff --git a/module/bdev/nvme/vbdev_opal.c b/module/bdev/nvme/vbdev_opal.c index 43893405b..5f55f2921 100644 --- a/module/bdev/nvme/vbdev_opal.c +++ b/module/bdev/nvme/vbdev_opal.c @@ -347,6 +347,7 @@ spdk_vbdev_opal_create(const char *nvme_ctrlr_name, uint32_t nsid, uint8_t locki struct spdk_bdev_part *part_bdev; SPDK_BDEV_PART_TAILQ *part_tailq; struct spdk_vbdev_opal_config *cfg; + struct nvme_bdev *nvme_bdev; if (nsid != NSID_SUPPORTED) { SPDK_ERRLOG("nsid %d not supported", nsid); @@ -389,7 +390,10 @@ spdk_vbdev_opal_create(const char *nvme_ctrlr_name, uint32_t nsid, uint8_t locki vbdev_opal_free_bdev(opal_bdev); return -EINVAL; } - base_bdev_name = nvme_ctrlr->namespaces[nsid - 1]->bdev->disk.name; + + nvme_bdev = TAILQ_FIRST(&nvme_ctrlr->namespaces[nsid - 1]->bdevs); + assert(nvme_bdev != NULL); + base_bdev_name = nvme_bdev->disk.name; /* traverse base list to see if part_base is already create for this base bdev */ TAILQ_FOREACH(opal_part_base, &g_opal_base, tailq) {