From 723d9443f936dc5da1273e125ca1ea9081c6ea21 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Sat, 19 Dec 2020 15:42:30 +0900 Subject: [PATCH] bdev/nvme: Separate ref. count between nvme_bdev_ctrlr and nvme_bdev_ns Separate reference count of nvme_bdev_ctrlr between nvme_bdev_ctrlr and nvme_bdev_ns. Set ctrlr->ref to 1 when creating ctrlr, increment ctrlr->ref when populating ns, decrement ctrlr->ref when destructing ctrlr or when ns->ref becomes 0, and destruct ctrlr actually when ctrlr->ref is 0. Set ns->ref to 1 when populating ns, increment ns->ref when adding bdev to ns, decrement ns->ref when depopulating ns or removing bdev from ns, and decrement ns->ctrlr->ref when ns->ref becomes 0. Signed-off-by: Shuhei Matsumoto Change-Id: I7810384d97a174d8f55d316e5cdf2a9ef4a11432 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5608 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ziye Yang Reviewed-by: Changpeng Liu --- module/bdev/nvme/bdev_nvme.c | 12 ++++++++++-- module/bdev/nvme/bdev_ocssd.c | 1 + module/bdev/nvme/common.c | 13 ++++++++++--- module/bdev/nvme/common.h | 1 + test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c | 8 ++++++-- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index 8e971e6e3..73209eb37 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -1218,6 +1218,7 @@ nvme_ctrlr_populate_standard_namespace(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, } nvme_ns->ns = ns; + nvme_ns->ref = 1; rc = nvme_bdev_create(nvme_bdev_ctrlr, nvme_ns); done: @@ -1311,9 +1312,16 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr, void nvme_ctrlr_depopulate_namespace_done(struct nvme_bdev_ns *nvme_ns) { - struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = nvme_ns->ctrlr; + pthread_mutex_lock(&g_bdev_nvme_mutex); + assert(nvme_ns->ref > 0); + nvme_ns->ref--; + if (nvme_ns->ref > 0) { + pthread_mutex_unlock(&g_bdev_nvme_mutex); + return; + } + pthread_mutex_unlock(&g_bdev_nvme_mutex); - nvme_bdev_ctrlr_destruct(nvme_bdev_ctrlr); + nvme_bdev_ctrlr_destruct(nvme_ns->ctrlr); } static void diff --git a/module/bdev/nvme/bdev_ocssd.c b/module/bdev/nvme/bdev_ocssd.c index e231d5632..bba71326c 100644 --- a/module/bdev/nvme/bdev_ocssd.c +++ b/module/bdev/nvme/bdev_ocssd.c @@ -1521,6 +1521,7 @@ bdev_ocssd_populate_namespace(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, nvme_ns->type_ctx = ocssd_ns; nvme_ns->ns = ns; + nvme_ns->ref = 1; ctx->nvme_ctx = nvme_ctx; ctx->nvme_ns = nvme_ns; diff --git a/module/bdev/nvme/common.c b/module/bdev/nvme/common.c index ac04b829a..28aef1740 100644 --- a/module/bdev/nvme/common.c +++ b/module/bdev/nvme/common.c @@ -189,7 +189,7 @@ nvme_bdev_attach_bdev_to_ns(struct nvme_bdev_ns *nvme_ns, struct nvme_bdev *nvme { assert(nvme_disk->nvme_ns == nvme_ns); - nvme_ns->ctrlr->ref++; + nvme_ns->ref++; TAILQ_INSERT_TAIL(&nvme_ns->bdevs, nvme_disk, tailq); } @@ -197,11 +197,18 @@ nvme_bdev_attach_bdev_to_ns(struct nvme_bdev_ns *nvme_ns, struct nvme_bdev *nvme void nvme_bdev_detach_bdev_from_ns(struct nvme_bdev *nvme_disk) { - struct nvme_bdev_ctrlr *ctrlr = nvme_disk->nvme_ns->ctrlr; + struct nvme_bdev_ns *nvme_ns = nvme_disk->nvme_ns; pthread_mutex_lock(&g_bdev_nvme_mutex); TAILQ_REMOVE(&nvme_disk->nvme_ns->bdevs, nvme_disk, tailq); + + assert(nvme_ns->ref > 0); + nvme_ns->ref--; + if (nvme_ns->ref > 0) { + pthread_mutex_unlock(&g_bdev_nvme_mutex); + return; + } pthread_mutex_unlock(&g_bdev_nvme_mutex); - nvme_bdev_ctrlr_destruct(ctrlr); + nvme_bdev_ctrlr_destruct(nvme_ns->ctrlr); } diff --git a/module/bdev/nvme/common.h b/module/bdev/nvme/common.h index 7b0a2f27f..83bd30247 100644 --- a/module/bdev/nvme/common.h +++ b/module/bdev/nvme/common.h @@ -61,6 +61,7 @@ struct nvme_bdev_ns { * or when a namespace becomes inactive. */ bool populated; + int ref; struct spdk_nvme_ns *ns; struct nvme_bdev_ctrlr *ctrlr; TAILQ_HEAD(, nvme_bdev) bdevs; diff --git a/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c b/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c index fb9208255..6f76a614c 100644 --- a/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c +++ b/test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c @@ -197,9 +197,13 @@ nvme_ctrlr_populate_namespace_done(struct nvme_async_probe_ctx *ctx, void nvme_ctrlr_depopulate_namespace_done(struct nvme_bdev_ns *ns) { - struct nvme_bdev_ctrlr *ctrlr = ns->ctrlr; + CU_ASSERT(ns->ref > 0); + ns->ref--; + if (ns->ref > 0) { + return; + } - nvme_bdev_ctrlr_destruct(ctrlr); + nvme_bdev_ctrlr_destruct(ns->ctrlr); } static struct nvme_bdev_ctrlr *