bdev/nvme: Separate ref. count between nvme_bdev_ctrlr and nvme_bdev_ns

Separate reference count of nvme_bdev_ctrlr between nvme_bdev_ctrlr
and nvme_bdev_ns.

Set ctrlr->ref to 1 when creating ctrlr, increment ctrlr->ref when
populating ns, decrement ctrlr->ref when destructing ctrlr or
when ns->ref becomes 0, and destruct ctrlr actually when ctrlr->ref is 0.

Set ns->ref to 1 when populating ns, increment ns->ref when
adding bdev to ns, decrement ns->ref when depopulating ns or removing
bdev from ns, and decrement ns->ctrlr->ref when ns->ref becomes 0.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I7810384d97a174d8f55d316e5cdf2a9ef4a11432
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5608
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

module/bdev/nvme/bdev_nvme.c

@@ -1218,6 +1218,7 @@ nvme_ctrlr_populate_standard_namespace(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr,
 	}
 
 	nvme_ns->ns = ns;
+	nvme_ns->ref = 1;
 
 	rc = nvme_bdev_create(nvme_bdev_ctrlr, nvme_ns);
 done:
@@ -1311,9 +1312,16 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
 void
 nvme_ctrlr_depopulate_namespace_done(struct nvme_bdev_ns *nvme_ns)
 {
-	struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = nvme_ns->ctrlr;
+	pthread_mutex_lock(&g_bdev_nvme_mutex);
+	assert(nvme_ns->ref > 0);
+	nvme_ns->ref--;
+	if (nvme_ns->ref > 0) {
+		pthread_mutex_unlock(&g_bdev_nvme_mutex);
+		return;
+	}
+	pthread_mutex_unlock(&g_bdev_nvme_mutex);
 
-	nvme_bdev_ctrlr_destruct(nvme_bdev_ctrlr);
+	nvme_bdev_ctrlr_destruct(nvme_ns->ctrlr);
 }
 
 static void

module/bdev/nvme/bdev_ocssd.c

@@ -1521,6 +1521,7 @@ bdev_ocssd_populate_namespace(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr,
 
 	nvme_ns->type_ctx = ocssd_ns;
 	nvme_ns->ns = ns;
+	nvme_ns->ref = 1;
 	ctx->nvme_ctx = nvme_ctx;
 	ctx->nvme_ns = nvme_ns;
 

module/bdev/nvme/common.c

@@ -189,7 +189,7 @@ nvme_bdev_attach_bdev_to_ns(struct nvme_bdev_ns *nvme_ns, struct nvme_bdev *nvme
 {
 	assert(nvme_disk->nvme_ns == nvme_ns);
 
-	nvme_ns->ctrlr->ref++;
+	nvme_ns->ref++;
 
 	TAILQ_INSERT_TAIL(&nvme_ns->bdevs, nvme_disk, tailq);
 }
@@ -197,11 +197,18 @@ nvme_bdev_attach_bdev_to_ns(struct nvme_bdev_ns *nvme_ns, struct nvme_bdev *nvme
 void
 nvme_bdev_detach_bdev_from_ns(struct nvme_bdev *nvme_disk)
 {
-	struct nvme_bdev_ctrlr *ctrlr = nvme_disk->nvme_ns->ctrlr;
+	struct nvme_bdev_ns *nvme_ns = nvme_disk->nvme_ns;
 
 	pthread_mutex_lock(&g_bdev_nvme_mutex);
 	TAILQ_REMOVE(&nvme_disk->nvme_ns->bdevs, nvme_disk, tailq);
+
+	assert(nvme_ns->ref > 0);
+	nvme_ns->ref--;
+	if (nvme_ns->ref > 0) {
+		pthread_mutex_unlock(&g_bdev_nvme_mutex);
+		return;
+	}
 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
 
-	nvme_bdev_ctrlr_destruct(ctrlr);
+	nvme_bdev_ctrlr_destruct(nvme_ns->ctrlr);
 }

module/bdev/nvme/common.h

@@ -61,6 +61,7 @@ struct nvme_bdev_ns {
 	 *  or when a namespace becomes inactive.
 	 */
 	bool			populated;
+	int			ref;
 	struct spdk_nvme_ns	*ns;
 	struct nvme_bdev_ctrlr	*ctrlr;
 	TAILQ_HEAD(, nvme_bdev)	bdevs;

test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c

@@ -197,9 +197,13 @@ nvme_ctrlr_populate_namespace_done(struct nvme_async_probe_ctx *ctx,
 void
 nvme_ctrlr_depopulate_namespace_done(struct nvme_bdev_ns *ns)
 {
-	struct nvme_bdev_ctrlr *ctrlr = ns->ctrlr;
+	CU_ASSERT(ns->ref > 0);
+	ns->ref--;
+	if (ns->ref > 0) {
+		return;
+	}
 
-	nvme_bdev_ctrlr_destruct(ctrlr);
+	nvme_bdev_ctrlr_destruct(ns->ctrlr);
 }
 
 static struct nvme_bdev_ctrlr *