nvmf: claim each bdev when constructing new Namespace

Claim the block device when adding it to a new Namespace,
and prevent the block device to be added twice for other
modules and Namespaces.  Also remove the test that using
same block device over different Namespaces.

Fix issue #371.

Change-Id: Ib7ce18e9fde4a15c0f19ce9e28e69145e54570e0
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/420472
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/bdev/bdev.c

@@ -3124,7 +3124,8 @@ spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_
 	pthread_mutex_lock(&bdev->internal.mutex);
 
 	if (write && bdev->internal.claim_module) {
-		SPDK_ERRLOG("Could not open %s - already claimed\n", bdev->name);
+		SPDK_ERRLOG("Could not open %s - %s module already claimed it\n",
+			    bdev->name, bdev->internal.claim_module->name);
 		free(desc);
 		pthread_mutex_unlock(&bdev->internal.mutex);
 		return -EPERM;

lib/nvmf/subsystem.c

@@ -898,6 +898,7 @@ _spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t n
 
 	subsystem->ns[nsid - 1] = NULL;
 
+	spdk_bdev_module_release_bdev(ns->bdev);
 	spdk_bdev_close(ns->desc);
 	free(ns);
 
@@ -971,6 +972,11 @@ spdk_nvmf_ns_opts_get_defaults(struct spdk_nvmf_ns_opts *opts, size_t opts_size)
 	memset(opts, 0, opts_size);
 }
 
+/* Dummy bdev module used to to claim bdevs. */
+static struct spdk_bdev_module ns_bdev_module = {
+	.name	= "NVMe-oF Target",
+};
+
 uint32_t
 spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bdev *bdev,
 			   const struct spdk_nvmf_ns_opts *user_opts, size_t opts_size)
@@ -1060,6 +1066,12 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
 		free(ns);
 		return 0;
 	}
+	rc = spdk_bdev_module_claim_bdev(bdev, ns->desc, &ns_bdev_module);
+	if (rc != 0) {
+		spdk_bdev_close(ns->desc);
+		free(ns);
+		return 0;
+	}
 	subsystem->ns[opts.nsid - 1] = ns;
 
 	SPDK_DEBUGLOG(SPDK_LOG_NVMF, "Subsystem %s: bdev %s assigned nsid %" PRIu32 "\n",

test/nvmf/filesystem/filesystem.sh

@@ -32,8 +32,8 @@ for incapsule in 0 4096; do
 	$rpc_py set_nvmf_target_options -u 8192 -p 4 -c $incapsule
 	$rpc_py start_subsystem_init
 
-	bdevs="$bdevs $($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
+	bdevs="$($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
-	bdevs="$bdevs $($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
+	bdevs+=" $($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
 
 	modprobe -v nvme-rdma
 

test/nvmf/nvme_cli/nvme_cli.sh

@@ -38,8 +38,8 @@ $rpc_py set_nvmf_target_options -u 8192 -p 4
 $rpc_py start_subsystem_init
 timing_exit start_nvmf_tgt
 
-bdevs="$bdevs $($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
+bdevs="$($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE) "
-bdevs="$bdevs $($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
+bdevs+="$($rpc_py construct_malloc_bdev $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)"
 
 modprobe -v nvme-rdma
 

test/nvmf/rpc/rpc.sh

@@ -78,7 +78,6 @@ do
 	for bdev in $bdevs; do
 		let j=j+1
 		$rpc_py construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode$j '' '' -s SPDK00000000000001
-		$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$j $bdev -n 10
 		$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$j $bdev -n 5
 		$rpc_py nvmf_subsystem_allow_any_host nqn.2016-06.io.spdk:cnode$j
 		nvme connect -t rdma -n nqn.2016-06.io.spdk:cnode$j -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT"
@@ -93,7 +92,6 @@ do
 	j=0
 	for bdev in $bdevs; do
 		let j=j+1
-		$rpc_py nvmf_subsystem_remove_ns nqn.2016-06.io.spdk:cnode$j 10
 		$rpc_py nvmf_subsystem_remove_ns nqn.2016-06.io.spdk:cnode$j 5
 	done
 

test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c

@@ -34,12 +34,21 @@
 #include "spdk/stdinc.h"
 
 #include "spdk_cunit.h"
+#include "spdk_internal/mock.h"
 
 #include "nvmf/ctrlr_discovery.c"
 #include "nvmf/subsystem.c"
 
 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF)
 
+DEFINE_STUB(spdk_bdev_module_claim_bdev,
+	    int,
+	    (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
+	     struct spdk_bdev_module *module), 0);
+
+DEFINE_STUB_V(spdk_bdev_module_release_bdev,
+	      (struct spdk_bdev *bdev));
+
 uint32_t
 spdk_env_get_current_core(void)
 {

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

@@ -34,11 +34,20 @@
 #include "spdk/stdinc.h"
 
 #include "spdk_cunit.h"
+#include "spdk_internal/mock.h"
 
 #include "nvmf/subsystem.c"
 
 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF)
 
+DEFINE_STUB(spdk_bdev_module_claim_bdev,
+	    int,
+	    (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
+	     struct spdk_bdev_module *module), 0);
+
+DEFINE_STUB_V(spdk_bdev_module_release_bdev,
+	      (struct spdk_bdev *bdev));
+
 static void
 _subsystem_send_msg(spdk_thread_fn fn, void *ctx, void *thread_ctx)
 {