nvmf: add support of hotplug for nvmf.

Change-Id: Iebd5b75e3525e77bf256f5b7f52aa2504d7a68c3
Signed-off-by: Cunyin Chang <cunyin.chang@intel.com>
Reviewed-on: https://review.gerrithub.io/390549
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>

lib/nvmf/ctrlr.c

@@ -813,7 +813,7 @@ spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_subsystem *subsystem,
 	struct spdk_nvmf_ns *ns;
 
 	ns = _spdk_nvmf_subsystem_get_ns(subsystem, cmd->nsid);
-	if (ns == NULL || ns->bdev == NULL) {
+	if (ns == NULL || ns->bdev == NULL || ns->is_removed) {
 		SPDK_ERRLOG("Identify Namespace for invalid NSID %u\n", cmd->nsid);
 		rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
 		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
@@ -913,7 +913,7 @@ spdk_nvmf_ctrlr_identify_active_ns_list(struct spdk_nvmf_subsystem *subsystem,
 
 	for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL;
 	     ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) {
-		if (ns->id <= cmd->nsid) {
+		if (ns->id <= cmd->nsid || ns->is_removed) {
 			continue;
 		}
 

lib/nvmf/ctrlr_bdev.c

@@ -390,9 +390,10 @@ spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req)
 	nsid = cmd->nsid;
 
 	ns = _spdk_nvmf_subsystem_get_ns(subsystem, nsid);
-	if (ns == NULL || ns->bdev == NULL) {
+	if (ns == NULL || ns->bdev == NULL || ns->is_removed) {
 		SPDK_ERRLOG("Unsuccessful query for nsid %u\n", cmd->nsid);
 		response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
+		response->status.dnr = 1;
 		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;
 	}
 

lib/nvmf/ctrlr_discovery.c

@@ -40,6 +40,7 @@
 #include "nvmf_internal.h"
 #include "transport.h"
 
+#include "spdk/event.h"
 #include "spdk/string.h"
 #include "spdk/trace.h"
 #include "spdk/nvmf_spec.h"

lib/nvmf/nvmf_internal.h

@@ -130,6 +130,7 @@ struct spdk_nvmf_ns {
 	struct spdk_bdev_desc *desc;
 	uint32_t id;
 	bool allocated;
+	bool is_removed;
 };
 
 enum spdk_nvmf_qpair_type {

lib/nvmf/subsystem.c

@@ -36,6 +36,7 @@
 #include "nvmf_internal.h"
 #include "transport.h"
 
+#include "spdk/event.h"
 #include "spdk/likely.h"
 #include "spdk/string.h"
 #include "spdk/trace.h"
@@ -488,6 +489,25 @@ spdk_nvmf_subsystem_ns_update_poll_group(struct spdk_io_channel_iter *i)
 	spdk_for_each_channel_continue(i, rc);
 }
 
+static void
+_spdk_nvmf_ns_hot_remove(void *ctx)
+{
+	struct spdk_nvmf_ns *ns = ctx;
+
+	spdk_nvmf_subsystem_remove_ns(ns->subsystem, ns->id);
+}
+
+static void
+spdk_nvmf_ns_hot_remove(void *remove_ctx)
+{
+	struct spdk_nvmf_ns *ns = remove_ctx;
+
+	ns->is_removed = true;
+	spdk_thread_send_msg(ns->subsystem->tgt->master_thread,
+			     _spdk_nvmf_ns_hot_remove,
+			     ns);
+}
+
 uint32_t
 spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bdev *bdev,
 			   uint32_t nsid)
@@ -554,7 +574,7 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd
 	ns->bdev = bdev;
 	ns->id = nsid;
 	ns->subsystem = subsystem;
-	rc = spdk_bdev_open(bdev, true, NULL, NULL, &ns->desc);
+	rc = spdk_bdev_open(bdev, true, spdk_nvmf_ns_hot_remove, ns, &ns->desc);
 	if (rc != 0) {
 		SPDK_ERRLOG("Subsystem %s: bdev %s cannot be opened, error=%d\n",
 			    subsystem->subnqn, spdk_bdev_get_name(bdev), rc);

test/nvmf/fio/fio.sh

@@ -45,8 +45,33 @@ $testdir/nvmf_fio.py 4096 128 write 1 verify
 $testdir/nvmf_fio.py 4096 128 randwrite 1 verify
 
 sync
+
+#start hotplug test case
+$testdir/nvmf_fio.py 4096 1 read 10 &
+fio_pid=$!
+
+sleep 3
+set +e
+
+for bdev in $bdevs; do
+	$rpc_py delete_bdev "$bdev"
+done
+
+wait $fio_pid
+fio_status=$?
+
 nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true
 
+if [ $fio_status -eq 0 ]; then
+        echo "nvmf hotplug test: fio successful - expected failure"
+        nvmfcleanup
+        killprocess $nvmfpid
+        exit 1
+else
+        echo "nvmf hotplug test: fio failed as expected"
+fi
+set -e
+
 $rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1
 
 rm -f ./local-job0-0-verify.state

test/nvmf/nvmf.sh

@@ -23,7 +23,6 @@ trap "kill_stub; exit 1" SIGINT SIGTERM EXIT
 
 export NVMF_APP="./app/nvmf_tgt/nvmf_tgt -i 0"
 
-run_test test/nvmf/fio/fio.sh
 run_test test/nvmf/filesystem/filesystem.sh
 run_test test/nvmf/discovery/discovery.sh
 run_test test/nvmf/nvme_cli/nvme_cli.sh
@@ -62,5 +61,6 @@ kill_stub
 # TODO: enable nvme device detachment for multi-process so that
 #  we can use the stub for this test
 run_test test/nvmf/rpc/rpc.sh
+run_test test/nvmf/fio/fio.sh
 revert_soft_roce
 timing_exit nvmf_tgt

test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c

@@ -39,6 +39,24 @@
 
 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF)
 
+uint32_t
+spdk_env_get_current_core(void)
+{
+	return 0;
+}
+
+struct spdk_event *
+spdk_event_allocate(uint32_t core, spdk_event_fn fn, void *arg1, void *arg2)
+{
+	return NULL;
+}
+
+void
+spdk_event_call(struct spdk_event *event)
+{
+
+}
+
 int
 spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,
 	       void *remove_ctx, struct spdk_bdev_desc **desc)

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

@@ -39,6 +39,24 @@
 
 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF)
 
+uint32_t
+spdk_env_get_current_core(void)
+{
+	return 0;
+}
+
+struct spdk_event *
+spdk_event_allocate(uint32_t core, spdk_event_fn fn, void *arg1, void *arg2)
+{
+	return NULL;
+}
+
+void
+spdk_event_call(struct spdk_event *event)
+{
+
+}
+
 int
 spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport,
 			   const struct spdk_nvme_transport_id *trid)