From bdcb0d709ae1fd33e587de6b42d56d7292139eda Mon Sep 17 00:00:00 2001 From: Cunyin Chang Date: Wed, 6 Dec 2017 10:07:47 +0800 Subject: [PATCH] nvmf: add support of hotplug for nvmf. Change-Id: Iebd5b75e3525e77bf256f5b7f52aa2504d7a68c3 Signed-off-by: Cunyin Chang Reviewed-on: https://review.gerrithub.io/390549 Reviewed-by: Ben Walker Reviewed-by: Jim Harris Tested-by: SPDK Automated Test System --- lib/nvmf/ctrlr.c | 4 +-- lib/nvmf/ctrlr_bdev.c | 3 ++- lib/nvmf/ctrlr_discovery.c | 1 + lib/nvmf/nvmf_internal.h | 1 + lib/nvmf/subsystem.c | 22 +++++++++++++++- test/nvmf/fio/fio.sh | 25 +++++++++++++++++++ test/nvmf/nvmf.sh | 2 +- .../ctrlr_discovery.c/ctrlr_discovery_ut.c | 18 +++++++++++++ test/unit/lib/nvmf/subsystem.c/subsystem_ut.c | 18 +++++++++++++ 9 files changed, 89 insertions(+), 5 deletions(-) diff --git a/lib/nvmf/ctrlr.c b/lib/nvmf/ctrlr.c index 55f87c4e0..639ffcb5e 100644 --- a/lib/nvmf/ctrlr.c +++ b/lib/nvmf/ctrlr.c @@ -813,7 +813,7 @@ spdk_nvmf_ctrlr_identify_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_nvmf_ns *ns; ns = _spdk_nvmf_subsystem_get_ns(subsystem, cmd->nsid); - if (ns == NULL || ns->bdev == NULL) { + if (ns == NULL || ns->bdev == NULL || ns->is_removed) { SPDK_ERRLOG("Identify Namespace for invalid NSID %u\n", cmd->nsid); rsp->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; @@ -913,7 +913,7 @@ spdk_nvmf_ctrlr_identify_active_ns_list(struct spdk_nvmf_subsystem *subsystem, for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { - if (ns->id <= cmd->nsid) { + if (ns->id <= cmd->nsid || ns->is_removed) { continue; } diff --git a/lib/nvmf/ctrlr_bdev.c b/lib/nvmf/ctrlr_bdev.c index d187a31a0..71dd62dfa 100644 --- a/lib/nvmf/ctrlr_bdev.c +++ b/lib/nvmf/ctrlr_bdev.c @@ -390,9 +390,10 @@ spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) nsid = cmd->nsid; ns = _spdk_nvmf_subsystem_get_ns(subsystem, nsid); - if (ns == NULL || ns->bdev == NULL) { + if (ns == NULL || ns->bdev == NULL || ns->is_removed) { SPDK_ERRLOG("Unsuccessful query for nsid %u\n", cmd->nsid); response->status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT; + response->status.dnr = 1; return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE; } diff --git a/lib/nvmf/ctrlr_discovery.c b/lib/nvmf/ctrlr_discovery.c index 76dd75c86..840da2f7b 100644 --- a/lib/nvmf/ctrlr_discovery.c +++ b/lib/nvmf/ctrlr_discovery.c @@ -40,6 +40,7 @@ #include "nvmf_internal.h" #include "transport.h" +#include "spdk/event.h" #include "spdk/string.h" #include "spdk/trace.h" #include "spdk/nvmf_spec.h" diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index 008a66023..665adb669 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -130,6 +130,7 @@ struct spdk_nvmf_ns { struct spdk_bdev_desc *desc; uint32_t id; bool allocated; + bool is_removed; }; enum spdk_nvmf_qpair_type { diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index bff5e9fc6..8e71c3c93 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -36,6 +36,7 @@ #include "nvmf_internal.h" #include "transport.h" +#include "spdk/event.h" #include "spdk/likely.h" #include "spdk/string.h" #include "spdk/trace.h" @@ -488,6 +489,25 @@ spdk_nvmf_subsystem_ns_update_poll_group(struct spdk_io_channel_iter *i) spdk_for_each_channel_continue(i, rc); } +static void +_spdk_nvmf_ns_hot_remove(void *ctx) +{ + struct spdk_nvmf_ns *ns = ctx; + + spdk_nvmf_subsystem_remove_ns(ns->subsystem, ns->id); +} + +static void +spdk_nvmf_ns_hot_remove(void *remove_ctx) +{ + struct spdk_nvmf_ns *ns = remove_ctx; + + ns->is_removed = true; + spdk_thread_send_msg(ns->subsystem->tgt->master_thread, + _spdk_nvmf_ns_hot_remove, + ns); +} + uint32_t spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bdev *bdev, uint32_t nsid) @@ -554,7 +574,7 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd ns->bdev = bdev; ns->id = nsid; ns->subsystem = subsystem; - rc = spdk_bdev_open(bdev, true, NULL, NULL, &ns->desc); + rc = spdk_bdev_open(bdev, true, spdk_nvmf_ns_hot_remove, ns, &ns->desc); if (rc != 0) { SPDK_ERRLOG("Subsystem %s: bdev %s cannot be opened, error=%d\n", subsystem->subnqn, spdk_bdev_get_name(bdev), rc); diff --git a/test/nvmf/fio/fio.sh b/test/nvmf/fio/fio.sh index 245ec6d12..2054477a4 100755 --- a/test/nvmf/fio/fio.sh +++ b/test/nvmf/fio/fio.sh @@ -45,8 +45,33 @@ $testdir/nvmf_fio.py 4096 128 write 1 verify $testdir/nvmf_fio.py 4096 128 randwrite 1 verify sync + +#start hotplug test case +$testdir/nvmf_fio.py 4096 1 read 10 & +fio_pid=$! + +sleep 3 +set +e + +for bdev in $bdevs; do + $rpc_py delete_bdev "$bdev" +done + +wait $fio_pid +fio_status=$? + nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true +if [ $fio_status -eq 0 ]; then + echo "nvmf hotplug test: fio successful - expected failure" + nvmfcleanup + killprocess $nvmfpid + exit 1 +else + echo "nvmf hotplug test: fio failed as expected" +fi +set -e + $rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 rm -f ./local-job0-0-verify.state diff --git a/test/nvmf/nvmf.sh b/test/nvmf/nvmf.sh index e0abcebaf..91675b639 100755 --- a/test/nvmf/nvmf.sh +++ b/test/nvmf/nvmf.sh @@ -23,7 +23,6 @@ trap "kill_stub; exit 1" SIGINT SIGTERM EXIT export NVMF_APP="./app/nvmf_tgt/nvmf_tgt -i 0" -run_test test/nvmf/fio/fio.sh run_test test/nvmf/filesystem/filesystem.sh run_test test/nvmf/discovery/discovery.sh run_test test/nvmf/nvme_cli/nvme_cli.sh @@ -62,5 +61,6 @@ kill_stub # TODO: enable nvme device detachment for multi-process so that # we can use the stub for this test run_test test/nvmf/rpc/rpc.sh +run_test test/nvmf/fio/fio.sh revert_soft_roce timing_exit nvmf_tgt diff --git a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c index 784ac4d80..c010193fd 100644 --- a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c +++ b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c @@ -39,6 +39,24 @@ SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) +uint32_t +spdk_env_get_current_core(void) +{ + return 0; +} + +struct spdk_event * +spdk_event_allocate(uint32_t core, spdk_event_fn fn, void *arg1, void *arg2) +{ + return NULL; +} + +void +spdk_event_call(struct spdk_event *event) +{ + +} + int spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb, void *remove_ctx, struct spdk_bdev_desc **desc) diff --git a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c index ce2f72ede..5a3c4bbd6 100644 --- a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c +++ b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c @@ -39,6 +39,24 @@ SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) +uint32_t +spdk_env_get_current_core(void) +{ + return 0; +} + +struct spdk_event * +spdk_event_allocate(uint32_t core, spdk_event_fn fn, void *arg1, void *arg2) +{ + return NULL; +} + +void +spdk_event_call(struct spdk_event *event) +{ + +} + int spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_transport_id *trid)