From 0801877b7d0f0d6b6d0599cdf156f73a866ca65c Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Fri, 10 Feb 2017 16:34:08 +0800 Subject: [PATCH] nvmf,direct: add AER support in direct mode. After checking the code, aerl in our session is 0, so there will be only 1 AER. So currently, we will only handle 1 AER case. When the AER event is triggered by real NVMe device owned by the subsystem, it notifies all sessions belonging to the subsystem. Change-Id: Ia80fb0f03e893c20d8dd14afbed8db10db38301c Signed-off-by: Ziye Yang --- autotest.sh | 1 + include/spdk/nvmf.h | 5 +++ lib/nvmf/direct.c | 28 ++++++++++-- lib/nvmf/session.h | 1 + lib/nvmf/subsystem.c | 2 + lib/nvmf/virtual.c | 6 +++ test/lib/nvme/aer/aer.c | 76 +++++++++++++++++++++++++++++++- test/lib/nvmf/direct/direct_ut.c | 7 +++ test/nvmf/host/aer.sh | 41 +++++++++++++++++ 9 files changed, 163 insertions(+), 4 deletions(-) create mode 100755 test/nvmf/host/aer.sh diff --git a/autotest.sh b/autotest.sh index 004927be5..88c7ef0c1 100755 --- a/autotest.sh +++ b/autotest.sh @@ -125,6 +125,7 @@ fi timing_enter host +run_test test/nvmf/host/aer.sh run_test test/nvmf/host/identify.sh run_test test/nvmf/host/perf.sh run_test test/nvmf/host/identify_kernel_nvmf.sh diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index e2c86b0ae..ad6d49b32 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -84,6 +84,11 @@ struct spdk_nvmf_host { }; struct spdk_nvmf_ctrlr_ops { + /** + * Set NVMe ctrlr AER. + */ + void (*set_aer_callback)(struct spdk_nvmf_subsystem *subsys); + /** * Get NVMe identify controller data. */ diff --git a/lib/nvmf/direct.c b/lib/nvmf/direct.c index bbad84e28..a525e3b51 100644 --- a/lib/nvmf/direct.c +++ b/lib/nvmf/direct.c @@ -177,9 +177,9 @@ nvmf_direct_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req) break; case SPDK_NVME_OPC_ASYNC_EVENT_REQUEST: SPDK_TRACELOG(SPDK_TRACE_NVMF, "Async Event Request\n"); - /* TODO: Just release the request as consumed. AER events will never - * be triggered. */ - return SPDK_NVMF_REQUEST_EXEC_STATUS_RELEASE; + session->aer_req = req; + + return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS; case SPDK_NVME_OPC_KEEP_ALIVE: SPDK_TRACELOG(SPDK_TRACE_NVMF, "Keep Alive\n"); /* @@ -249,7 +249,29 @@ nvmf_direct_ctrlr_detach(struct spdk_nvmf_subsystem *subsystem) } } +static void +nvmf_direct_ctrlr_complete_aer(void *arg, const struct spdk_nvme_cpl *cpl) +{ + struct spdk_nvmf_subsystem *subsystem = (struct spdk_nvmf_subsystem *) arg; + struct spdk_nvmf_session *session; + + TAILQ_FOREACH(session, &subsystem->sessions, link) { + if (session->aer_req) { + nvmf_direct_ctrlr_complete_cmd(session->aer_req, cpl); + session->aer_req = NULL; + } + } +} + +static void +nvmf_direct_ctrlr_set_aer_callback(struct spdk_nvmf_subsystem *subsys) +{ + spdk_nvme_ctrlr_register_aer_callback(subsys->dev.direct.ctrlr, + nvmf_direct_ctrlr_complete_aer, subsys); +} + const struct spdk_nvmf_ctrlr_ops spdk_nvmf_direct_ctrlr_ops = { + .set_aer_callback = nvmf_direct_ctrlr_set_aer_callback, .ctrlr_get_data = nvmf_direct_ctrlr_get_data, .process_admin_cmd = nvmf_direct_ctrlr_process_admin_cmd, .process_io_cmd = nvmf_direct_ctrlr_process_io_cmd, diff --git a/lib/nvmf/session.h b/lib/nvmf/session.h index b23601811..9079ea57b 100644 --- a/lib/nvmf/session.h +++ b/lib/nvmf/session.h @@ -91,6 +91,7 @@ struct spdk_nvmf_session { uint8_t fw_activation_notice : 1; } bits; } async_event_config; + struct spdk_nvmf_request *aer_req; uint8_t hostid[16]; const struct spdk_nvmf_transport *transport; diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index 0eafcd58e..fae797da1 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -311,6 +311,8 @@ nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem, SPDK_ERRLOG("spdk_nvme_ctrlr_alloc_io_qpair() failed\n"); return -1; } + + subsystem->ops->set_aer_callback(subsystem); return 0; } diff --git a/lib/nvmf/virtual.c b/lib/nvmf/virtual.c index b7f6694b5..edeee7491 100644 --- a/lib/nvmf/virtual.c +++ b/lib/nvmf/virtual.c @@ -532,7 +532,13 @@ nvmf_virtual_ctrlr_detach(struct spdk_nvmf_subsystem *subsystem) } } +static void +nvmf_virtual_ctrlr_set_aer_callback(struct spdk_nvmf_subsystem *subsys) +{ +} + const struct spdk_nvmf_ctrlr_ops spdk_nvmf_virtual_ctrlr_ops = { + .set_aer_callback = nvmf_virtual_ctrlr_set_aer_callback, .ctrlr_get_data = nvmf_virtual_ctrlr_get_data, .process_admin_cmd = nvmf_virtual_ctrlr_process_admin_cmd, .process_io_cmd = nvmf_virtual_ctrlr_process_io_cmd, diff --git a/test/lib/nvme/aer/aer.c b/test/lib/nvme/aer/aer.c index 7088a92ab..1570b43fa 100644 --- a/test/lib/nvme/aer/aer.c +++ b/test/lib/nvme/aer/aer.c @@ -31,13 +31,17 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include #include #include +#include +#include #include #include +#include "spdk/log.h" #include "spdk/nvme.h" #include "spdk/env.h" @@ -62,6 +66,7 @@ static int aer_done = 0; static int temperature_done = 0; static int failed = 0; +static struct spdk_nvme_transport_id g_trid; static void set_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl) { @@ -183,6 +188,69 @@ static void aer_cb(void *arg, const struct spdk_nvme_cpl *cpl) get_health_log_page(dev); } +static void +usage(const char *program_name) +{ + printf("%s [options]", program_name); + printf("\n"); + printf("options:\n"); + printf(" -r trid remote NVMe over Fabrics target address\n"); + printf(" Format: 'key:value [key:value] ...'\n"); + printf(" Keys:\n"); + printf(" trtype Transport type (e.g. RDMA)\n"); + printf(" adrfam Address family (e.g. IPv4, IPv6)\n"); + printf(" traddr Transport address (e.g. 192.168.100.8)\n"); + printf(" trsvcid Transport service identifier (e.g. 4420)\n"); + printf(" subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN); + printf(" Example: -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420'\n"); + + spdk_tracelog_usage(stdout, "-t"); + + printf(" -v verbose (enable warnings)\n"); + printf(" -H show this usage\n"); +} + +static int +parse_args(int argc, char **argv) +{ + int op, rc; + + g_trid.trtype = SPDK_NVME_TRANSPORT_PCIE; + snprintf(g_trid.subnqn, sizeof(g_trid.subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); + + while ((op = getopt(argc, argv, "r:t:H")) != -1) { + switch (op) { + case 't': + rc = spdk_log_set_trace_flag(optarg); + if (rc < 0) { + fprintf(stderr, "unknown flag\n"); + usage(argv[0]); + exit(EXIT_FAILURE); + } +#ifndef DEBUG + fprintf(stderr, "%s must be rebuilt with CONFIG_DEBUG=y for -t flag.\n", + argv[0]); + usage(argv[0]); + return 0; +#endif + break; + case 'r': + if (spdk_nvme_transport_id_parse(&g_trid, optarg) != 0) { + fprintf(stderr, "Error parsing transport address\n"); + return 1; + } + break; + case 'H': + default: + usage(argv[0]); + return 1; + } + } + + optind = 1; + + return 0; +} static bool probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, @@ -221,6 +289,12 @@ int main(int argc, char **argv) struct dev *dev; int i; struct spdk_env_opts opts; + int rc; + + rc = parse_args(argc, argv); + if (rc != 0) { + return rc; + } spdk_env_opts_init(&opts); opts.name = "aer"; @@ -229,7 +303,7 @@ int main(int argc, char **argv) printf("Asynchronous Event Request test\n"); - if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) { + if (spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL) != 0) { fprintf(stderr, "spdk_nvme_probe() failed\n"); return 1; } diff --git a/test/lib/nvmf/direct/direct_ut.c b/test/lib/nvmf/direct/direct_ut.c index ea81b501f..6bbd9dbd3 100644 --- a/test/lib/nvmf/direct/direct_ut.c +++ b/test/lib/nvmf/direct/direct_ut.c @@ -66,6 +66,13 @@ spdk_nvme_ctrlr_get_data(struct spdk_nvme_ctrlr *ctrlr) return NULL; } +void +spdk_nvme_ctrlr_register_aer_callback(struct spdk_nvme_ctrlr *ctrlr, + spdk_nvme_aer_cb aer_cb_fn, + void *aer_cb_arg) +{ +} + int spdk_nvme_ctrlr_cmd_io_raw(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, diff --git a/test/nvmf/host/aer.sh b/test/nvmf/host/aer.sh new file mode 100755 index 000000000..dc6bfc5c5 --- /dev/null +++ b/test/nvmf/host/aer.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/scripts/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +rpc_py="python $rootdir/scripts/rpc.py" + +set -e + +if ! rdma_nic_available; then + echo "no NIC for nvmf test" + exit 0 +fi + +timing_enter aer + +# Start up the NVMf target in another process +$rootdir/app/nvmf_tgt/nvmf_tgt -c $testdir/../nvmf.conf -m 0x2 -p 1 -s 512 -t nvmf & +nvmfpid=$! + +trap "killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT + +waitforlisten $nvmfpid ${RPC_PORT} + +$rpc_py construct_nvmf_subsystem Direct nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -p "*" + +$rootdir/test/lib/nvme/aer/aer -r "\ + trtype:RDMA \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:nqn.2014-08.org.nvmexpress.discovery" -t all +sync +$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +killprocess $nvmfpid +timing_exit aer