From 4fbe54f10e76c1819e493fd96ee9abffeec9c05e Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Thu, 23 Jun 2016 07:23:36 +0800 Subject: [PATCH] nvmf: add discovery subsystem to NVMf target Change-Id: I4ee79ad268ae75208feddd62e22d6210a9c0d944 Signed-off-by: Changpeng Liu --- autotest.sh | 1 + lib/nvmf/port.c | 8 +++ lib/nvmf/port.h | 5 ++ lib/nvmf/request.c | 78 +++++++++++++++++++++++--- lib/nvmf/session.c | 54 +++++++++++++++++- lib/nvmf/subsystem_grp.c | 95 +++++++++++++++++++++++++++++++- lib/nvmf/subsystem_grp.h | 8 ++- test/lib/nvmf/nvmf_c/nvmf_ut.c | 4 +- test/nvmf/discovery/discovery.sh | 37 +++++++++++++ 9 files changed, 273 insertions(+), 17 deletions(-) create mode 100755 test/nvmf/discovery/discovery.sh diff --git a/autotest.sh b/autotest.sh index 43f8130e2..753e92ceb 100755 --- a/autotest.sh +++ b/autotest.sh @@ -59,6 +59,7 @@ timing_enter nvmf time test/nvmf/fio/fio.sh time test/nvmf/filesystem/filesystem.sh +time test/nvmf/discovery/discovery.sh timing_exit nvmf diff --git a/lib/nvmf/port.c b/lib/nvmf/port.c index 0a5030973..13a4ea183 100644 --- a/lib/nvmf/port.c +++ b/lib/nvmf/port.c @@ -39,6 +39,7 @@ #include "port.h" #include "spdk/log.h" #include "spdk/trace.h" +#include "spdk/nvmf_spec.h" #define MAX_FABRIC_INTF_PER_PORT 4 #define MAX_PORTS 4 @@ -65,6 +66,9 @@ spdk_nvmf_fabric_intf_create(char *host, char *sin_port) fabric_intf->host = host; fabric_intf->sin_port = sin_port; + fabric_intf->trtype = SPDK_NVMF_TRANS_RDMA; + fabric_intf->adrfam = SPDK_NVMF_ADDR_FAMILY_IPV4; + fabric_intf->treq = SPDK_NVMF_TREQ_NOT_SPECIFIED; return fabric_intf; } @@ -132,6 +136,10 @@ spdk_nvmf_port_create(int tag) port->state = GROUP_INIT; port->tag = tag; port->type = FABRIC_RDMA; + port->rdma.rdma_qptype = SPDK_NVMF_QP_TYPE_RELIABLE_CONNECTED; + /* No provider specified */ + port->rdma.rdma_prtype = SPDK_NVMF_RDMA_NO_PROVIDER; + port->rdma.rdma_cms = SPDK_NVMF_RDMA_CMS_RDMA_CM; TAILQ_INIT(&port->head); diff --git a/lib/nvmf/port.h b/lib/nvmf/port.h index fc2aebe26..8a2b73905 100644 --- a/lib/nvmf/port.h +++ b/lib/nvmf/port.h @@ -38,6 +38,7 @@ #include "spdk/conf.h" #include "spdk/queue.h" +#include "spdk/nvmf_spec.h" /** \file * An NVMf subsystem port, referred to as simply "port" is defined by the @@ -65,6 +66,9 @@ struct spdk_nvmf_fabric_intf { char *host; char *sin_port; struct spdk_nvmf_port *port; + enum spdk_nvmf_transport_types trtype; + enum spdk_nvmf_address_family_types adrfam; + enum spdk_nvmf_transport_requirements treq; uint32_t num_sessions; TAILQ_ENTRY(spdk_nvmf_fabric_intf) tailq; }; @@ -73,6 +77,7 @@ struct spdk_nvmf_port { int tag; enum group_state state; enum fabric_type type; + struct spdk_nvmf_rdma_transport_specific_address rdma; TAILQ_HEAD(, spdk_nvmf_fabric_intf) head; TAILQ_ENTRY(spdk_nvmf_port) tailq; }; diff --git a/lib/nvmf/request.c b/lib/nvmf/request.c index 19a2c7c82..9f99efd87 100644 --- a/lib/nvmf/request.c +++ b/lib/nvmf/request.c @@ -91,6 +91,64 @@ spdk_nvmf_request_release(struct spdk_nvmf_request *req) return spdk_nvmf_rdma_request_release(req->conn, req); } +static bool +nvmf_process_discovery_cmd(struct spdk_nvmf_request *req) +{ + struct nvmf_session *session = req->conn->sess; + struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd; + struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl; + struct spdk_nvmf_discovery_log_page *log; + + /* pre-set response details for this command */ + response->status.sc = SPDK_NVME_SC_SUCCESS; + response->cid = cmd->cid; + + if (req->data == NULL) { + SPDK_ERRLOG("discovery command with no buffer\n"); + response->status.sc = SPDK_NVME_SC_INVALID_FIELD; + return true; + } + + switch (cmd->opc) { + case SPDK_NVME_OPC_IDENTIFY: + /* Only identify controller can be supported */ + if (cmd->cdw10 == 1) { + /* identify controller */ + SPDK_TRACELOG(SPDK_TRACE_NVMF, "Identify Controller\n"); + memcpy(req->data, (char *)&session->vcdata, sizeof(struct spdk_nvme_ctrlr_data)); + return true; + } else { + SPDK_ERRLOG("Unsupported identify command\n"); + response->status.sc = SPDK_NVME_SC_INVALID_FIELD; + return true; + } + break; + case SPDK_NVME_OPC_GET_LOG_PAGE: + if ((cmd->cdw10 & 0xFF) == SPDK_NVME_LOG_DISCOVERY) { + log = (struct spdk_nvmf_discovery_log_page *)req->data; + /* + * Does not support change discovery + * information at runtime now. + */ + log->genctr = 0; + log->numrec = 0; + spdk_format_discovery_log(log, req->length); + return true; + } else { + SPDK_ERRLOG("Unsupported log page %u\n", cmd->cdw10 & 0xFF); + response->status.sc = SPDK_NVME_SC_INVALID_FIELD; + return true; + } + break; + default: + SPDK_ERRLOG("Unsupported Opcode 0x%x for Discovery service\n", cmd->opc); + response->status.sc = SPDK_NVME_SC_INVALID_FIELD; + return true; + } + + return true; +} + static bool nvmf_process_admin_cmd(struct spdk_nvmf_request *req) { @@ -107,13 +165,6 @@ nvmf_process_admin_cmd(struct spdk_nvmf_request *req) response->status.sc = SPDK_NVME_SC_SUCCESS; response->cid = cmd->cid; - /* verify subsystem */ - if (subsystem == NULL) { - SPDK_TRACELOG(SPDK_TRACE_NVMF, "Subsystem Not Initialized!\n"); - response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; - return true; - } - if (cmd->nsid == 0) { /* may be valid for the requested command. but need to at least map to a known valid controller. @@ -653,7 +704,18 @@ spdk_nvmf_request_exec(struct spdk_nvmf_request *req) if (cmd->opc == SPDK_NVME_OPC_FABRIC) { done = nvmf_process_fabrics_command(req); } else if (req->conn->type == CONN_TYPE_AQ) { - done = nvmf_process_admin_cmd(req); + struct nvmf_session *session; + struct spdk_nvmf_subsystem *subsystem; + + session = req->conn->sess; + RTE_VERIFY(session != NULL); + subsystem = session->subsys; + RTE_VERIFY(subsystem != NULL); + if (subsystem->subtype == SPDK_NVMF_SUB_DISCOVERY) { + done = nvmf_process_discovery_cmd(req); + } else { + done = nvmf_process_admin_cmd(req); + } } else { done = nvmf_process_io_cmd(req); } diff --git a/lib/nvmf/session.c b/lib/nvmf/session.c index 07bcb2582..02029a8ee 100644 --- a/lib/nvmf/session.c +++ b/lib/nvmf/session.c @@ -38,6 +38,7 @@ #include "subsystem_grp.h" #include "spdk/log.h" #include "spdk/trace.h" +#include "spdk/nvme_spec.h" static struct nvmf_session * nvmf_create_session(const char *subnqn) @@ -81,8 +82,47 @@ nvmf_delete_session(struct nvmf_session *session) free(session); } -void -nvmf_init_session_properties(struct nvmf_session *session, int aq_depth) +static void +nvmf_init_discovery_session_properties(struct nvmf_session *session) +{ + struct spdk_nvmf_extended_identify_ctrlr_data *nvmfdata; + + session->vcdata.maxcmd = SPDK_NVMF_DEFAULT_MAX_QUEUE_DEPTH; + /* extended data for get log page supportted */ + session->vcdata.lpa.edlp = 1; + /* reset cntlid in vcdata to match the logical cntlid known to NVMf */ + session->vcdata.cntlid = session->cntlid; + nvmfdata = (struct spdk_nvmf_extended_identify_ctrlr_data *)session->vcdata.nvmf_specific; + nvmfdata->ioccsz = (NVMF_H2C_MAX_MSG / 16); + nvmfdata->iorcsz = (NVMF_C2H_MAX_MSG / 16); + nvmfdata->icdoff = 0; /* offset starts directly after SQE */ + nvmfdata->ctrattr = 0; /* dynamic controller model */ + nvmfdata->msdbd = 1; /* target supports single SGL in capsule */ + session->vcdata.sgls.keyed_sgl = 1; + session->vcdata.sgls.sgl_offset = 1; + + /* Properties */ + session->vcprop.cap_lo.raw = 0; + session->vcprop.cap_lo.bits.cqr = 1; /* NVMF specification required */ + session->vcprop.cap_lo.bits.mqes = (session->vcdata.maxcmd - 1); /* max queue depth */ + session->vcprop.cap_lo.bits.ams = 0; /* optional arb mechanisms */ + + session->vcprop.cap_hi.raw = 0; + session->vcprop.cap_hi.bits.dstrd = 0; /* fixed to 0 for NVMf */ + session->vcprop.cap_hi.bits.css_nvm = 1; /* NVM command set */ + session->vcprop.cap_hi.bits.mpsmin = 0; /* 2 ^ 12 + mpsmin == 4k */ + session->vcprop.cap_hi.bits.mpsmax = 0; /* 2 ^ 12 + mpsmax == 4k */ + + session->vcprop.vs = 0x10000; /* Version Supported: Major 1, Minor 0 */ + + session->vcprop.cc.raw = 0; + + session->vcprop.csts.raw = 0; + session->vcprop.csts.bits.rdy = 0; /* Init controller as not ready */ +} + +static void +nvmf_init_nvme_session_properties(struct nvmf_session *session, int aq_depth) { /* for now base virtual controller properties on first namespace controller */ struct spdk_nvme_ctrlr *ctrlr = session->subsys->ns_list_map[0].ctrlr; @@ -191,6 +231,16 @@ nvmf_init_session_properties(struct nvmf_session *session, int aq_depth) session->vcprop.capattr_hi.raw); } +void +nvmf_init_session_properties(struct nvmf_session *session, int aq_depth) +{ + if (session->subsys->subtype == SPDK_NVMF_SUB_NVME) { + nvmf_init_nvme_session_properties(session, aq_depth); + } else { + nvmf_init_discovery_session_properties(session); + } +} + static struct nvmf_session * nvmf_find_session_by_id(const char *subnqn, uint16_t cntl_id) { diff --git a/lib/nvmf/subsystem_grp.c b/lib/nvmf/subsystem_grp.c index f3e1003aa..7384b70de 100644 --- a/lib/nvmf/subsystem_grp.c +++ b/lib/nvmf/subsystem_grp.c @@ -42,6 +42,7 @@ #include "spdk/log.h" #include "spdk/string.h" #include "spdk/trace.h" +#include "spdk/nvmf_spec.h" #define MAX_TMPBUF 1024 #define SPDK_CN_TAG_MAX 0x0000ffff @@ -70,7 +71,7 @@ nvmf_find_subsystem(const char *subnqn) } struct spdk_nvmf_subsystem * -nvmf_create_subsystem(int num, char *name) +nvmf_create_subsystem(int num, char *name, enum spdk_nvmf_subsystem_types sub_type) { struct spdk_nvmf_subsystem *subsystem; @@ -83,6 +84,7 @@ nvmf_create_subsystem(int num, char *name) SPDK_TRACELOG(SPDK_TRACE_NVMF, "nvmf_create_subsystem: allocated subsystem %p\n", subsystem); subsystem->num = num; + subsystem->subtype = sub_type; snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", name); TAILQ_INIT(&subsystem->sessions); @@ -332,7 +334,7 @@ spdk_cf_add_nvmf_subsystem(struct spdk_conf_section *sp) } /* register this subsystem with the NVMf library */ - ss_group->subsystem = nvmf_create_subsystem(ss_group->num, ss_group->name); + ss_group->subsystem = nvmf_create_subsystem(ss_group->num, ss_group->name, SPDK_NVMF_SUB_NVME); if (ss_group->subsystem == NULL) { SPDK_ERRLOG("Failed creating new nvmf library subsystem\n"); goto err0; @@ -380,6 +382,87 @@ err0: return -1; } +static int +spdk_add_nvmf_discovery_subsystem(void) +{ + struct spdk_nvmf_subsystem_grp *ss_group; + + ss_group = calloc(1, sizeof(*ss_group)); + if (!ss_group) { + SPDK_ERRLOG("could not allocate discovery subsystem group\n"); + return -1; + } + + ss_group->num = 0xffff; + ss_group->name = strdup(SPDK_NVMF_DISCOVERY_NQN); + if (ss_group->name == NULL) { + SPDK_ERRLOG("strdup ss_group->name error\n"); + free(ss_group); + return -1; + } + + ss_group->subsystem = nvmf_create_subsystem(ss_group->num, ss_group->name, SPDK_NVMF_SUB_DISCOVERY); + if (ss_group->subsystem == NULL) { + SPDK_ERRLOG("Failed creating discovery nvmf library subsystem\n"); + free(ss_group); + return -1; + } + TAILQ_INSERT_TAIL(&g_ssg_head, ss_group, tailq); + + return 0; +} + +void +spdk_format_discovery_log(struct spdk_nvmf_discovery_log_page *disc_log, uint32_t length) +{ + int i, numrec = 0; + struct spdk_nvmf_subsystem_grp *ss_group; + struct spdk_nvmf_subsystem *subsystem; + struct spdk_nvmf_access_map *map; + struct spdk_nvmf_port *port; + struct spdk_nvmf_fabric_intf *fabric_intf; + struct spdk_nvmf_discovery_log_page_entry *entry; + + TAILQ_FOREACH(ss_group, &g_ssg_head, tailq) { + subsystem = ss_group->subsystem; + if (subsystem->subtype == SPDK_NVMF_SUB_DISCOVERY) + continue; + + for (i = 0; i < ss_group->map_count; i++) { + map = &ss_group->map[i]; + port = map->port; + if (port != NULL) { + TAILQ_FOREACH(fabric_intf, &port->head, tailq) { + /* include the discovery log entry */ + if (length > sizeof(struct spdk_nvmf_discovery_log_page)) { + if (sizeof(struct spdk_nvmf_discovery_log_page) + (numrec + 1) * sizeof( + struct spdk_nvmf_discovery_log_page_entry) > length) { + break; + } + entry = &disc_log->entries[numrec]; + entry->trtype = fabric_intf->trtype; + entry->adrfam = fabric_intf->adrfam; + entry->treq = fabric_intf->treq; + entry->portid = port->tag; + /* Dynamic controllers */ + entry->cntlid = 0xffff; + entry->subtype = subsystem->subtype; + snprintf(entry->trsvcid, 32, "%s", fabric_intf->sin_port); + snprintf(entry->traddr, 256, "%s", fabric_intf->host); + snprintf(entry->subnqn, 256, "%s", subsystem->subnqn); + entry->tsas.rdma.rdma_qptype = port->rdma.rdma_qptype; + entry->tsas.rdma.rdma_prtype = port->rdma.rdma_prtype; + entry->tsas.rdma.rdma_cms = port->rdma.rdma_cms; + } + numrec++; + } + } + } + } + + disc_log->numrec = numrec; +} + int spdk_initialize_nvmf_subsystems(void) { @@ -407,7 +490,13 @@ spdk_initialize_nvmf_subsystems(void) } sp = spdk_conf_next_section(sp); } - return 0; + + /* Discovery subsystem */ + rc = spdk_add_nvmf_discovery_subsystem(); + if (rc == 0) + printf(" Discovery Service Enabled\n"); + + return rc; } int diff --git a/lib/nvmf/subsystem_grp.h b/lib/nvmf/subsystem_grp.h index c0467907d..f31a05db7 100644 --- a/lib/nvmf/subsystem_grp.h +++ b/lib/nvmf/subsystem_grp.h @@ -59,6 +59,7 @@ struct spdk_nvmf_subsystem { uint16_t num; char subnqn[MAX_NQN_SIZE]; int num_sessions; + enum spdk_nvmf_subsystem_types subtype; TAILQ_HEAD(session_q, nvmf_session) sessions; struct spdk_nvmf_namespace ns_list_map[MAX_PER_SUBSYSTEM_NAMESPACES]; int ns_count; @@ -73,7 +74,7 @@ struct spdk_nvmf_access_map { struct spdk_nvmf_subsystem_grp { int num; - char *name;; + char *name; struct spdk_nvmf_subsystem *subsystem; int map_count; struct spdk_nvmf_access_map map[MAX_PER_SUBSYSTEM_ACCESS_MAP]; @@ -81,7 +82,7 @@ struct spdk_nvmf_subsystem_grp { }; struct spdk_nvmf_subsystem * -nvmf_create_subsystem(int num, char *name); +nvmf_create_subsystem(int num, char *name, enum spdk_nvmf_subsystem_types sub_type); int nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem); @@ -99,4 +100,7 @@ spdk_initialize_nvmf_subsystems(void); int spdk_shutdown_nvmf_subsystems(void); +void +spdk_format_discovery_log(struct spdk_nvmf_discovery_log_page *disc_log, uint32_t length); + #endif /* _NVMF_SUBSYSTEM_GROUP_H_ */ diff --git a/test/lib/nvmf/nvmf_c/nvmf_ut.c b/test/lib/nvmf/nvmf_c/nvmf_ut.c index 40311d347..28ceb8d1d 100644 --- a/test/lib/nvmf/nvmf_c/nvmf_ut.c +++ b/test/lib/nvmf/nvmf_c/nvmf_ut.c @@ -343,7 +343,7 @@ nvmf_test_create_subsystem(void) char wrong_name[512]; struct spdk_nvmf_subsystem *subsystem; struct spdk_nvmf_ctrlr *nvmf_ctrlr; - subsystem = nvmf_create_subsystem(1, correct_name); + subsystem = nvmf_create_subsystem(1, correct_name, SPDK_NVMF_SUB_NVME); SPDK_CU_ASSERT_FATAL(subsystem != NULL); CU_ASSERT_EQUAL(subsystem->num, 1); CU_ASSERT_STRING_EQUAL(subsystem->subnqn, correct_name); @@ -353,7 +353,7 @@ nvmf_test_create_subsystem(void) /* test long name */ memset(wrong_name, 'a', 512); - subsystem = nvmf_create_subsystem(2, wrong_name); + subsystem = nvmf_create_subsystem(2, wrong_name, SPDK_NVMF_SUB_NVME); SPDK_CU_ASSERT_FATAL(subsystem != NULL); CU_ASSERT_EQUAL(subsystem->num, 2); CU_ASSERT_STRING_NOT_EQUAL(subsystem->subnqn, wrong_name); diff --git a/test/nvmf/discovery/discovery.sh b/test/nvmf/discovery/discovery.sh new file mode 100755 index 000000000..d9c74b5fa --- /dev/null +++ b/test/nvmf/discovery/discovery.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$testdir/../../.. +source $rootdir/scripts/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +if ! hash nvme; then + echo "nvme command not found; skipping discovery test" + exit 0 +fi + +rdma_device_init + +timing_enter discovery + +# Start up the NVMf target in another process +$rootdir/app/nvmf_tgt/nvmf_tgt -c $testdir/../nvmf.conf -t nvmf -t rdma & +nvmfpid=$! + +trap "process_core; killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT + +sleep 10 + +modprobe -v nvme-rdma + +if [ -e "/dev/nvme-fabrics" ]; then + chmod a+rw /dev/nvme-fabrics +fi + +nvme discover -t rdma -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +trap - SIGINT SIGTERM EXIT + +nvmfcleanup +killprocess $nvmfpid +timing_exit discovery