ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf: add discovery subsystem to NVMf target

Browse Source 
Change-Id: I4ee79ad268ae75208feddd62e22d6210a9c0d944
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
This commit is contained in:
Changpeng Liu 2016-06-23 07:23:36 +08:00 committed by Daniel Verkamp
parent 771d2c1a52
commit 4fbe54f10e
9 changed files with 273 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
autotest.sh
View File

@ -59,6 +59,7 @@ timing_enter nvmf
time test/nvmf/fio/fio.sh
time test/nvmf/filesystem/filesystem.sh
time test/nvmf/discovery/discovery.sh
timing_exit nvmf

8
lib/nvmf/port.c
View File

@ -39,6 +39,7 @@
#include "port.h"
#include "spdk/log.h"
#include "spdk/trace.h"
#include "spdk/nvmf_spec.h"
#define MAX_FABRIC_INTF_PER_PORT 4
#define MAX_PORTS 4
@ -65,6 +66,9 @@ spdk_nvmf_fabric_intf_create(char *host, char *sin_port)
fabric_intf->host = host;
fabric_intf->sin_port = sin_port;
fabric_intf->trtype = SPDK_NVMF_TRANS_RDMA;
fabric_intf->adrfam = SPDK_NVMF_ADDR_FAMILY_IPV4;
fabric_intf->treq = SPDK_NVMF_TREQ_NOT_SPECIFIED;
return fabric_intf;
}
@ -132,6 +136,10 @@ spdk_nvmf_port_create(int tag)
port->state = GROUP_INIT;
port->tag = tag;
port->type = FABRIC_RDMA;
port->rdma.rdma_qptype = SPDK_NVMF_QP_TYPE_RELIABLE_CONNECTED;
/* No provider specified */
port->rdma.rdma_prtype = SPDK_NVMF_RDMA_NO_PROVIDER;
port->rdma.rdma_cms = SPDK_NVMF_RDMA_CMS_RDMA_CM;
TAILQ_INIT(&port->head);

5
lib/nvmf/port.h
View File

@ -38,6 +38,7 @@
#include "spdk/conf.h"
#include "spdk/queue.h"
#include "spdk/nvmf_spec.h"
/** \file
* An NVMf subsystem port, referred to as simply "port" is defined by the
@ -65,6 +66,9 @@ struct spdk_nvmf_fabric_intf {
char *host;
char *sin_port;
struct spdk_nvmf_port *port;
enum spdk_nvmf_transport_types trtype;
enum spdk_nvmf_address_family_types adrfam;
enum spdk_nvmf_transport_requirements treq;
uint32_t num_sessions;
TAILQ_ENTRY(spdk_nvmf_fabric_intf) tailq;
};
@ -73,6 +77,7 @@ struct spdk_nvmf_port {
int tag;
enum group_state state;
enum fabric_type type;
struct spdk_nvmf_rdma_transport_specific_address rdma;
TAILQ_HEAD(, spdk_nvmf_fabric_intf) head;
TAILQ_ENTRY(spdk_nvmf_port) tailq;
};

78
lib/nvmf/request.c
View File

@ -91,6 +91,64 @@ spdk_nvmf_request_release(struct spdk_nvmf_request *req)
return spdk_nvmf_rdma_request_release(req->conn, req);
}
static bool
nvmf_process_discovery_cmd(struct spdk_nvmf_request *req)
{
struct nvmf_session *session = req->conn->sess;
struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
struct spdk_nvmf_discovery_log_page *log;
/* pre-set response details for this command */
response->status.sc = SPDK_NVME_SC_SUCCESS;
response->cid = cmd->cid;
if (req->data == NULL) {
SPDK_ERRLOG("discovery command with no buffer\n");
response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
return true;
}
switch (cmd->opc) {
case SPDK_NVME_OPC_IDENTIFY:
/* Only identify controller can be supported */
if (cmd->cdw10 == 1) {
/* identify controller */
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Identify Controller\n");
memcpy(req->data, (char *)&session->vcdata, sizeof(struct spdk_nvme_ctrlr_data));
return true;
} else {
SPDK_ERRLOG("Unsupported identify command\n");
response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
return true;
}
break;
case SPDK_NVME_OPC_GET_LOG_PAGE:
if ((cmd->cdw10 & 0xFF) == SPDK_NVME_LOG_DISCOVERY) {
log = (struct spdk_nvmf_discovery_log_page *)req->data;
/*
* Does not support change discovery
* information at runtime now.
*/
log->genctr = 0;
log->numrec = 0;
spdk_format_discovery_log(log, req->length);
return true;
} else {
SPDK_ERRLOG("Unsupported log page %u\n", cmd->cdw10 & 0xFF);
response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
return true;
}
break;
default:
SPDK_ERRLOG("Unsupported Opcode 0x%x for Discovery service\n", cmd->opc);
response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
return true;
}
return true;
}
static bool
nvmf_process_admin_cmd(struct spdk_nvmf_request *req)
{
@ -107,13 +165,6 @@ nvmf_process_admin_cmd(struct spdk_nvmf_request *req)
response->status.sc = SPDK_NVME_SC_SUCCESS;
response->cid = cmd->cid;
/* verify subsystem */
if (subsystem == NULL) {
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Subsystem Not Initialized!\n");
response->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
return true;
}
if (cmd->nsid == 0) {
/* may be valid for the requested command. but need
to at least map to a known valid controller.
@ -653,7 +704,18 @@ spdk_nvmf_request_exec(struct spdk_nvmf_request *req)
if (cmd->opc == SPDK_NVME_OPC_FABRIC) {
done = nvmf_process_fabrics_command(req);
} else if (req->conn->type == CONN_TYPE_AQ) {
done = nvmf_process_admin_cmd(req);
struct nvmf_session *session;
struct spdk_nvmf_subsystem *subsystem;
session = req->conn->sess;
RTE_VERIFY(session != NULL);
subsystem = session->subsys;
RTE_VERIFY(subsystem != NULL);
if (subsystem->subtype == SPDK_NVMF_SUB_DISCOVERY) {
done = nvmf_process_discovery_cmd(req);
} else {
done = nvmf_process_admin_cmd(req);
}
} else {
done = nvmf_process_io_cmd(req);
}

54
lib/nvmf/session.c
View File

@ -38,6 +38,7 @@
#include "subsystem_grp.h"
#include "spdk/log.h"
#include "spdk/trace.h"
#include "spdk/nvme_spec.h"
static struct nvmf_session *
nvmf_create_session(const char *subnqn)
@ -81,8 +82,47 @@ nvmf_delete_session(struct nvmf_session *session)
free(session);
}
void
nvmf_init_session_properties(struct nvmf_session *session, int aq_depth)
static void
nvmf_init_discovery_session_properties(struct nvmf_session *session)
{
struct spdk_nvmf_extended_identify_ctrlr_data *nvmfdata;
session->vcdata.maxcmd = SPDK_NVMF_DEFAULT_MAX_QUEUE_DEPTH;
/* extended data for get log page supportted */
session->vcdata.lpa.edlp = 1;
/* reset cntlid in vcdata to match the logical cntlid known to NVMf */
session->vcdata.cntlid = session->cntlid;
nvmfdata = (struct spdk_nvmf_extended_identify_ctrlr_data *)session->vcdata.nvmf_specific;
nvmfdata->ioccsz = (NVMF_H2C_MAX_MSG / 16);
nvmfdata->iorcsz = (NVMF_C2H_MAX_MSG / 16);
nvmfdata->icdoff = 0; /* offset starts directly after SQE */
nvmfdata->ctrattr = 0; /* dynamic controller model */
nvmfdata->msdbd = 1; /* target supports single SGL in capsule */
session->vcdata.sgls.keyed_sgl = 1;
session->vcdata.sgls.sgl_offset = 1;
/* Properties */
session->vcprop.cap_lo.raw = 0;
session->vcprop.cap_lo.bits.cqr = 1; /* NVMF specification required */
session->vcprop.cap_lo.bits.mqes = (session->vcdata.maxcmd - 1); /* max queue depth */
session->vcprop.cap_lo.bits.ams = 0; /* optional arb mechanisms */
session->vcprop.cap_hi.raw = 0;
session->vcprop.cap_hi.bits.dstrd = 0; /* fixed to 0 for NVMf */
session->vcprop.cap_hi.bits.css_nvm = 1; /* NVM command set */
session->vcprop.cap_hi.bits.mpsmin = 0; /* 2 ^ 12 + mpsmin == 4k */
session->vcprop.cap_hi.bits.mpsmax = 0; /* 2 ^ 12 + mpsmax == 4k */
session->vcprop.vs = 0x10000; /* Version Supported: Major 1, Minor 0 */
session->vcprop.cc.raw = 0;
session->vcprop.csts.raw = 0;
session->vcprop.csts.bits.rdy = 0; /* Init controller as not ready */
}
static void
nvmf_init_nvme_session_properties(struct nvmf_session *session, int aq_depth)
{
/* for now base virtual controller properties on first namespace controller */
struct spdk_nvme_ctrlr *ctrlr = session->subsys->ns_list_map[0].ctrlr;
@ -191,6 +231,16 @@ nvmf_init_session_properties(struct nvmf_session *session, int aq_depth)
session->vcprop.capattr_hi.raw);
}
void
nvmf_init_session_properties(struct nvmf_session *session, int aq_depth)
{
if (session->subsys->subtype == SPDK_NVMF_SUB_NVME) {
nvmf_init_nvme_session_properties(session, aq_depth);
} else {
nvmf_init_discovery_session_properties(session);
}
}
static struct nvmf_session *
nvmf_find_session_by_id(const char *subnqn, uint16_t cntl_id)
{

95
lib/nvmf/subsystem_grp.c
View File

@ -42,6 +42,7 @@
#include "spdk/log.h"
#include "spdk/string.h"
#include "spdk/trace.h"
#include "spdk/nvmf_spec.h"
#define MAX_TMPBUF 1024
#define SPDK_CN_TAG_MAX 0x0000ffff
@ -70,7 +71,7 @@ nvmf_find_subsystem(const char *subnqn)
}
struct spdk_nvmf_subsystem *
nvmf_create_subsystem(int num, char *name)
nvmf_create_subsystem(int num, char *name, enum spdk_nvmf_subsystem_types sub_type)
{
struct spdk_nvmf_subsystem *subsystem;
@ -83,6 +84,7 @@ nvmf_create_subsystem(int num, char *name)
SPDK_TRACELOG(SPDK_TRACE_NVMF, "nvmf_create_subsystem: allocated subsystem %p\n", subsystem);
subsystem->num = num;
subsystem->subtype = sub_type;
snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", name);
TAILQ_INIT(&subsystem->sessions);
@ -332,7 +334,7 @@ spdk_cf_add_nvmf_subsystem(struct spdk_conf_section *sp)
}
/* register this subsystem with the NVMf library */
ss_group->subsystem = nvmf_create_subsystem(ss_group->num, ss_group->name);
ss_group->subsystem = nvmf_create_subsystem(ss_group->num, ss_group->name, SPDK_NVMF_SUB_NVME);
if (ss_group->subsystem == NULL) {
SPDK_ERRLOG("Failed creating new nvmf library subsystem\n");
goto err0;
@ -380,6 +382,87 @@ err0:
return -1;
}
static int
spdk_add_nvmf_discovery_subsystem(void)
{
struct spdk_nvmf_subsystem_grp *ss_group;
ss_group = calloc(1, sizeof(*ss_group));
if (!ss_group) {
SPDK_ERRLOG("could not allocate discovery subsystem group\n");
return -1;
}
ss_group->num = 0xffff;
ss_group->name = strdup(SPDK_NVMF_DISCOVERY_NQN);
if (ss_group->name == NULL) {
SPDK_ERRLOG("strdup ss_group->name error\n");
free(ss_group);
return -1;
}
ss_group->subsystem = nvmf_create_subsystem(ss_group->num, ss_group->name, SPDK_NVMF_SUB_DISCOVERY);
if (ss_group->subsystem == NULL) {
SPDK_ERRLOG("Failed creating discovery nvmf library subsystem\n");
free(ss_group);
return -1;
}
TAILQ_INSERT_TAIL(&g_ssg_head, ss_group, tailq);
return 0;
}
void
spdk_format_discovery_log(struct spdk_nvmf_discovery_log_page *disc_log, uint32_t length)
{
int i, numrec = 0;
struct spdk_nvmf_subsystem_grp *ss_group;
struct spdk_nvmf_subsystem *subsystem;
struct spdk_nvmf_access_map *map;
struct spdk_nvmf_port *port;
struct spdk_nvmf_fabric_intf *fabric_intf;
struct spdk_nvmf_discovery_log_page_entry *entry;
TAILQ_FOREACH(ss_group, &g_ssg_head, tailq) {
subsystem = ss_group->subsystem;
if (subsystem->subtype == SPDK_NVMF_SUB_DISCOVERY)
continue;
for (i = 0; i < ss_group->map_count; i++) {
map = &ss_group->map[i];
port = map->port;
if (port != NULL) {
TAILQ_FOREACH(fabric_intf, &port->head, tailq) {
/* include the discovery log entry */
if (length > sizeof(struct spdk_nvmf_discovery_log_page)) {
if (sizeof(struct spdk_nvmf_discovery_log_page) + (numrec + 1) * sizeof(
struct spdk_nvmf_discovery_log_page_entry) > length) {
break;
}
entry = &disc_log->entries[numrec];
entry->trtype = fabric_intf->trtype;
entry->adrfam = fabric_intf->adrfam;
entry->treq = fabric_intf->treq;
entry->portid = port->tag;
/* Dynamic controllers */
entry->cntlid = 0xffff;
entry->subtype = subsystem->subtype;
snprintf(entry->trsvcid, 32, "%s", fabric_intf->sin_port);
snprintf(entry->traddr, 256, "%s", fabric_intf->host);
snprintf(entry->subnqn, 256, "%s", subsystem->subnqn);
entry->tsas.rdma.rdma_qptype = port->rdma.rdma_qptype;
entry->tsas.rdma.rdma_prtype = port->rdma.rdma_prtype;
entry->tsas.rdma.rdma_cms = port->rdma.rdma_cms;
}
numrec++;
}
}
}
}
disc_log->numrec = numrec;
}
int
spdk_initialize_nvmf_subsystems(void)
{
@ -407,7 +490,13 @@ spdk_initialize_nvmf_subsystems(void)
}
sp = spdk_conf_next_section(sp);
}
return 0;
/* Discovery subsystem */
rc = spdk_add_nvmf_discovery_subsystem();
if (rc == 0)
printf(" Discovery Service Enabled\n");
return rc;
}
int

8
lib/nvmf/subsystem_grp.h
View File

@ -59,6 +59,7 @@ struct spdk_nvmf_subsystem {
uint16_t num;
char subnqn[MAX_NQN_SIZE];
int num_sessions;
enum spdk_nvmf_subsystem_types subtype;
TAILQ_HEAD(session_q, nvmf_session) sessions;
struct spdk_nvmf_namespace ns_list_map[MAX_PER_SUBSYSTEM_NAMESPACES];
int ns_count;
@ -73,7 +74,7 @@ struct spdk_nvmf_access_map {
struct spdk_nvmf_subsystem_grp {
int num;
char *name;;
char *name;
struct spdk_nvmf_subsystem *subsystem;
int map_count;
struct spdk_nvmf_access_map map[MAX_PER_SUBSYSTEM_ACCESS_MAP];
@ -81,7 +82,7 @@ struct spdk_nvmf_subsystem_grp {
};
struct spdk_nvmf_subsystem *
nvmf_create_subsystem(int num, char *name);
nvmf_create_subsystem(int num, char *name, enum spdk_nvmf_subsystem_types sub_type);
int
nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem);
@ -99,4 +100,7 @@ spdk_initialize_nvmf_subsystems(void);
int
spdk_shutdown_nvmf_subsystems(void);
void
spdk_format_discovery_log(struct spdk_nvmf_discovery_log_page *disc_log, uint32_t length);
#endif /* _NVMF_SUBSYSTEM_GROUP_H_ */

4
test/lib/nvmf/nvmf_c/nvmf_ut.c
View File

@ -343,7 +343,7 @@ nvmf_test_create_subsystem(void)
char wrong_name[512];
struct spdk_nvmf_subsystem *subsystem;
struct spdk_nvmf_ctrlr *nvmf_ctrlr;
subsystem = nvmf_create_subsystem(1, correct_name);
subsystem = nvmf_create_subsystem(1, correct_name, SPDK_NVMF_SUB_NVME);
SPDK_CU_ASSERT_FATAL(subsystem != NULL);
CU_ASSERT_EQUAL(subsystem->num, 1);
CU_ASSERT_STRING_EQUAL(subsystem->subnqn, correct_name);
@ -353,7 +353,7 @@ nvmf_test_create_subsystem(void)
/* test long name */
memset(wrong_name, 'a', 512);
subsystem = nvmf_create_subsystem(2, wrong_name);
subsystem = nvmf_create_subsystem(2, wrong_name, SPDK_NVMF_SUB_NVME);
SPDK_CU_ASSERT_FATAL(subsystem != NULL);
CU_ASSERT_EQUAL(subsystem->num, 2);
CU_ASSERT_STRING_NOT_EQUAL(subsystem->subnqn, wrong_name);

37
test/nvmf/discovery/discovery.sh Executable file
View File

@ -0,0 +1,37 @@
#!/usr/bin/env bash
testdir=$(readlink -f $(dirname $0))
rootdir=$testdir/../../..
source $rootdir/scripts/autotest_common.sh
source $rootdir/test/nvmf/common.sh
if ! hash nvme; then
echo "nvme command not found; skipping discovery test"
exit 0
fi
rdma_device_init
timing_enter discovery
# Start up the NVMf target in another process
$rootdir/app/nvmf_tgt/nvmf_tgt -c $testdir/../nvmf.conf -t nvmf -t rdma &
nvmfpid=$!
trap "process_core; killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT
sleep 10
modprobe -v nvme-rdma
if [ -e "/dev/nvme-fabrics" ]; then
chmod a+rw /dev/nvme-fabrics
fi
nvme discover -t rdma -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT
trap - SIGINT SIGTERM EXIT
nvmfcleanup
killprocess $nvmfpid
timing_exit discovery