ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf: add transport function pointer table

Browse Source 
Make the core NVMf to transport interface generic and allow for multiple
transport types to be registered.

Change-Id: I0a2767a47d55999c45f788ae1318bb50af60ab4e
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2016-07-14 15:25:23 -07:00
parent 6a1383813c
commit 21c450e187
13 changed files with 252 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/nvmf_tgt/nvmf_tgt.c
View File

@ -44,9 +44,9 @@
#include "spdk/event.h"
#include "nvmf/rdma.h"
#include "nvmf/port.h"
#include "nvmf/host.h"
#include "nvmf/transport.h"
#include "spdk/log.h"
#include "spdk/nvme.h"
@ -59,7 +59,7 @@ struct rte_mempool *request_mempool;
static void
spdk_nvmf_shutdown_cb(void)
{
nvmf_acceptor_stop();
spdk_nvmf_acceptor_stop();
spdk_app_stop(0);
fprintf(stdout, "\n=========================\n");
@ -99,9 +99,9 @@ spdk_nvmf_startup(spdk_event_t event)
/* start the rdma poller that will listen
on all available ports */
rc = nvmf_acceptor_start();
rc = spdk_nvmf_acceptor_start();
if (rc < 0) {
SPDK_ERRLOG("nvmf_acceptor_start() failed\n");
SPDK_ERRLOG("spdk_nvmf_acceptor_start() failed\n");
goto initialize_error;
}

2
lib/nvmf/Makefile
View File

@ -37,7 +37,7 @@ CFLAGS += $(DPDK_INC)
LIBNAME = nvmf
C_SRCS = port.c conn.c controller.c \
host.c subsystem.c conf.c \
nvmf.c request.c session.c
nvmf.c request.c session.c transport.c
C_SRCS-$(CONFIG_RDMA) += rdma.c

8
lib/nvmf/conf.c
View File

@ -45,6 +45,7 @@
#include "nvmf_internal.h"
#include "port.h"
#include "subsystem.h"
#include "transport.h"
#include "spdk/conf.h"
#include "spdk/log.h"
@ -197,12 +198,15 @@ spdk_nvmf_parse_port(struct spdk_conf_section *sp)
/* Loop over the listen addresses and add them to the port */
for (i = 0; ; i++) {
const struct spdk_nvmf_transport *transport;
transport_name = spdk_conf_section_get_nmval(sp, "Listen", i, 0);
if (transport_name == NULL) {
break;
}
if (strcasecmp(transport_name, "RDMA") != 0) {
transport = spdk_nvmf_transport_get(transport_name);
if (transport == NULL) {
SPDK_ERRLOG("Unknown transport type '%s'\n", transport_name);
return -1;
}
@ -216,7 +220,7 @@ spdk_nvmf_parse_port(struct spdk_conf_section *sp)
if (rc < 0) {
continue;
}
fabric_intf = spdk_nvmf_fabric_intf_create(host, listen_port);
fabric_intf = spdk_nvmf_fabric_intf_create(transport, host, listen_port);
if (!fabric_intf) {
continue;
}

4
lib/nvmf/conn.c
View File

@ -47,10 +47,10 @@
#include "spdk/nvmf_spec.h"
#include "conn.h"
#include "rdma.h"
#include "request.h"
#include "session.h"
#include "subsystem.h"
#include "transport.h"
#include "spdk/queue.h"
#include "spdk/log.h"
#include "spdk/trace.h"
@ -64,5 +64,5 @@ void
spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn)
{
nvmf_disconnect(conn->sess, conn);
nvmf_rdma_conn_cleanup(conn);
conn->transport->conn_fini(conn);
}

4
lib/nvmf/conn.h
View File

@ -40,14 +40,16 @@
#include "nvmf_internal.h"
#include "spdk/queue.h"
struct spdk_nvmf_transport;
enum conn_type {
CONN_TYPE_AQ = 0,
CONN_TYPE_IOQ = 1,
};
struct spdk_nvmf_conn {
const struct spdk_nvmf_transport *transport;
struct nvmf_session *sess;
enum conn_type type;
uint16_t sq_head;

10
lib/nvmf/nvmf.c
View File

@ -43,8 +43,8 @@
#include "controller.h"
#include "port.h"
#include "host.h"
#include "rdma.h"
#include "subsystem.h"
#include "transport.h"
#include "spdk/trace.h"
SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF)
@ -165,10 +165,10 @@ nvmf_tgt_subsystem_initialize(void)
}
/* initialize with the NVMf transport */
rc = spdk_nvmf_rdma_init();
rc = spdk_nvmf_transport_init();
if (rc <= 0) {
SPDK_ERRLOG("spdk_nvmf_rdma_init() failed\n");
return rc;
SPDK_ERRLOG("Transport initialization failed\n");
return -1;
}
rc = spdk_add_nvmf_discovery_subsystem();
@ -187,7 +187,7 @@ nvmf_tgt_subsystem_fini(void)
spdk_nvmf_shutdown_nvme();
spdk_nvmf_host_destroy_all();
spdk_nvmf_port_destroy_all();
spdk_nvmf_rdma_fini();
spdk_nvmf_transport_fini();
pthread_mutex_destroy(&g_nvmf_tgt.mutex);

6
lib/nvmf/port.c
View File

@ -35,8 +35,8 @@
#include <rte_debug.h>
#include "conn.h"
#include "rdma.h"
#include "port.h"
#include "transport.h"
#include "spdk/log.h"
#include "spdk/trace.h"
#include "spdk/nvmf_spec.h"
@ -45,7 +45,8 @@ static TAILQ_HEAD(, spdk_nvmf_port) g_port_head = TAILQ_HEAD_INITIALIZER(g_port_
/* Assumes caller allocated host and port strings on the heap */
struct spdk_nvmf_fabric_intf *
spdk_nvmf_fabric_intf_create(char *host, char *sin_port)
spdk_nvmf_fabric_intf_create(const struct spdk_nvmf_transport *transport, char *host,
char *sin_port)
{
struct spdk_nvmf_fabric_intf *fabric_intf = NULL;
@ -63,6 +64,7 @@ spdk_nvmf_fabric_intf_create(char *host, char *sin_port)
fabric_intf->host = host;
fabric_intf->sin_port = sin_port;
fabric_intf->transport = transport;
fabric_intf->trtype = SPDK_NVMF_TRANS_RDMA;
fabric_intf->adrfam = SPDK_NVMF_ADDR_FAMILY_IPV4;
fabric_intf->treq = SPDK_NVMF_TREQ_NOT_SPECIFIED;

6
lib/nvmf/port.h
View File

@ -50,6 +50,8 @@
* ports for the group of aggregated links constitute a single NVM subsystem port.
*/
struct spdk_nvmf_transport;
enum group_state {
GROUP_INIT = 0x0,
GROUP_READY = 0x1,
@ -60,6 +62,7 @@ struct spdk_nvmf_fabric_intf {
char *host;
char *sin_port;
struct spdk_nvmf_port *port;
const struct spdk_nvmf_transport *transport;
enum spdk_nvmf_transport_types trtype;
enum spdk_nvmf_address_family_types adrfam;
enum spdk_nvmf_transport_requirements treq;
@ -76,7 +79,8 @@ struct spdk_nvmf_port {
};
struct spdk_nvmf_fabric_intf *
spdk_nvmf_fabric_intf_create(char *host, char *sin_port);
spdk_nvmf_fabric_intf_create(const struct spdk_nvmf_transport *transport, char *host,
char *sin_port);
void
spdk_nvmf_fabric_intf_destroy(struct spdk_nvmf_fabric_intf *fabric_intf);

52
lib/nvmf/rdma.c
View File

@ -31,8 +31,6 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "rdma.h"
#include <arpa/inet.h>
#include <fcntl.h>
#include <infiniband/verbs.h>
@ -53,6 +51,7 @@
#include "host.h"
#include "session.h"
#include "subsystem.h"
#include "transport.h"
#include "spdk/assert.h"
#include "spdk/log.h"
#include "spdk/trace.h"
@ -315,7 +314,7 @@ nvmf_drain_cq(struct spdk_nvmf_conn *conn)
}
void
static void
nvmf_rdma_conn_cleanup(struct spdk_nvmf_conn *conn)
{
struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn);
@ -382,11 +381,11 @@ nvmf_ibv_send_wr_init(struct ibv_send_wr *wr,
nvmf_trace_ibv_sge(wr->sg_list);
}
int
nvmf_post_rdma_read(struct spdk_nvmf_conn *conn,
struct spdk_nvmf_request *req)
static int
nvmf_post_rdma_read(struct spdk_nvmf_request *req)
{
struct ibv_send_wr wr, *bad_wr = NULL;
struct spdk_nvmf_conn *conn = req->conn;
struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn);
struct spdk_nvmf_rdma_request *rdma_req = get_rdma_req(req);
int rc;
@ -481,10 +480,10 @@ nvmf_post_rdma_send(struct spdk_nvmf_conn *conn,
return (rc);
}
int
spdk_nvmf_rdma_request_complete(struct spdk_nvmf_conn *conn,
struct spdk_nvmf_request *req)
static int
spdk_nvmf_rdma_request_complete(struct spdk_nvmf_request *req)
{
struct spdk_nvmf_conn *conn = req->conn;
struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
int ret;
@ -535,7 +534,7 @@ spdk_nvmf_rdma_request_release(struct spdk_nvmf_conn *conn,
return 0;
}
int
static int
spdk_nvmf_rdma_alloc_reqs(struct spdk_nvmf_conn *conn)
{
struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn);
@ -636,6 +635,7 @@ nvmf_rdma_connect(struct rdma_cm_event *event)
}
conn = &rdma_conn->conn;
conn->transport = &spdk_nvmf_transport_rdma;
/*
* Save the rdma_cm context id in our fabric connection context. This
@ -826,6 +826,12 @@ nvmf_recv(struct spdk_nvmf_rdma_request *rdma_req, struct ibv_wc *wc)
* Pending transfer from host to controller; command will continue
* once transfer is complete.
*/
ret = nvmf_post_rdma_read(req);
if (ret) {
SPDK_ERRLOG("Unable to transfer data from host to controller\n");
return -1;
}
return 0;
}
@ -940,7 +946,8 @@ nvmf_rdma_accept(struct rte_timer *timer, void *arg)
}
}
int nvmf_acceptor_start(void)
static int
spdk_nvmf_rdma_acceptor_start(void)
{
struct sockaddr_in addr;
uint16_t sin_port;
@ -995,7 +1002,8 @@ create_id_error:
return -1;
}
void nvmf_acceptor_stop(void)
static void
spdk_nvmf_rdma_acceptor_stop(void)
{
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "nvmf_acceptor_stop: shutdown\n");
rte_timer_stop_sync(&g_rdma.acceptor_timer);
@ -1006,7 +1014,7 @@ void nvmf_acceptor_stop(void)
Initialize with RDMA transport. Query OFED for device list.
*/
int
static int
spdk_nvmf_rdma_init(void)
{
struct ibv_device **dev_list;
@ -1077,14 +1085,14 @@ spdk_nvmf_rdma_init(void)
return num_devices_found;
}
int
static int
spdk_nvmf_rdma_fini(void)
{
/* Nothing to do */
return 0;
}
int
static int
nvmf_check_rdma_completions(struct spdk_nvmf_conn *conn)
{
struct ibv_wc wc;
@ -1167,4 +1175,18 @@ nvmf_check_rdma_completions(struct spdk_nvmf_conn *conn)
return cq_count;
}
const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = {
.name = "rdma",
.transport_init = spdk_nvmf_rdma_init,
.transport_fini = spdk_nvmf_rdma_fini,
.transport_start = spdk_nvmf_rdma_acceptor_start,
.transport_stop = spdk_nvmf_rdma_acceptor_stop,
.req_complete = spdk_nvmf_rdma_request_complete,
.conn_init = spdk_nvmf_rdma_alloc_reqs,
.conn_fini = nvmf_rdma_conn_cleanup,
.conn_poll = nvmf_check_rdma_completions,
};
SPDK_LOG_REGISTER_TRACE_FLAG("rdma", SPDK_TRACE_RDMA)

17
lib/nvmf/request.c
View File

@ -35,10 +35,10 @@
#include <rte_debug.h>
#include "conn.h"
#include "rdma.h"
#include "request.h"
#include "session.h"
#include "subsystem.h"
#include "transport.h"
#include "spdk/log.h"
#include "spdk/nvme.h"
@ -60,7 +60,7 @@ spdk_nvmf_request_complete(struct spdk_nvmf_request *req)
response->cid, response->cdw0, response->rsvd1, response->sqhd,
*(uint16_t *)&response->status);
if (spdk_nvmf_rdma_request_complete(req->conn, req)) {
if (req->conn->transport->req_complete(req)) {
SPDK_ERRLOG("Transport request completion error!\n");
return -1;
}
@ -401,9 +401,8 @@ nvmf_handle_connect(spdk_event_t event)
spdk_nvmf_session_connect(conn, connect, connect_data, response);
/* Allocate RDMA reqs according to the queue depth and conn type*/
if (spdk_nvmf_rdma_alloc_reqs(conn)) {
SPDK_ERRLOG("Unable to allocate sufficient RDMA work requests\n");
if (conn->transport->conn_init(conn)) {
SPDK_ERRLOG("Transport connection initialization failed\n");
nvmf_disconnect(conn->sess, conn);
req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
spdk_nvmf_request_complete(req);
@ -533,7 +532,6 @@ spdk_nvmf_request_prep_data(struct spdk_nvmf_request *req,
struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
enum spdk_nvme_data_transfer xfer;
int ret;
nvmf_trace_command(req->cmd, conn->type);
@ -614,13 +612,6 @@ spdk_nvmf_request_prep_data(struct spdk_nvmf_request *req,
if (xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {
if (sgl->generic.type == SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK) {
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Initiating Host to Controller data transfer\n");
ret = nvmf_post_rdma_read(conn, req);
if (ret) {
SPDK_ERRLOG("Unable to post rdma read tx descriptor\n");
rsp->status.sc = SPDK_NVME_SC_DATA_TRANSFER_ERROR;
return -1;
}
/* Wait for transfer to complete before executing command. */
return 1;
}

5
lib/nvmf/session.c
View File

@ -36,8 +36,8 @@
#include "session.h"
#include "nvmf_internal.h"
#include "rdma.h"
#include "subsystem.h"
#include "transport.h"
#include "spdk/log.h"
#include "spdk/trace.h"
#include "spdk/nvme_spec.h"
@ -464,7 +464,8 @@ spdk_nvmf_session_poll(struct nvmf_session *session)
struct spdk_nvmf_conn *conn, *tmp;
TAILQ_FOREACH_SAFE(conn, &session->connections, link, tmp) {
if (nvmf_check_rdma_completions(conn) < 0) {
if (conn->transport->conn_poll(conn) < 0) {
SPDK_ERRLOG("Transport poll failed for conn %p; closing connection\n", conn);
nvmf_disconnect(session, conn);
}
}

120
lib/nvmf/transport.c Normal file
View File

@ -0,0 +1,120 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "transport.h"
#include <stdlib.h>
#include <strings.h>
#include "spdk/log.h"
#include "spdk/queue.h"
static const struct spdk_nvmf_transport *const g_transports[] = {
#ifdef SPDK_CONFIG_RDMA
&spdk_nvmf_transport_rdma,
#endif
};
#define NUM_TRANSPORTS (sizeof(g_transports) / sizeof(*g_transports))
int
spdk_nvmf_transport_init(void)
{
size_t i;
int count = 0;
for (i = 0; i != NUM_TRANSPORTS; i++) {
if (g_transports[i]->transport_init() < 0) {
SPDK_NOTICELOG("%s transport init failed\n", g_transports[i]->name);
} else {
count++;
}
}
return count;
}
int
spdk_nvmf_transport_fini(void)
{
size_t i;
int count = 0;
for (i = 0; i != NUM_TRANSPORTS; i++) {
if (g_transports[i]->transport_fini() < 0) {
SPDK_NOTICELOG("%s transport fini failed\n", g_transports[i]->name);
} else {
count++;
}
}
return count;
}
int
spdk_nvmf_acceptor_start(void)
{
size_t i;
for (i = 0; i != NUM_TRANSPORTS; i++) {
if (g_transports[i]->transport_start() < 0) {
return -1;
}
}
return 0;
}
void
spdk_nvmf_acceptor_stop(void)
{
size_t i;
for (i = 0; i != NUM_TRANSPORTS; i++) {
g_transports[i]->transport_stop();
}
}
const struct spdk_nvmf_transport *
spdk_nvmf_transport_get(const char *name)
{
size_t i;
for (i = 0; i != NUM_TRANSPORTS; i++) {
if (strcasecmp(name, g_transports[i]->name) == 0) {
return g_transports[i];
}
}
return NULL;
}

68
lib/nvmf/rdma.h → lib/nvmf/transport.h
View File

@ -31,26 +31,66 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _NVMF_RDMA_H_
#define _NVMF_RDMA_H_
#ifndef SPDK_NVMF_TRANSPORT_H
#define SPDK_NVMF_TRANSPORT_H
struct spdk_nvmf_conn;
struct spdk_nvmf_request;
int spdk_nvmf_rdma_init(void);
int spdk_nvmf_rdma_fini(void);
struct spdk_nvmf_transport {
/**
* Name of the transport.
*/
const char *name;
int nvmf_post_rdma_read(struct spdk_nvmf_conn *conn,
struct spdk_nvmf_request *req);
int spdk_nvmf_rdma_request_complete(struct spdk_nvmf_conn *conn,
struct spdk_nvmf_request *req);
/**
* Initialize the transport.
*/
int (*transport_init)(void);
int spdk_nvmf_rdma_alloc_reqs(struct spdk_nvmf_conn *conn);
void nvmf_rdma_conn_cleanup(struct spdk_nvmf_conn *conn);
/**
* Shut down the transport.
*/
int (*transport_fini)(void);
int nvmf_acceptor_start(void);
void nvmf_acceptor_stop(void);
/**
* Start accepting connections on the transport.
*/
int (*transport_start)(void);
int nvmf_check_rdma_completions(struct spdk_nvmf_conn *conn);
/**
* Stop accepting connections on the transport.
*/
void (*transport_stop)(void);
#endif /* _NVMF_RDMA_H_ */
/*
* Signal request completion.
*/
int (*req_complete)(struct spdk_nvmf_request *req);
/*
* Initialize resources for a new connection.
*/
int (*conn_init)(struct spdk_nvmf_conn *conn);
/*
* Deinitialize a connection.
*/
void (*conn_fini)(struct spdk_nvmf_conn *conn);
/*
* Poll a connection for events.
*/
int (*conn_poll)(struct spdk_nvmf_conn *conn);
};
int spdk_nvmf_transport_init(void);
int spdk_nvmf_transport_fini(void);
const struct spdk_nvmf_transport *spdk_nvmf_transport_get(const char *name);
int spdk_nvmf_acceptor_start(void);
void spdk_nvmf_acceptor_stop(void);
extern const struct spdk_nvmf_transport spdk_nvmf_transport_rdma;
#endif /* SPDK_NVMF_TRANSPORT_H */