ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvmf: Move poller from connection to subsystem

Browse Source 
Change-Id: Iea6be8156152367c0fd48b8cee8e2ca1e67f340a
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Ben Walker 2016-07-15 11:34:11 -07:00
parent 96a54158bd
commit d6a499fec2
8 changed files with 104 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
lib/nvmf/conn.c
View File

@ -60,55 +60,9 @@
*/
static void spdk_nvmf_conn_do_work(void *arg);
int
spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn)
{
conn->state = CONN_STATE_RUNNING;
conn->poller.fn = spdk_nvmf_conn_do_work;
conn->poller.arg = conn;
spdk_poller_register(&conn->poller, conn->sess->subsys->lcore, NULL);
return 0;
}
void
spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn)
{
spdk_poller_unregister(&conn->poller, NULL);
nvmf_disconnect(conn->sess, conn);
nvmf_rdma_conn_cleanup(conn);
}
static void
spdk_nvmf_conn_do_work(void *arg)
{
struct spdk_nvmf_conn *conn = arg;
struct nvmf_session *session = conn->sess;
/* process pending NVMe device completions */
if (session) {
if (conn->type == CONN_TYPE_AQ) {
nvmf_check_admin_completions(session);
} else {
nvmf_check_io_completions(session);
}
}
/* process pending RDMA completions */
if (nvmf_check_rdma_completions(conn) < 0) {
SPDK_ERRLOG("Transport poll failed for conn %p; closing connection\n", conn);
conn->state = CONN_STATE_EXITING;
}
if (conn->state == CONN_STATE_EXITING ||
conn->state == CONN_STATE_FABRIC_DISCONNECT) {
spdk_nvmf_conn_destruct(conn);
if (session && (session->num_connections == 0)) {
spdk_nvmf_session_destruct(session);
}
}
}

10
lib/nvmf/conn.h
View File

@ -40,14 +40,6 @@
#include "nvmf_internal.h"
#include "spdk/queue.h"
/* RDMA transport connection states */
enum conn_state {
CONN_STATE_INVALID = 0,
CONN_STATE_RUNNING = 1,
CONN_STATE_FABRIC_DISCONNECT = 2,
CONN_STATE_EXITING = 4,
};
enum conn_type {
CONN_TYPE_AQ = 0,
CONN_TYPE_IOQ = 1,
@ -57,12 +49,10 @@ struct spdk_nvmf_conn {
struct nvmf_session *sess;
enum conn_type type;
volatile enum conn_state state;
uint16_t sq_head;
TAILQ_ENTRY(spdk_nvmf_conn) link;
struct spdk_poller poller;
};
int spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn);

54
lib/nvmf/rdma.c
View File

@ -51,6 +51,8 @@
#include "request.h"
#include "port.h"
#include "host.h"
#include "session.h"
#include "subsystem.h"
#include "spdk/assert.h"
#include "spdk/log.h"
#include "spdk/trace.h"
@ -719,36 +721,52 @@ err0:
return -1;
}
static int
nvmf_rdma_disconnect(struct rdma_cm_event *event)
static void
spdk_nvmf_handle_disconnect(spdk_event_t event)
{
struct nvmf_session *session = spdk_event_get_arg1(event);
struct spdk_nvmf_conn *conn = spdk_event_get_arg2(event);
nvmf_disconnect(session, conn);
}
static int
nvmf_rdma_disconnect(struct rdma_cm_event *evt)
{
struct rdma_cm_id *conn_id;
struct spdk_nvmf_conn *conn;
struct nvmf_session *session;
struct spdk_nvmf_rdma_conn *rdma_conn;
spdk_event_t event;
/* Check to make sure we know about this rdma device */
if (event->id == NULL) {
if (evt->id == NULL) {
SPDK_ERRLOG("disconnect request: missing cm_id\n");
goto err0;
return -1;
}
conn_id = event->id;
conn = conn_id->context;
conn = evt->id->context;
if (conn == NULL) {
SPDK_ERRLOG("disconnect request: no active connection\n");
goto err0;
return -1;
}
/*
* Modify connection state to trigger async termination
* next time the connection poller executes
*/
conn->state = CONN_STATE_FABRIC_DISCONNECT;
rdma_conn = get_rdma_conn(conn);
session = conn->sess;
if (session == NULL) {
/* No session has been established yet. That means the conn
* must be in the pending connections list. Remove it. */
TAILQ_REMOVE(&g_pending_conns, rdma_conn, link);
nvmf_rdma_conn_cleanup(conn);
return 0;
}
/* Pass an event to the core that owns this connection */
event = spdk_event_allocate(session->subsys->poller.lcore,
spdk_nvmf_handle_disconnect,
session, conn, NULL);
spdk_event_call(event);
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "rdma connection %p state set to CONN_STATE_FABRIC_DISCONNECT\n",
conn);
return 0;
err0:
return -1;
}
#ifdef DEBUG

40
lib/nvmf/request.c
View File

@ -398,23 +398,13 @@ nvmf_handle_connect(spdk_event_t event)
req->data;
struct spdk_nvmf_fabric_connect_rsp *response = &req->rsp->connect_rsp;
struct spdk_nvmf_conn *conn = req->conn;
int rc;
spdk_nvmf_session_connect(conn, connect, connect_data, response);
/* Allocate RDMA reqs according to the queue depth and conn type*/
if (spdk_nvmf_rdma_alloc_reqs(conn)) {
SPDK_ERRLOG("Unable to allocate sufficient RDMA work requests\n");
/* TODO: Needs to shutdown poller */
req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
spdk_nvmf_request_complete(req);
return;
}
/* Start the connection poller */
rc = spdk_nvmf_startup_conn(conn);
if (rc) {
SPDK_ERRLOG("Unable to start connection poller\n");
nvmf_disconnect(conn->sess, conn);
req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
spdk_nvmf_request_complete(req);
return;
@ -427,11 +417,25 @@ nvmf_handle_connect(spdk_event_t event)
return;
}
static void
invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp, uint8_t iattr, uint16_t ipo)
{
rsp->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
rsp->status.sc = SPDK_NVMF_FABRIC_SC_INVALID_PARAM;
rsp->status_code_specific.invalid.iattr = iattr;
rsp->status_code_specific.invalid.ipo = ipo;
}
static bool
nvmf_process_connect(struct spdk_nvmf_request *req)
{
struct spdk_nvmf_conn *conn = req->conn;
struct spdk_nvmf_subsystem *subsystem;
spdk_event_t event;
struct spdk_nvmf_fabric_connect_data *data = (struct spdk_nvmf_fabric_connect_data *)
req->data;
struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
#define INVALID_CONNECT_DATA(field) invalid_connect_response(rsp, 1, offsetof(struct spdk_nvmf_fabric_connect_data, field))
if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) {
SPDK_ERRLOG("Connect command data length 0x%x too small\n", req->length);
@ -439,8 +443,16 @@ nvmf_process_connect(struct spdk_nvmf_request *req)
return true;
}
/* Pass an event to the lcore that owns this connection */
event = spdk_event_allocate(conn->poller.lcore, nvmf_handle_connect, req, NULL, NULL);
/* Look up the requested subsystem */
subsystem = nvmf_find_subsystem(data->subnqn);
if (subsystem == NULL) {
SPDK_ERRLOG("Could not find subsystem '%s'\n", data->subnqn);
INVALID_CONNECT_DATA(subnqn);
return true;
}
/* Pass an event to the lcore that owns this subsystem */
event = spdk_event_allocate(subsystem->poller.lcore, nvmf_handle_connect, req, NULL, NULL);
spdk_event_call(event);
return false;

22
lib/nvmf/session.c
View File

@ -36,6 +36,7 @@
#include "session.h"
#include "nvmf_internal.h"
#include "rdma.h"
#include "subsystem.h"
#include "spdk/log.h"
#include "spdk/trace.h"
@ -457,17 +458,16 @@ nvmf_property_set(struct nvmf_session *session,
}
}
void
nvmf_check_admin_completions(struct nvmf_session *session)
int
spdk_nvmf_session_poll(struct nvmf_session *session)
{
/* Discovery subsystem won't have a real NVMe controller, so check ctrlr first */
if (session->subsys->ctrlr) {
spdk_nvme_ctrlr_process_admin_completions(session->subsys->ctrlr);
}
}
struct spdk_nvmf_conn *conn, *tmp;
void
nvmf_check_io_completions(struct nvmf_session *session)
{
spdk_nvme_qpair_process_completions(session->subsys->io_qpair, 0);
TAILQ_FOREACH_SAFE(conn, &session->connections, link, tmp) {
if (nvmf_check_rdma_completions(conn) < 0) {
nvmf_disconnect(session, conn);
}
}
return 0;
}

6
lib/nvmf/session.h
View File

@ -87,11 +87,7 @@ nvmf_property_set(struct nvmf_session *session,
struct spdk_nvmf_fabric_prop_set_rsp *response,
bool *shutdown);
void
nvmf_check_io_completions(struct nvmf_session *session);
void
nvmf_check_admin_completions(struct nvmf_session *session);
int spdk_nvmf_session_poll(struct nvmf_session *session);
void spdk_nvmf_session_destruct(struct nvmf_session *session);

26
lib/nvmf/subsystem.c
View File

@ -66,6 +66,27 @@ nvmf_find_subsystem(const char *subnqn)
return NULL;
}
static void
spdk_nvmf_subsystem_poller(void *arg)
{
struct spdk_nvmf_subsystem *subsystem = arg;
struct nvmf_session *session = subsystem->session;
if (!session) {
/* No active connections, so just return */
return;
}
/* For NVMe subsystems, check the backing physical device for completions. */
if (subsystem->subtype == SPDK_NVMF_SUB_NVME) {
spdk_nvme_ctrlr_process_admin_completions(subsystem->ctrlr);
spdk_nvme_qpair_process_completions(subsystem->io_qpair, 0);
}
/* For each connection in the session, check for RDMA completions */
spdk_nvmf_session_poll(session);
}
struct spdk_nvmf_subsystem *
nvmf_create_subsystem(int num, const char *name,
enum spdk_nvmf_subsystem_types sub_type,
@ -83,7 +104,10 @@ nvmf_create_subsystem(int num, const char *name,
subsystem->num = num;
subsystem->subtype = sub_type;
snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", name);
subsystem->lcore = lcore;
subsystem->poller.fn = spdk_nvmf_subsystem_poller;
subsystem->poller.arg = subsystem;
spdk_poller_register(&subsystem->poller, lcore, NULL);
TAILQ_INSERT_HEAD(&g_subsystems, subsystem, entries);

4
lib/nvmf/subsystem.h
View File

@ -34,6 +34,7 @@
#ifndef SPDK_NVMF_SUBSYSTEM_H
#define SPDK_NVMF_SUBSYSTEM_H
#include "spdk/event.h"
#include "spdk/nvme.h"
#include "spdk/queue.h"
@ -59,7 +60,8 @@ struct spdk_nvmf_subsystem {
struct nvmf_session *session;
struct spdk_nvme_ctrlr *ctrlr;
struct spdk_nvme_qpair *io_qpair;
uint32_t lcore;
struct spdk_poller poller;
int map_count;
struct spdk_nvmf_access_map map[MAX_PER_SUBSYSTEM_ACCESS_MAP];