From a1a085cf6bdffee2a5866f223a4dc822e8742eef Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Wed, 19 Oct 2016 12:41:11 -0700 Subject: [PATCH] nvmf: Transports now allocate sessions This makes it consistent with the way connections and requests work. Change-Id: Ifb97499ba72f7dfd02ac54ba1b622726d266262c Signed-off-by: Ben Walker --- lib/nvmf/rdma.c | 37 ++++++++++++++++++++++--------------- lib/nvmf/session.c | 9 +-------- lib/nvmf/session.h | 4 ---- lib/nvmf/transport.h | 6 +++--- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index c7970c124..a65cb3f04 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -132,6 +132,8 @@ struct spdk_nvmf_rdma_conn { static TAILQ_HEAD(, spdk_nvmf_rdma_conn) g_pending_conns = TAILQ_HEAD_INITIALIZER(g_pending_conns); struct spdk_nvmf_rdma_session { + struct spdk_nvmf_session session; + SLIST_HEAD(, spdk_nvmf_rdma_buf) data_buf_pool; struct ibv_context *verbs; @@ -179,6 +181,13 @@ get_rdma_req(struct spdk_nvmf_request *req) req)); } +static inline struct spdk_nvmf_rdma_session * +get_rdma_sess(struct spdk_nvmf_session *sess) +{ + return (struct spdk_nvmf_rdma_session *)((uintptr_t)sess - offsetof(struct spdk_nvmf_rdma_session, + session)); +} + static int nvmf_post_rdma_recv(struct spdk_nvmf_request *req); static void @@ -375,7 +384,7 @@ nvmf_post_rdma_read(struct spdk_nvmf_request *req) rdma_req->sg_list[0].addr = (uintptr_t)req->data; if (req->length > g_rdma.in_capsule_data_size) { - rdma_sess = conn->sess->trctx; + rdma_sess = get_rdma_sess(conn->sess); rdma_req->sg_list[0].lkey = rdma_sess->buf_mr->lkey; } else { rdma_req->sg_list[0].lkey = rdma_conn->bufs_mr->lkey; @@ -410,7 +419,7 @@ nvmf_post_rdma_write(struct spdk_nvmf_request *req) rdma_req->sg_list[0].addr = (uintptr_t)req->data; if (req->length > g_rdma.in_capsule_data_size) { - rdma_sess = conn->sess->trctx; + rdma_sess = get_rdma_sess(conn->sess); rdma_req->sg_list[0].lkey = rdma_sess->buf_mr->lkey; } else { rdma_req->sg_list[0].lkey = rdma_conn->bufs_mr->lkey; @@ -555,7 +564,7 @@ spdk_nvmf_rdma_request_send_completion(struct spdk_nvmf_request *req) if (req->length > g_rdma.in_capsule_data_size) { /* Put the buffer back in the pool */ - rdma_sess = conn->sess->trctx; + rdma_sess = get_rdma_sess(conn->sess); buf = req->data; SLIST_INSERT_HEAD(&rdma_sess->data_buf_pool, buf, link); @@ -828,7 +837,7 @@ spdk_nvmf_request_prep_data(struct spdk_nvmf_request *req) /* TODO: In Capsule Data Size should be tracked per queue (admin, for instance, should always have 4k and no more). */ if (sgl->keyed.length > g_rdma.in_capsule_data_size) { - rdma_sess = req->conn->sess->trctx; + rdma_sess = get_rdma_sess(req->conn->sess); req->data = SLIST_FIRST(&rdma_sess->data_buf_pool); if (!req->data) { /* No available buffers. Queue this request up. */ @@ -898,7 +907,7 @@ spdk_nvmf_rdma_handle_pending_rdma_rw(struct spdk_nvmf_conn *conn) /* First, try to assign free data buffers to requests that need one */ if (conn->sess) { - rdma_sess = conn->sess->trctx; + rdma_sess = get_rdma_sess(conn->sess); TAILQ_FOREACH_SAFE(rdma_req, &rdma_conn->pending_data_buf_queue, link, tmp) { assert(rdma_req->req.data == NULL); rdma_req->req.data = SLIST_FIRST(&rdma_sess->data_buf_pool); @@ -1170,8 +1179,8 @@ spdk_nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr, entry->tsas.rdma.rdma_cms = SPDK_NVMF_RDMA_CMS_RDMA_CM; } -static int -spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session) +static struct spdk_nvmf_session * +spdk_nvmf_rdma_session_init(void) { struct spdk_nvmf_rdma_session *rdma_sess; int i; @@ -1179,7 +1188,7 @@ spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session) rdma_sess = calloc(1, sizeof(*rdma_sess)); if (!rdma_sess) { - return -1; + return NULL; } /* TODO: Make the number of elements in this pool configurable. For now, one full queue @@ -1191,7 +1200,7 @@ spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session) SPDK_ERRLOG("Large buffer pool allocation failed (%d x %d)\n", g_rdma.max_queue_depth, g_rdma.max_io_size); free(rdma_sess); - return -1; + return NULL; } SLIST_INIT(&rdma_sess->data_buf_pool); @@ -1200,16 +1209,15 @@ spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session) SLIST_INSERT_HEAD(&rdma_sess->data_buf_pool, buf, link); } - session->transport = &spdk_nvmf_transport_rdma; - session->trctx = rdma_sess; + rdma_sess->session.transport = &spdk_nvmf_transport_rdma; - return 0; + return &rdma_sess->session; } static void spdk_nvmf_rdma_session_fini(struct spdk_nvmf_session *session) { - struct spdk_nvmf_rdma_session *rdma_sess = session->trctx; + struct spdk_nvmf_rdma_session *rdma_sess = get_rdma_sess(session); if (!rdma_sess) { return; @@ -1218,14 +1226,13 @@ spdk_nvmf_rdma_session_fini(struct spdk_nvmf_session *session) ibv_dereg_mr(rdma_sess->buf_mr); spdk_free(rdma_sess->buf); free(rdma_sess); - session->trctx = NULL; } static int spdk_nvmf_rdma_session_add_conn(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn) { - struct spdk_nvmf_rdma_session *rdma_sess = session->trctx; + struct spdk_nvmf_rdma_session *rdma_sess = get_rdma_sess(session); struct spdk_nvmf_rdma_conn *rdma_conn = get_rdma_conn(conn); if (rdma_sess->verbs != NULL) { diff --git a/lib/nvmf/session.c b/lib/nvmf/session.c index 28fa89ade..32ae3a49c 100644 --- a/lib/nvmf/session.c +++ b/lib/nvmf/session.c @@ -160,7 +160,6 @@ static void session_destruct(struct spdk_nvmf_session *session) { TAILQ_REMOVE(&session->subsys->sessions, session, link); session->transport->session_fini(session); - free(session); } void @@ -246,7 +245,7 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, } /* Establish a new session */ - session = calloc(1, sizeof(struct spdk_nvmf_session)); + session = conn->transport->session_init(); if (session == NULL) { SPDK_ERRLOG("Memory allocation failure\n"); rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; @@ -259,12 +258,6 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, session->num_connections = 0; session->subsys = subsystem; session->max_connections_allowed = g_nvmf_tgt.max_queues_per_session; - if (conn->transport->session_init(session)) { - rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; - conn->transport->session_fini(session); - free(session); - return; - } if (conn->transport->session_add_conn(session, conn)) { rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; diff --git a/lib/nvmf/session.h b/lib/nvmf/session.h index 0b36edd86..6a7d8ab69 100644 --- a/lib/nvmf/session.h +++ b/lib/nvmf/session.h @@ -84,10 +84,6 @@ struct spdk_nvmf_session { uint32_t kato; const struct spdk_nvmf_transport *transport; - /* This is filled in by calling the transport's - * session_init function. */ - void *trctx; - TAILQ_ENTRY(spdk_nvmf_session) link; }; diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 9c0fb601e..c6fe22ae1 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -73,12 +73,12 @@ struct spdk_nvmf_transport { struct spdk_nvmf_discovery_log_page_entry *entry); /** - * Initialize the transport for the given session + * Create a new session */ - int (*session_init)(struct spdk_nvmf_session *session); + struct spdk_nvmf_session *(*session_init)(void); /** - * Deinitiallize the transport for the given session + * Destroy a session */ void (*session_fini)(struct spdk_nvmf_session *session);