nvmf: Fail when sessions span RNICs

The shared memory pool for a session is associated with
a particular RNIC via the protection domain. New connections
attempting to join a session that came in on a different RNIC
can't use that memory, so must be rejected.

Change-Id: Ibd79fe90566a231f76b7472e5e9b484c3e528454
Signed-off-by: Ben Walker <benjamin.walker@intel.com>

lib/nvmf/rdma.c

@@ -134,6 +134,8 @@ static TAILQ_HEAD(, spdk_nvmf_rdma_conn) g_pending_conns = TAILQ_HEAD_INITIALIZE
 struct spdk_nvmf_rdma_session {
 	SLIST_HEAD(, spdk_nvmf_rdma_buf)	data_buf_pool;
 
+	struct ibv_context			*verbs;
+
 	uint8_t					*buf;
 	struct ibv_mr				*buf_mr;
 };
@@ -1169,10 +1171,9 @@ spdk_nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr,
 }
 
 static int
-spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn)
+spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session)
 {
 	struct spdk_nvmf_rdma_session	*rdma_sess;
-	struct spdk_nvmf_rdma_conn	*rdma_conn = get_rdma_conn(conn);
 	int				i;
 	struct spdk_nvmf_rdma_buf	*buf;
 
@@ -1193,27 +1194,13 @@ spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session, struct spdk_nvmf_
 		return -1;
 	}
 
-	rdma_sess->buf_mr = ibv_reg_mr(rdma_conn->cm_id->pd, rdma_sess->buf,
-				       g_rdma.max_queue_depth * g_rdma.max_io_size,
-				       IBV_ACCESS_LOCAL_WRITE);
-	if (!rdma_sess->buf_mr) {
-		SPDK_ERRLOG("Large buffer pool registration failed (%d x %d)\n",
-			    g_rdma.max_queue_depth, g_rdma.max_io_size);
-		spdk_free(rdma_sess->buf);
-		free(rdma_sess);
-		return -1;
-	}
-
-	SPDK_TRACELOG(SPDK_TRACE_RDMA, "Session Shared Data Pool: %p Length: %x LKey: %x\n",
-		      rdma_sess->buf,  g_rdma.max_queue_depth * g_rdma.max_io_size, rdma_sess->buf_mr->lkey);
-
 	SLIST_INIT(&rdma_sess->data_buf_pool);
 	for (i = 0; i < g_rdma.max_queue_depth; i++) {
 		buf = (struct spdk_nvmf_rdma_buf *)(rdma_sess->buf + (i * g_rdma.max_io_size));
 		SLIST_INSERT_HEAD(&rdma_sess->data_buf_pool, buf, link);
 	}
 
-	session->transport = conn->transport;
+	session->transport = &spdk_nvmf_transport_rdma;
 	session->trctx = rdma_sess;
 
 	return 0;
@@ -1234,6 +1221,48 @@ spdk_nvmf_rdma_session_fini(struct spdk_nvmf_session *session)
 	session->trctx = NULL;
 }
 
+static int
+spdk_nvmf_rdma_session_add_conn(struct spdk_nvmf_session *session,
+				struct spdk_nvmf_conn *conn)
+{
+	struct spdk_nvmf_rdma_session	*rdma_sess = session->trctx;
+	struct spdk_nvmf_rdma_conn	*rdma_conn = get_rdma_conn(conn);
+
+	if (rdma_sess->verbs != NULL) {
+		if (rdma_sess->verbs != rdma_conn->cm_id->verbs) {
+			SPDK_ERRLOG("Two connections belonging to the same session cannot connect using different RDMA devices.\n");
+			return -1;
+		}
+
+		/* Nothing else to do. */
+		return 0;
+	}
+
+	rdma_sess->verbs = rdma_conn->cm_id->verbs;
+	rdma_sess->buf_mr = ibv_reg_mr(rdma_conn->cm_id->pd, rdma_sess->buf,
+				       g_rdma.max_queue_depth * g_rdma.max_io_size,
+				       IBV_ACCESS_LOCAL_WRITE);
+	if (!rdma_sess->buf_mr) {
+		SPDK_ERRLOG("Large buffer pool registration failed (%d x %d)\n",
+			    g_rdma.max_queue_depth, g_rdma.max_io_size);
+		spdk_free(rdma_sess->buf);
+		free(rdma_sess);
+		return -1;
+	}
+
+	SPDK_TRACELOG(SPDK_TRACE_RDMA, "Session Shared Data Pool: %p Length: %x LKey: %x\n",
+		      rdma_sess->buf,  g_rdma.max_queue_depth * g_rdma.max_io_size, rdma_sess->buf_mr->lkey);
+
+	return 0;
+}
+
+static int
+spdk_nvmf_rdma_session_remove_conn(struct spdk_nvmf_session *session,
+				   struct spdk_nvmf_conn *conn)
+{
+	return 0;
+}
+
 static int
 spdk_nvmf_rdma_request_complete(struct spdk_nvmf_request *req)
 {
@@ -1412,6 +1441,8 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = {
 
 	.session_init = spdk_nvmf_rdma_session_init,
 	.session_fini = spdk_nvmf_rdma_session_fini,
+	.session_add_conn = spdk_nvmf_rdma_session_add_conn,
+	.session_remove_conn = spdk_nvmf_rdma_session_remove_conn,
 
 	.req_complete = spdk_nvmf_rdma_request_complete,
 	.req_release = spdk_nvmf_rdma_request_release,

lib/nvmf/session.c

@@ -259,7 +259,14 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn,
 		session->num_connections = 0;
 		session->subsys = subsystem;
 		session->max_connections_allowed = g_nvmf_tgt.max_queues_per_session;
-		if (conn->transport->session_init(session, conn)) {
+		if (conn->transport->session_init(session)) {
+			rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
+			conn->transport->session_fini(session);
+			free(session);
+			return;
+		}
+
+		if (conn->transport->session_add_conn(session, conn)) {
 			rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
 			conn->transport->session_fini(session);
 			free(session);
@@ -319,6 +326,11 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn,
 			rsp->status.sc = SPDK_NVMF_FABRIC_SC_CONTROLLER_BUSY;
 			return;
 		}
+
+		if (conn->transport->session_add_conn(session, conn)) {
+			INVALID_CONNECT_CMD(qid);
+			return;
+		}
 	}
 
 	session->num_connections++;
@@ -339,6 +351,8 @@ spdk_nvmf_session_disconnect(struct spdk_nvmf_conn *conn)
 	assert(session != NULL);
 	session->num_connections--;
 	TAILQ_REMOVE(&session->connections, conn, link);
+
+	conn->transport->session_remove_conn(session, conn);
 	conn->transport->conn_fini(conn);
 
 	if (session->num_connections == 0) {

lib/nvmf/transport.h

@@ -75,13 +75,23 @@ struct spdk_nvmf_transport {
 	/**
 	 * Initialize the transport for the given session
 	 */
-	int (*session_init)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
+	int (*session_init)(struct spdk_nvmf_session *session);
 
 	/**
 	 * Deinitiallize the transport for the given session
 	 */
 	void (*session_fini)(struct spdk_nvmf_session *session);
 
+	/**
+	 * Add a connection to a session
+	 */
+	int (*session_add_conn)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
+
+	/**
+	 * Remove a connection from a session
+	 */
+	int (*session_remove_conn)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);
+
 	/*
 	 * Signal request completion, which sends a response
 	 * to the originator. A request can either