nvmf: Assign each subsystem to an lcore

Each subsystem will run on a single core, which is more than enough
to fully saturate a device and a NIC. For now, all subsystems
run on the master lcore.

Change-Id: I95340a262d70fd346fa81fe519e7d4190a369e64
Signed-off-by: Ben Walker <benjamin.walker@intel.com>

lib/nvmf/conf.c

@@ -35,6 +35,9 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <rte_config.h>
+#include <rte_lcore.h>
+
 #include "conf.h"
 #include "controller.h"
 #include "host.h"
@@ -427,7 +430,7 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
 		return -1;
 	}
 
-	subsystem = nvmf_create_subsystem(sp->num, nqn, SPDK_NVMF_SUB_NVME);
+	subsystem = nvmf_create_subsystem(sp->num, nqn, SPDK_NVMF_SUB_NVME, rte_get_master_lcore());
 	if (subsystem == NULL) {
 		return -1;
 	}

lib/nvmf/conn.c

@@ -50,6 +50,7 @@
 #include "rdma.h"
 #include "request.h"
 #include "session.h"
+#include "subsystem.h"
 #include "spdk/queue.h"
 #include "spdk/log.h"
 #include "spdk/trace.h"
@@ -59,41 +60,16 @@
 
 */
 
-static int nvmf_allocate_reactor(uint64_t cpumask);
 static void spdk_nvmf_conn_do_work(void *arg);
 
-/**
-
-\brief Create an NVMf fabric connection from the given parameters and schedule it
-       on a reactor thread.
-
-\code
-
-# identify reactor where the new connections work item will be scheduled
-reactor = nvmf_allocate_reactor()
-schedule fabric connection work item on reactor
-
-\endcode
-
-*/
 int
 spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn)
 {
-	int lcore;
-	uint64_t nvmf_session_core = spdk_app_get_core_mask();
-
-	lcore = nvmf_allocate_reactor(nvmf_session_core);
-	if (lcore < 0) {
-		SPDK_ERRLOG("Unable to find core to launch connection.\n");
-		return -1;
-	}
-
 	conn->state = CONN_STATE_RUNNING;
-	SPDK_NOTICELOG("Launching nvmf connection on core: %d\n", lcore);
 	conn->poller.fn = spdk_nvmf_conn_do_work;
 	conn->poller.arg = conn;
 
-	spdk_poller_register(&conn->poller, lcore, NULL);
+	spdk_poller_register(&conn->poller, conn->sess->subsys->lcore, NULL);
 
 	return 0;
 }
@@ -136,9 +112,3 @@ spdk_nvmf_conn_do_work(void *arg)
 		}
 	}
 }
-
-static int
-nvmf_allocate_reactor(uint64_t cpumask)
-{
-	return rte_get_master_lcore();
-}

lib/nvmf/request.c

@@ -398,6 +398,7 @@ nvmf_handle_connect(spdk_event_t event)
 			req->data;
 	struct spdk_nvmf_fabric_connect_rsp *response = &req->rsp->connect_rsp;
 	struct spdk_nvmf_conn *conn = req->conn;
+	int rc;
 
 	spdk_nvmf_session_connect(conn, connect, connect_data, response);
 
@@ -410,6 +411,15 @@ nvmf_handle_connect(spdk_event_t event)
 		return;
 	}
 
+	/* Start the connection poller */
+	rc = spdk_nvmf_startup_conn(conn);
+	if (rc) {
+		SPDK_ERRLOG("Unable to start connection poller\n");
+		req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
+		spdk_nvmf_request_complete(req);
+		return;
+	}
+
 	SPDK_TRACELOG(SPDK_TRACE_NVMF, "connect capsule response: cntlid = 0x%04x\n",
 		      response->status_code_specific.success.cntlid);
 
@@ -421,7 +431,6 @@ static bool
 nvmf_process_connect(struct spdk_nvmf_request *req)
 {
 	struct spdk_nvmf_conn *conn = req->conn;
-	int rc;
 	spdk_event_t event;
 
 	if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) {
@@ -430,14 +439,6 @@ nvmf_process_connect(struct spdk_nvmf_request *req)
 		return true;
 	}
 
-	/* Start the connection poller */
-	rc = spdk_nvmf_startup_conn(conn);
-	if (rc) {
-		req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
-		SPDK_ERRLOG("Unable to start connection poller\n");
-		return true;
-	}
-
 	/* Pass an event to the lcore that owns this connection */
 	event = spdk_event_allocate(conn->poller.lcore, nvmf_handle_connect, req, NULL, NULL);
 	spdk_event_call(event);

lib/nvmf/subsystem.c

@@ -67,7 +67,9 @@ nvmf_find_subsystem(const char *subnqn)
 }
 
 struct spdk_nvmf_subsystem *
-nvmf_create_subsystem(int num, const char *name, enum spdk_nvmf_subsystem_types sub_type)
+nvmf_create_subsystem(int num, const char *name,
+		      enum spdk_nvmf_subsystem_types sub_type,
+		      uint32_t lcore)
 {
 	struct spdk_nvmf_subsystem	*subsystem;
 
@@ -81,6 +83,7 @@ nvmf_create_subsystem(int num, const char *name, enum spdk_nvmf_subsystem_types
 	subsystem->num = num;
 	subsystem->subtype = sub_type;
 	snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", name);
+	subsystem->lcore = lcore;
 
 	TAILQ_INSERT_HEAD(&g_subsystems, subsystem, entries);
 
@@ -175,7 +178,7 @@ spdk_add_nvmf_discovery_subsystem(void)
 		return -1;
 	}
 
-	subsystem = nvmf_create_subsystem(0, name, SPDK_NVMF_SUB_DISCOVERY);
+	subsystem = nvmf_create_subsystem(0, name, SPDK_NVMF_SUB_DISCOVERY, rte_get_master_lcore());
 	if (subsystem == NULL) {
 		SPDK_ERRLOG("Failed creating discovery nvmf library subsystem\n");
 		free(name);

lib/nvmf/subsystem.h

@@ -59,6 +59,7 @@ struct spdk_nvmf_subsystem {
 	struct nvmf_session *session;
 	struct spdk_nvme_ctrlr *ctrlr;
 	struct spdk_nvme_qpair *io_qpair;
+	uint32_t		lcore;
 
 	int map_count;
 	struct spdk_nvmf_access_map map[MAX_PER_SUBSYSTEM_ACCESS_MAP];
@@ -67,7 +68,9 @@ struct spdk_nvmf_subsystem {
 };
 
 struct spdk_nvmf_subsystem *
-nvmf_create_subsystem(int num, const char *name, enum spdk_nvmf_subsystem_types sub_type);
+nvmf_create_subsystem(int num, const char *name,
+		      enum spdk_nvmf_subsystem_types sub_type,
+		      uint32_t lcore);
 
 int
 nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem);