nvmf: move acceptor poller into nvmf_tgt app

The NVMe over Fabrics target library now exposes a simple function call
that polls the acceptor once, and the application handles registration
of the poller.

Also rename the transport function pointers related to the acceptor so
they better reflect their purpose.

Change-Id: I5fa0d516586bf17e73afeb88ff3c2d5b0d46794d
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>

app/nvmf_tgt/conf.c

@@ -76,6 +76,8 @@ struct spdk_nvmf_probe_ctx {
 #define SPDK_NVMF_CONFIG_MAX_IO_SIZE_MIN 4096
 #define SPDK_NVMF_CONFIG_MAX_IO_SIZE_MAX 131072
 
+struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf;
+
 static int
 spdk_add_nvmf_discovery_subsystem(void)
 {
@@ -146,9 +148,9 @@ spdk_nvmf_parse_nvmf_tgt(void)
 	if (acceptor_lcore < 0) {
 		acceptor_lcore = rte_lcore_id();
 	}
+	g_spdk_nvmf_tgt_conf.acceptor_lcore = acceptor_lcore;
 
-	rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size,
+	rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size);
-			   acceptor_lcore);
 	if (rc != 0) {
 		SPDK_ERRLOG("nvmf_tgt_init() failed\n");
 		return rc;

app/nvmf_tgt/conf.h

@@ -34,6 +34,12 @@
 #ifndef NVMF_CONF_H
 #define NVMF_CONF_H
 
+struct spdk_nvmf_tgt_conf {
+	uint32_t acceptor_lcore;
+};
+
+extern struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf;
+
 int
 spdk_nvmf_parse_conf(void);
 

app/nvmf_tgt/nvmf_tgt.c

@@ -56,15 +56,30 @@ struct rte_mempool *request_mempool;
 #define SPDK_NVMF_BUILD_ETC "/usr/local/etc/nvmf"
 #define SPDK_NVMF_DEFAULT_CONFIG SPDK_NVMF_BUILD_ETC "/nvmf.conf"
 
+#define ACCEPT_TIMEOUT_US		1000 /* 1ms */
+
+static struct spdk_poller *g_acceptor_poller = NULL;
+
+static void
+acceptor_poller_unregistered_event(struct spdk_event *event)
+{
+	spdk_nvmf_acceptor_fini();
+
+	spdk_app_stop(0);
+}
+
 static void
 spdk_nvmf_shutdown_cb(void)
 {
-	spdk_nvmf_acceptor_stop();
+	struct spdk_event *event;
-	spdk_app_stop(0);
 
 	fprintf(stdout, "\n=========================\n");
 	fprintf(stdout, "   NVMF shutdown signal\n");
 	fprintf(stdout, "=========================\n");
+
+	event = spdk_event_allocate(spdk_app_get_current_core(), acceptor_poller_unregistered_event,
+				    NULL, NULL, NULL);
+	spdk_poller_unregister(&g_acceptor_poller, event);
 }
 
 static void
@@ -92,6 +107,12 @@ usage(void)
 	printf(" -d         - disable coredump file enabling\n");
 }
 
+static void
+acceptor_poll(void *arg)
+{
+	spdk_nvmf_acceptor_poll();
+}
+
 static void
 spdk_nvmf_startup(spdk_event_t event)
 {
@@ -109,14 +130,17 @@ spdk_nvmf_startup(spdk_event_t event)
 		goto initialize_error;
 	}
 
-	/* start the rdma poller that will listen
+	rc = spdk_nvmf_acceptor_init();
-	   on all available ports */
-	rc = spdk_nvmf_acceptor_start();
 	if (rc < 0) {
 		SPDK_ERRLOG("spdk_nvmf_acceptor_start() failed\n");
 		goto initialize_error;
 	}
 
+	spdk_poller_register(&g_acceptor_poller, acceptor_poll, NULL,
+			     g_spdk_nvmf_tgt_conf.acceptor_lcore, NULL, ACCEPT_TIMEOUT_US);
+
+	SPDK_NOTICELOG("Acceptor running on core %u\n", g_spdk_nvmf_tgt_conf.acceptor_lcore);
+
 	if (getenv("MEMZONE_DUMP") != NULL) {
 		rte_memzone_dump(stdout);
 		fflush(stdout);

lib/nvmf/nvmf.c

@@ -116,8 +116,7 @@ spdk_nvmf_check_pools(void)
 
 int
 nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess,
-	      uint32_t in_capsule_data_size, uint32_t max_io_size,
+	      uint32_t in_capsule_data_size, uint32_t max_io_size)
-	      uint32_t acceptor_lcore)
 {
 	int rc;
 
@@ -125,13 +124,11 @@ nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess,
 	g_nvmf_tgt.max_queue_depth = max_queue_depth;
 	g_nvmf_tgt.in_capsule_data_size = in_capsule_data_size;
 	g_nvmf_tgt.max_io_size = max_io_size;
-	g_nvmf_tgt.acceptor_lcore = acceptor_lcore;
 
 	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queues Per Session: %d\n", max_queues_per_sess);
 	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queue Depth: %d\n", max_queue_depth);
 	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max In Capsule Data: %d bytes\n", in_capsule_data_size);
 	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max I/O Size: %d bytes\n", max_io_size);
-	SPDK_TRACELOG(SPDK_TRACE_NVMF, "NVMf Acceptor lcore: %d \n", acceptor_lcore);
 
 	/* init nvmf specific config options */
 	if (!g_nvmf_tgt.sin_port) {

lib/nvmf/nvmf_internal.h

@@ -69,14 +69,11 @@ struct spdk_nvmf_globals {
 	uint32_t in_capsule_data_size;
 	uint32_t max_io_size;
 
-	uint32_t acceptor_lcore;
-
 	uint16_t	   sin_port;
 };
 
 int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess,
-		  uint32_t in_capsule_data_size, uint32_t max_io_size,
+		  uint32_t in_capsule_data_size, uint32_t max_io_size);
-		  uint32_t acceptor_lcore);
 
 static inline uint32_t
 nvmf_u32log2(uint32_t x)

lib/nvmf/rdma.c

@@ -52,13 +52,10 @@
 #include "subsystem.h"
 #include "transport.h"
 #include "spdk/assert.h"
-#include "spdk/event.h"
 #include "spdk/log.h"
 #include "spdk/nvmf_spec.h"
 #include "spdk/trace.h"
 
-#define ACCEPT_TIMEOUT_US		1000 /* 1ms */
-
 /*
  RDMA Connection Resouce Defaults
  */
@@ -140,7 +137,6 @@ struct spdk_nvmf_rdma_session {
 };
 
 struct spdk_nvmf_rdma {
-	struct spdk_poller		*acceptor_poller;
 	struct rdma_event_channel	*acceptor_event_channel;
 	struct rdma_cm_id		*acceptor_listen_id;
 
@@ -911,7 +907,7 @@ spdk_nvmf_request_prep_data(struct spdk_nvmf_request *req)
 static int spdk_nvmf_rdma_poll(struct spdk_nvmf_conn *conn);
 
 static void
-nvmf_rdma_accept(void *arg)
+spdk_nvmf_rdma_acceptor_poll(void)
 {
 	struct rdma_cm_event		*event;
 	int				rc;
@@ -976,7 +972,7 @@ nvmf_rdma_accept(void *arg)
 }
 
 static int
-spdk_nvmf_rdma_acceptor_start(void)
+spdk_nvmf_rdma_acceptor_init(void)
 {
 	struct sockaddr_in	addr;
 	uint16_t		sin_port;
@@ -1023,8 +1019,6 @@ spdk_nvmf_rdma_acceptor_start(void)
 	sin_port = ntohs(rdma_get_src_port(g_rdma.acceptor_listen_id));
 	SPDK_NOTICELOG("*** NVMf Target Listening on port %d ***\n", sin_port);
 
-	spdk_poller_register(&g_rdma.acceptor_poller, nvmf_rdma_accept, NULL, g_nvmf_tgt.acceptor_lcore,
-			     NULL, ACCEPT_TIMEOUT_US);
 	return rc;
 
 listen_error:
@@ -1035,10 +1029,8 @@ create_id_error:
 }
 
 static void
-spdk_nvmf_rdma_acceptor_stop(void)
+spdk_nvmf_rdma_acceptor_fini(void)
 {
-	SPDK_TRACELOG(SPDK_TRACE_RDMA, "nvmf_acceptor_stop: shutdown\n");
-	spdk_poller_unregister(&g_rdma.acceptor_poller, NULL);
 }
 
 static int
@@ -1457,8 +1449,10 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = {
 	.name = "rdma",
 	.transport_init = spdk_nvmf_rdma_init,
 	.transport_fini = spdk_nvmf_rdma_fini,
-	.transport_start = spdk_nvmf_rdma_acceptor_start,
+
-	.transport_stop = spdk_nvmf_rdma_acceptor_stop,
+	.acceptor_init = spdk_nvmf_rdma_acceptor_init,
+	.acceptor_poll = spdk_nvmf_rdma_acceptor_poll,
+	.acceptor_fini = spdk_nvmf_rdma_acceptor_fini,
 
 	.session_init = spdk_nvmf_rdma_session_init,
 	.session_fini = spdk_nvmf_rdma_session_fini,

lib/nvmf/transport.c

@@ -85,12 +85,12 @@ spdk_nvmf_transport_fini(void)
 }
 
 int
-spdk_nvmf_acceptor_start(void)
+spdk_nvmf_acceptor_init(void)
 {
 	size_t i;
 
 	for (i = 0; i != NUM_TRANSPORTS; i++) {
-		if (g_transports[i]->transport_start() < 0) {
+		if (g_transports[i]->acceptor_init() < 0) {
 			return -1;
 		}
 	}
@@ -99,12 +99,22 @@ spdk_nvmf_acceptor_start(void)
 }
 
 void
-spdk_nvmf_acceptor_stop(void)
+spdk_nvmf_acceptor_poll(void)
 {
 	size_t i;
 
 	for (i = 0; i != NUM_TRANSPORTS; i++) {
-		g_transports[i]->transport_stop();
+		g_transports[i]->acceptor_poll();
+	}
+}
+
+void
+spdk_nvmf_acceptor_fini(void)
+{
+	size_t i;
+
+	for (i = 0; i != NUM_TRANSPORTS; i++) {
+		g_transports[i]->acceptor_fini();
 	}
 }
 

lib/nvmf/transport.h

@@ -63,12 +63,17 @@ struct spdk_nvmf_transport {
 	/**
 	 * Start accepting connections on the transport.
 	 */
-	int (*transport_start)(void);
+	int (*acceptor_init)(void);
+
+	/**
+	 * Check for new connections on the transport.
+	 */
+	void (*acceptor_poll)(void);
 
 	/**
 	 * Stop accepting connections on the transport.
 	 */
-	void (*transport_stop)(void);
+	void (*acceptor_fini)(void);
 
 	/**
 	 * Initialize the transport for the given session
@@ -115,8 +120,9 @@ int spdk_nvmf_transport_init(void);
 int spdk_nvmf_transport_fini(void);
 const struct spdk_nvmf_transport *spdk_nvmf_transport_get(const char *name);
 
-int spdk_nvmf_acceptor_start(void);
+int spdk_nvmf_acceptor_init(void);
-void spdk_nvmf_acceptor_stop(void);
+void spdk_nvmf_acceptor_poll(void);
+void spdk_nvmf_acceptor_fini(void);
 
 extern const struct spdk_nvmf_transport spdk_nvmf_transport_rdma;