From 4c6e4d49631f2157b95a2ffe4813ed71fbf89796 Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Tue, 16 Aug 2016 09:35:59 -0700 Subject: [PATCH] nvmf: move acceptor poller into nvmf_tgt app The NVMe over Fabrics target library now exposes a simple function call that polls the acceptor once, and the application handles registration of the poller. Also rename the transport function pointers related to the acceptor so they better reflect their purpose. Change-Id: I5fa0d516586bf17e73afeb88ff3c2d5b0d46794d Signed-off-by: Daniel Verkamp --- app/nvmf_tgt/conf.c | 6 ++++-- app/nvmf_tgt/conf.h | 6 ++++++ app/nvmf_tgt/nvmf_tgt.c | 34 +++++++++++++++++++++++++++++----- lib/nvmf/nvmf.c | 5 +---- lib/nvmf/nvmf_internal.h | 5 +---- lib/nvmf/rdma.c | 20 +++++++------------- lib/nvmf/transport.c | 18 ++++++++++++++---- lib/nvmf/transport.h | 14 ++++++++++---- 8 files changed, 72 insertions(+), 36 deletions(-) diff --git a/app/nvmf_tgt/conf.c b/app/nvmf_tgt/conf.c index c96c79cd4..4957fbfbc 100644 --- a/app/nvmf_tgt/conf.c +++ b/app/nvmf_tgt/conf.c @@ -76,6 +76,8 @@ struct spdk_nvmf_probe_ctx { #define SPDK_NVMF_CONFIG_MAX_IO_SIZE_MIN 4096 #define SPDK_NVMF_CONFIG_MAX_IO_SIZE_MAX 131072 +struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf; + static int spdk_add_nvmf_discovery_subsystem(void) { @@ -146,9 +148,9 @@ spdk_nvmf_parse_nvmf_tgt(void) if (acceptor_lcore < 0) { acceptor_lcore = rte_lcore_id(); } + g_spdk_nvmf_tgt_conf.acceptor_lcore = acceptor_lcore; - rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size, - acceptor_lcore); + rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size); if (rc != 0) { SPDK_ERRLOG("nvmf_tgt_init() failed\n"); return rc; diff --git a/app/nvmf_tgt/conf.h b/app/nvmf_tgt/conf.h index 712230c68..0d5016e8d 100644 --- a/app/nvmf_tgt/conf.h +++ b/app/nvmf_tgt/conf.h @@ -34,6 +34,12 @@ #ifndef NVMF_CONF_H #define NVMF_CONF_H +struct spdk_nvmf_tgt_conf { + uint32_t acceptor_lcore; +}; + +extern struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf; + int spdk_nvmf_parse_conf(void); diff --git a/app/nvmf_tgt/nvmf_tgt.c b/app/nvmf_tgt/nvmf_tgt.c index ff3f276fb..73e214ad1 100644 --- a/app/nvmf_tgt/nvmf_tgt.c +++ b/app/nvmf_tgt/nvmf_tgt.c @@ -56,15 +56,30 @@ struct rte_mempool *request_mempool; #define SPDK_NVMF_BUILD_ETC "/usr/local/etc/nvmf" #define SPDK_NVMF_DEFAULT_CONFIG SPDK_NVMF_BUILD_ETC "/nvmf.conf" +#define ACCEPT_TIMEOUT_US 1000 /* 1ms */ + +static struct spdk_poller *g_acceptor_poller = NULL; + +static void +acceptor_poller_unregistered_event(struct spdk_event *event) +{ + spdk_nvmf_acceptor_fini(); + + spdk_app_stop(0); +} + static void spdk_nvmf_shutdown_cb(void) { - spdk_nvmf_acceptor_stop(); - spdk_app_stop(0); + struct spdk_event *event; fprintf(stdout, "\n=========================\n"); fprintf(stdout, " NVMF shutdown signal\n"); fprintf(stdout, "=========================\n"); + + event = spdk_event_allocate(spdk_app_get_current_core(), acceptor_poller_unregistered_event, + NULL, NULL, NULL); + spdk_poller_unregister(&g_acceptor_poller, event); } static void @@ -92,6 +107,12 @@ usage(void) printf(" -d - disable coredump file enabling\n"); } +static void +acceptor_poll(void *arg) +{ + spdk_nvmf_acceptor_poll(); +} + static void spdk_nvmf_startup(spdk_event_t event) { @@ -109,14 +130,17 @@ spdk_nvmf_startup(spdk_event_t event) goto initialize_error; } - /* start the rdma poller that will listen - on all available ports */ - rc = spdk_nvmf_acceptor_start(); + rc = spdk_nvmf_acceptor_init(); if (rc < 0) { SPDK_ERRLOG("spdk_nvmf_acceptor_start() failed\n"); goto initialize_error; } + spdk_poller_register(&g_acceptor_poller, acceptor_poll, NULL, + g_spdk_nvmf_tgt_conf.acceptor_lcore, NULL, ACCEPT_TIMEOUT_US); + + SPDK_NOTICELOG("Acceptor running on core %u\n", g_spdk_nvmf_tgt_conf.acceptor_lcore); + if (getenv("MEMZONE_DUMP") != NULL) { rte_memzone_dump(stdout); fflush(stdout); diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 91a03b935..7081d9516 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -116,8 +116,7 @@ spdk_nvmf_check_pools(void) int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, - uint32_t in_capsule_data_size, uint32_t max_io_size, - uint32_t acceptor_lcore) + uint32_t in_capsule_data_size, uint32_t max_io_size) { int rc; @@ -125,13 +124,11 @@ nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, g_nvmf_tgt.max_queue_depth = max_queue_depth; g_nvmf_tgt.in_capsule_data_size = in_capsule_data_size; g_nvmf_tgt.max_io_size = max_io_size; - g_nvmf_tgt.acceptor_lcore = acceptor_lcore; SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queues Per Session: %d\n", max_queues_per_sess); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queue Depth: %d\n", max_queue_depth); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max In Capsule Data: %d bytes\n", in_capsule_data_size); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max I/O Size: %d bytes\n", max_io_size); - SPDK_TRACELOG(SPDK_TRACE_NVMF, "NVMf Acceptor lcore: %d \n", acceptor_lcore); /* init nvmf specific config options */ if (!g_nvmf_tgt.sin_port) { diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index b9d11c193..8a4c95aef 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -69,14 +69,11 @@ struct spdk_nvmf_globals { uint32_t in_capsule_data_size; uint32_t max_io_size; - uint32_t acceptor_lcore; - uint16_t sin_port; }; int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess, - uint32_t in_capsule_data_size, uint32_t max_io_size, - uint32_t acceptor_lcore); + uint32_t in_capsule_data_size, uint32_t max_io_size); static inline uint32_t nvmf_u32log2(uint32_t x) diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 49405c633..1009f32ae 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -52,13 +52,10 @@ #include "subsystem.h" #include "transport.h" #include "spdk/assert.h" -#include "spdk/event.h" #include "spdk/log.h" #include "spdk/nvmf_spec.h" #include "spdk/trace.h" -#define ACCEPT_TIMEOUT_US 1000 /* 1ms */ - /* RDMA Connection Resouce Defaults */ @@ -140,7 +137,6 @@ struct spdk_nvmf_rdma_session { }; struct spdk_nvmf_rdma { - struct spdk_poller *acceptor_poller; struct rdma_event_channel *acceptor_event_channel; struct rdma_cm_id *acceptor_listen_id; @@ -911,7 +907,7 @@ spdk_nvmf_request_prep_data(struct spdk_nvmf_request *req) static int spdk_nvmf_rdma_poll(struct spdk_nvmf_conn *conn); static void -nvmf_rdma_accept(void *arg) +spdk_nvmf_rdma_acceptor_poll(void) { struct rdma_cm_event *event; int rc; @@ -976,7 +972,7 @@ nvmf_rdma_accept(void *arg) } static int -spdk_nvmf_rdma_acceptor_start(void) +spdk_nvmf_rdma_acceptor_init(void) { struct sockaddr_in addr; uint16_t sin_port; @@ -1023,8 +1019,6 @@ spdk_nvmf_rdma_acceptor_start(void) sin_port = ntohs(rdma_get_src_port(g_rdma.acceptor_listen_id)); SPDK_NOTICELOG("*** NVMf Target Listening on port %d ***\n", sin_port); - spdk_poller_register(&g_rdma.acceptor_poller, nvmf_rdma_accept, NULL, g_nvmf_tgt.acceptor_lcore, - NULL, ACCEPT_TIMEOUT_US); return rc; listen_error: @@ -1035,10 +1029,8 @@ create_id_error: } static void -spdk_nvmf_rdma_acceptor_stop(void) +spdk_nvmf_rdma_acceptor_fini(void) { - SPDK_TRACELOG(SPDK_TRACE_RDMA, "nvmf_acceptor_stop: shutdown\n"); - spdk_poller_unregister(&g_rdma.acceptor_poller, NULL); } static int @@ -1457,8 +1449,10 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = { .name = "rdma", .transport_init = spdk_nvmf_rdma_init, .transport_fini = spdk_nvmf_rdma_fini, - .transport_start = spdk_nvmf_rdma_acceptor_start, - .transport_stop = spdk_nvmf_rdma_acceptor_stop, + + .acceptor_init = spdk_nvmf_rdma_acceptor_init, + .acceptor_poll = spdk_nvmf_rdma_acceptor_poll, + .acceptor_fini = spdk_nvmf_rdma_acceptor_fini, .session_init = spdk_nvmf_rdma_session_init, .session_fini = spdk_nvmf_rdma_session_fini, diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index e33b552f4..e1e88e888 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -85,12 +85,12 @@ spdk_nvmf_transport_fini(void) } int -spdk_nvmf_acceptor_start(void) +spdk_nvmf_acceptor_init(void) { size_t i; for (i = 0; i != NUM_TRANSPORTS; i++) { - if (g_transports[i]->transport_start() < 0) { + if (g_transports[i]->acceptor_init() < 0) { return -1; } } @@ -99,12 +99,22 @@ spdk_nvmf_acceptor_start(void) } void -spdk_nvmf_acceptor_stop(void) +spdk_nvmf_acceptor_poll(void) { size_t i; for (i = 0; i != NUM_TRANSPORTS; i++) { - g_transports[i]->transport_stop(); + g_transports[i]->acceptor_poll(); + } +} + +void +spdk_nvmf_acceptor_fini(void) +{ + size_t i; + + for (i = 0; i != NUM_TRANSPORTS; i++) { + g_transports[i]->acceptor_fini(); } } diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 2bc7a108d..48b082806 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -63,12 +63,17 @@ struct spdk_nvmf_transport { /** * Start accepting connections on the transport. */ - int (*transport_start)(void); + int (*acceptor_init)(void); + + /** + * Check for new connections on the transport. + */ + void (*acceptor_poll)(void); /** * Stop accepting connections on the transport. */ - void (*transport_stop)(void); + void (*acceptor_fini)(void); /** * Initialize the transport for the given session @@ -115,8 +120,9 @@ int spdk_nvmf_transport_init(void); int spdk_nvmf_transport_fini(void); const struct spdk_nvmf_transport *spdk_nvmf_transport_get(const char *name); -int spdk_nvmf_acceptor_start(void); -void spdk_nvmf_acceptor_stop(void); +int spdk_nvmf_acceptor_init(void); +void spdk_nvmf_acceptor_poll(void); +void spdk_nvmf_acceptor_fini(void); extern const struct spdk_nvmf_transport spdk_nvmf_transport_rdma;