diff --git a/app/nvmf_tgt/conf.c b/app/nvmf_tgt/conf.c index c96c79cd4..4957fbfbc 100644 --- a/app/nvmf_tgt/conf.c +++ b/app/nvmf_tgt/conf.c @@ -76,6 +76,8 @@ struct spdk_nvmf_probe_ctx { #define SPDK_NVMF_CONFIG_MAX_IO_SIZE_MIN 4096 #define SPDK_NVMF_CONFIG_MAX_IO_SIZE_MAX 131072 +struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf; + static int spdk_add_nvmf_discovery_subsystem(void) { @@ -146,9 +148,9 @@ spdk_nvmf_parse_nvmf_tgt(void) if (acceptor_lcore < 0) { acceptor_lcore = rte_lcore_id(); } + g_spdk_nvmf_tgt_conf.acceptor_lcore = acceptor_lcore; - rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size, - acceptor_lcore); + rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size); if (rc != 0) { SPDK_ERRLOG("nvmf_tgt_init() failed\n"); return rc; diff --git a/app/nvmf_tgt/conf.h b/app/nvmf_tgt/conf.h index 712230c68..0d5016e8d 100644 --- a/app/nvmf_tgt/conf.h +++ b/app/nvmf_tgt/conf.h @@ -34,6 +34,12 @@ #ifndef NVMF_CONF_H #define NVMF_CONF_H +struct spdk_nvmf_tgt_conf { + uint32_t acceptor_lcore; +}; + +extern struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf; + int spdk_nvmf_parse_conf(void); diff --git a/app/nvmf_tgt/nvmf_tgt.c b/app/nvmf_tgt/nvmf_tgt.c index ff3f276fb..73e214ad1 100644 --- a/app/nvmf_tgt/nvmf_tgt.c +++ b/app/nvmf_tgt/nvmf_tgt.c @@ -56,15 +56,30 @@ struct rte_mempool *request_mempool; #define SPDK_NVMF_BUILD_ETC "/usr/local/etc/nvmf" #define SPDK_NVMF_DEFAULT_CONFIG SPDK_NVMF_BUILD_ETC "/nvmf.conf" +#define ACCEPT_TIMEOUT_US 1000 /* 1ms */ + +static struct spdk_poller *g_acceptor_poller = NULL; + +static void +acceptor_poller_unregistered_event(struct spdk_event *event) +{ + spdk_nvmf_acceptor_fini(); + + spdk_app_stop(0); +} + static void spdk_nvmf_shutdown_cb(void) { - spdk_nvmf_acceptor_stop(); - spdk_app_stop(0); + struct spdk_event *event; fprintf(stdout, "\n=========================\n"); fprintf(stdout, " NVMF shutdown signal\n"); fprintf(stdout, "=========================\n"); + + event = spdk_event_allocate(spdk_app_get_current_core(), acceptor_poller_unregistered_event, + NULL, NULL, NULL); + spdk_poller_unregister(&g_acceptor_poller, event); } static void @@ -92,6 +107,12 @@ usage(void) printf(" -d - disable coredump file enabling\n"); } +static void +acceptor_poll(void *arg) +{ + spdk_nvmf_acceptor_poll(); +} + static void spdk_nvmf_startup(spdk_event_t event) { @@ -109,14 +130,17 @@ spdk_nvmf_startup(spdk_event_t event) goto initialize_error; } - /* start the rdma poller that will listen - on all available ports */ - rc = spdk_nvmf_acceptor_start(); + rc = spdk_nvmf_acceptor_init(); if (rc < 0) { SPDK_ERRLOG("spdk_nvmf_acceptor_start() failed\n"); goto initialize_error; } + spdk_poller_register(&g_acceptor_poller, acceptor_poll, NULL, + g_spdk_nvmf_tgt_conf.acceptor_lcore, NULL, ACCEPT_TIMEOUT_US); + + SPDK_NOTICELOG("Acceptor running on core %u\n", g_spdk_nvmf_tgt_conf.acceptor_lcore); + if (getenv("MEMZONE_DUMP") != NULL) { rte_memzone_dump(stdout); fflush(stdout); diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 91a03b935..7081d9516 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -116,8 +116,7 @@ spdk_nvmf_check_pools(void) int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, - uint32_t in_capsule_data_size, uint32_t max_io_size, - uint32_t acceptor_lcore) + uint32_t in_capsule_data_size, uint32_t max_io_size) { int rc; @@ -125,13 +124,11 @@ nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, g_nvmf_tgt.max_queue_depth = max_queue_depth; g_nvmf_tgt.in_capsule_data_size = in_capsule_data_size; g_nvmf_tgt.max_io_size = max_io_size; - g_nvmf_tgt.acceptor_lcore = acceptor_lcore; SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queues Per Session: %d\n", max_queues_per_sess); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queue Depth: %d\n", max_queue_depth); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max In Capsule Data: %d bytes\n", in_capsule_data_size); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max I/O Size: %d bytes\n", max_io_size); - SPDK_TRACELOG(SPDK_TRACE_NVMF, "NVMf Acceptor lcore: %d \n", acceptor_lcore); /* init nvmf specific config options */ if (!g_nvmf_tgt.sin_port) { diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index b9d11c193..8a4c95aef 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -69,14 +69,11 @@ struct spdk_nvmf_globals { uint32_t in_capsule_data_size; uint32_t max_io_size; - uint32_t acceptor_lcore; - uint16_t sin_port; }; int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess, - uint32_t in_capsule_data_size, uint32_t max_io_size, - uint32_t acceptor_lcore); + uint32_t in_capsule_data_size, uint32_t max_io_size); static inline uint32_t nvmf_u32log2(uint32_t x) diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 49405c633..1009f32ae 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -52,13 +52,10 @@ #include "subsystem.h" #include "transport.h" #include "spdk/assert.h" -#include "spdk/event.h" #include "spdk/log.h" #include "spdk/nvmf_spec.h" #include "spdk/trace.h" -#define ACCEPT_TIMEOUT_US 1000 /* 1ms */ - /* RDMA Connection Resouce Defaults */ @@ -140,7 +137,6 @@ struct spdk_nvmf_rdma_session { }; struct spdk_nvmf_rdma { - struct spdk_poller *acceptor_poller; struct rdma_event_channel *acceptor_event_channel; struct rdma_cm_id *acceptor_listen_id; @@ -911,7 +907,7 @@ spdk_nvmf_request_prep_data(struct spdk_nvmf_request *req) static int spdk_nvmf_rdma_poll(struct spdk_nvmf_conn *conn); static void -nvmf_rdma_accept(void *arg) +spdk_nvmf_rdma_acceptor_poll(void) { struct rdma_cm_event *event; int rc; @@ -976,7 +972,7 @@ nvmf_rdma_accept(void *arg) } static int -spdk_nvmf_rdma_acceptor_start(void) +spdk_nvmf_rdma_acceptor_init(void) { struct sockaddr_in addr; uint16_t sin_port; @@ -1023,8 +1019,6 @@ spdk_nvmf_rdma_acceptor_start(void) sin_port = ntohs(rdma_get_src_port(g_rdma.acceptor_listen_id)); SPDK_NOTICELOG("*** NVMf Target Listening on port %d ***\n", sin_port); - spdk_poller_register(&g_rdma.acceptor_poller, nvmf_rdma_accept, NULL, g_nvmf_tgt.acceptor_lcore, - NULL, ACCEPT_TIMEOUT_US); return rc; listen_error: @@ -1035,10 +1029,8 @@ create_id_error: } static void -spdk_nvmf_rdma_acceptor_stop(void) +spdk_nvmf_rdma_acceptor_fini(void) { - SPDK_TRACELOG(SPDK_TRACE_RDMA, "nvmf_acceptor_stop: shutdown\n"); - spdk_poller_unregister(&g_rdma.acceptor_poller, NULL); } static int @@ -1457,8 +1449,10 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = { .name = "rdma", .transport_init = spdk_nvmf_rdma_init, .transport_fini = spdk_nvmf_rdma_fini, - .transport_start = spdk_nvmf_rdma_acceptor_start, - .transport_stop = spdk_nvmf_rdma_acceptor_stop, + + .acceptor_init = spdk_nvmf_rdma_acceptor_init, + .acceptor_poll = spdk_nvmf_rdma_acceptor_poll, + .acceptor_fini = spdk_nvmf_rdma_acceptor_fini, .session_init = spdk_nvmf_rdma_session_init, .session_fini = spdk_nvmf_rdma_session_fini, diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index e33b552f4..e1e88e888 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -85,12 +85,12 @@ spdk_nvmf_transport_fini(void) } int -spdk_nvmf_acceptor_start(void) +spdk_nvmf_acceptor_init(void) { size_t i; for (i = 0; i != NUM_TRANSPORTS; i++) { - if (g_transports[i]->transport_start() < 0) { + if (g_transports[i]->acceptor_init() < 0) { return -1; } } @@ -99,12 +99,22 @@ spdk_nvmf_acceptor_start(void) } void -spdk_nvmf_acceptor_stop(void) +spdk_nvmf_acceptor_poll(void) { size_t i; for (i = 0; i != NUM_TRANSPORTS; i++) { - g_transports[i]->transport_stop(); + g_transports[i]->acceptor_poll(); + } +} + +void +spdk_nvmf_acceptor_fini(void) +{ + size_t i; + + for (i = 0; i != NUM_TRANSPORTS; i++) { + g_transports[i]->acceptor_fini(); } } diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 2bc7a108d..48b082806 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -63,12 +63,17 @@ struct spdk_nvmf_transport { /** * Start accepting connections on the transport. */ - int (*transport_start)(void); + int (*acceptor_init)(void); + + /** + * Check for new connections on the transport. + */ + void (*acceptor_poll)(void); /** * Stop accepting connections on the transport. */ - void (*transport_stop)(void); + void (*acceptor_fini)(void); /** * Initialize the transport for the given session @@ -115,8 +120,9 @@ int spdk_nvmf_transport_init(void); int spdk_nvmf_transport_fini(void); const struct spdk_nvmf_transport *spdk_nvmf_transport_get(const char *name); -int spdk_nvmf_acceptor_start(void); -void spdk_nvmf_acceptor_stop(void); +int spdk_nvmf_acceptor_init(void); +void spdk_nvmf_acceptor_poll(void); +void spdk_nvmf_acceptor_fini(void); extern const struct spdk_nvmf_transport spdk_nvmf_transport_rdma;