diff --git a/app/nvmf_tgt/nvmf_tgt.c b/app/nvmf_tgt/nvmf_tgt.c index d03e2b7c4..8a580e368 100644 --- a/app/nvmf_tgt/nvmf_tgt.c +++ b/app/nvmf_tgt/nvmf_tgt.c @@ -44,7 +44,6 @@ #include "spdk/event.h" -#include "nvmf/conn.h" #include "nvmf/rdma.h" #include "nvmf/port.h" #include "nvmf/host.h" @@ -61,7 +60,7 @@ static void spdk_nvmf_shutdown_cb(void) { nvmf_acceptor_stop(); - spdk_shutdown_nvmf_conns(); + spdk_app_stop(0); fprintf(stdout, "\n=========================\n"); fprintf(stdout, " NVMF shutdown signal\n"); diff --git a/lib/nvmf/conn.c b/lib/nvmf/conn.c index a71d445ff..f6dae51db 100644 --- a/lib/nvmf/conn.c +++ b/lib/nvmf/conn.c @@ -59,84 +59,21 @@ */ -static int g_max_conns; -static struct spdk_nvmf_conn *g_conns_array; - -static pthread_mutex_t g_conns_mutex; - -static struct rte_timer g_shutdown_timer; - static int nvmf_allocate_reactor(uint64_t cpumask); static void spdk_nvmf_conn_do_work(void *arg); -static struct spdk_nvmf_conn * -allocate_conn(void) -{ - struct spdk_nvmf_conn *conn; - int i; - - pthread_mutex_lock(&g_conns_mutex); - for (i = 0; i < g_max_conns; i++) { - conn = &g_conns_array[i]; - if (!conn->is_valid) { - memset(conn, 0, sizeof(*conn)); - conn->is_valid = 1; - pthread_mutex_unlock(&g_conns_mutex); - return conn; - } - } - pthread_mutex_unlock(&g_conns_mutex); - - return NULL; -} - -static void -free_conn(struct spdk_nvmf_conn *conn) -{ - conn->sess = NULL; - conn->is_valid = 0; -} - -int spdk_initialize_nvmf_conns(int max_connections) -{ - int rc; - - rc = pthread_mutex_init(&g_conns_mutex, NULL); - if (rc != 0) { - SPDK_ERRLOG("mutex_init() failed\n"); - return -1; - } - - g_max_conns = max_connections; - g_conns_array = calloc(g_max_conns, sizeof(struct spdk_nvmf_conn)); - - return 0; -} - struct spdk_nvmf_conn * spdk_nvmf_allocate_conn(void) { struct spdk_nvmf_conn *conn; - conn = allocate_conn(); + conn = calloc(1, sizeof(struct spdk_nvmf_conn)); if (conn == NULL) { SPDK_ERRLOG("Could not allocate new connection.\n"); - goto err0; + return NULL; } - /* all new connections initially default as AQ until nvmf connect */ - conn->type = CONN_TYPE_AQ; - - /* no session association until nvmf connect */ - conn->sess = NULL; - - conn->state = CONN_STATE_INVALID; - conn->sq_head = 0; - return conn; - -err0: - return NULL; } /** @@ -175,100 +112,18 @@ spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn) return 0; err0: - free_conn(conn); + free(conn); return -1; } -static void -_conn_destruct(spdk_event_t event) +void +spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn) { - struct spdk_nvmf_conn *conn = spdk_event_get_arg1(event); + spdk_poller_unregister(&conn->poller, NULL); - /* - * Notify NVMf library of the fabric connection - * going away. If this is the AQ connection then - * set state for other connections to abort. - */ nvmf_disconnect(conn->sess, conn); - - if (conn->type == CONN_TYPE_AQ) { - SPDK_TRACELOG(SPDK_TRACE_DEBUG, "AQ connection destruct, trigger session closure\n"); - /* Trigger all I/O connections to shutdown */ - conn->state = CONN_STATE_FABRIC_DISCONNECT; - } - nvmf_rdma_conn_cleanup(conn); - - pthread_mutex_lock(&g_conns_mutex); - free_conn(conn); - pthread_mutex_unlock(&g_conns_mutex); -} - -static void spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn) -{ - struct spdk_event *event; - - SPDK_TRACELOG(SPDK_TRACE_DEBUG, "conn %p\n", conn); - conn->state = CONN_STATE_INVALID; - - event = spdk_event_allocate(rte_lcore_id(), _conn_destruct, conn, NULL, NULL); - spdk_poller_unregister(&conn->poller, event); -} - -static int -spdk_nvmf_get_active_conns(void) -{ - struct spdk_nvmf_conn *conn; - int num = 0; - int i; - - pthread_mutex_lock(&g_conns_mutex); - for (i = 0; i < g_max_conns; i++) { - conn = &g_conns_array[i]; - if (!conn->is_valid) - continue; - num++; - } - pthread_mutex_unlock(&g_conns_mutex); - return num; -} - -static void -spdk_nvmf_cleanup_conns(void) -{ - free(g_conns_array); -} - -static void -spdk_nvmf_conn_check_shutdown(struct rte_timer *timer, void *arg) -{ - if (spdk_nvmf_get_active_conns() == 0) { - RTE_VERIFY(timer == &g_shutdown_timer); - rte_timer_stop(timer); - spdk_nvmf_cleanup_conns(); - spdk_app_stop(0); - } -} - -void spdk_shutdown_nvmf_conns(void) -{ - struct spdk_nvmf_conn *conn; - int i; - - pthread_mutex_lock(&g_conns_mutex); - - for (i = 0; i < g_max_conns; i++) { - conn = &g_conns_array[i]; - if (!conn->is_valid) - continue; - SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Set conn %d state to exiting\n", i); - conn->state = CONN_STATE_EXITING; - } - - pthread_mutex_unlock(&g_conns_mutex); - rte_timer_init(&g_shutdown_timer); - rte_timer_reset(&g_shutdown_timer, rte_get_timer_hz() / 1000, PERIODICAL, - rte_get_master_lcore(), spdk_nvmf_conn_check_shutdown, NULL); + free(conn); } static void diff --git a/lib/nvmf/conn.h b/lib/nvmf/conn.h index 04420b29f..e604f4fc6 100644 --- a/lib/nvmf/conn.h +++ b/lib/nvmf/conn.h @@ -55,8 +55,6 @@ enum conn_type { }; struct spdk_nvmf_conn { - uint32_t is_valid; - struct nvmf_session *sess; uint16_t qid; @@ -72,13 +70,10 @@ struct spdk_nvmf_conn { struct spdk_poller poller; }; -int spdk_initialize_nvmf_conns(int max_connections); - -void spdk_shutdown_nvmf_conns(void); - struct spdk_nvmf_conn * spdk_nvmf_allocate_conn(void); int spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn); +void spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn); #endif /* NVMF_CONN_H */ diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 73e41d1cb..4965b5b3e 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -77,10 +77,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF) #define SPDK_NVMF_DESC_POOL_SIZE(spdk) (SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \ SPDK_NVMF_IOQ_POOL_SIZE(spdk)) -#define SPDK_NVMF_MAX_CONNECTIONS(spdk) (MAX_SUBSYSTEMS * \ - ((spdk)->MaxSessionsPerSubsystem) * \ - ((spdk)->MaxConnectionsPerSession)) - struct spdk_nvmf_globals g_nvmf_tgt; extern struct rte_mempool *request_mempool; @@ -238,11 +234,7 @@ nvmf_tgt_subsystem_initialize(void) SPDK_ERRLOG("spdk_initialize_nvmf_subsystems failed\n"); return rc; } - rc = spdk_initialize_nvmf_conns(SPDK_NVMF_MAX_CONNECTIONS(&g_nvmf_tgt)); - if (rc < 0) { - SPDK_ERRLOG("spdk_initialize_nvmf_conns() failed\n"); - return rc; - } + return rc; } diff --git a/lib/nvmf/session.c b/lib/nvmf/session.c index f89eb0f3c..460584c01 100644 --- a/lib/nvmf/session.c +++ b/lib/nvmf/session.c @@ -195,10 +195,18 @@ nvmf_create_session(const char *subnqn) return session; } -static void -nvmf_delete_session(struct nvmf_session *session) +void +spdk_nvmf_session_destruct(struct nvmf_session *session) { session->subsys->session = NULL; + + while (!TAILQ_EMPTY(&session->connections)) { + struct spdk_nvmf_conn *conn = TAILQ_FIRST(&session->connections); + + TAILQ_REMOVE(&session->connections, conn, link); + spdk_nvmf_conn_destruct(conn); + } + free(session); } @@ -268,15 +276,11 @@ void nvmf_disconnect(struct nvmf_session *session, struct spdk_nvmf_conn *conn) { - if (session) { - if (session->num_connections > 0) { - session->num_connections--; - TAILQ_REMOVE(&session->connections, conn, link); - } + session->num_connections--; + TAILQ_REMOVE(&session->connections, conn, link); - if (session->num_connections == 0) { - nvmf_delete_session(session); - } + if (session->num_connections == 0 || conn->type == CONN_TYPE_AQ) { + spdk_nvmf_session_destruct(session); } } diff --git a/lib/nvmf/session.h b/lib/nvmf/session.h index 2d44dfe66..d1278290d 100644 --- a/lib/nvmf/session.h +++ b/lib/nvmf/session.h @@ -94,4 +94,6 @@ nvmf_check_io_completions(struct nvmf_session *session); void nvmf_check_admin_completions(struct nvmf_session *session); +void spdk_nvmf_session_destruct(struct nvmf_session *session); + #endif diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index fe5f6353f..93f8fa28a 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -101,8 +101,7 @@ nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem) } if (subsystem->session) { - /* TODO: Call a session function that closes all connections */ - free(subsystem->session); + spdk_nvmf_session_destruct(subsystem->session); } TAILQ_REMOVE(&g_subsystems, subsystem, entries);