nvmf: simplify session and connection cleanup

The whole cleanup process is now started by
spdk_shutdown_nvmf_subsystems().  Each subsystem will clean up its
session, if any, and each session will clean up its connections.

Change-Id: I9915d4547751ed4ffc4baa2c45c628698dd0b881
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>

app/nvmf_tgt/nvmf_tgt.c

@@ -44,7 +44,6 @@
 
 #include "spdk/event.h"
 
-#include "nvmf/conn.h"
 #include "nvmf/rdma.h"
 #include "nvmf/port.h"
 #include "nvmf/host.h"
@@ -61,7 +60,7 @@ static void
 spdk_nvmf_shutdown_cb(void)
 {
 	nvmf_acceptor_stop();
-	spdk_shutdown_nvmf_conns();
+	spdk_app_stop(0);
 
 	fprintf(stdout, "\n=========================\n");
 	fprintf(stdout, "   NVMF shutdown signal\n");

lib/nvmf/conn.c

@@ -59,84 +59,21 @@
 
 */
 
-static int g_max_conns;
-static struct spdk_nvmf_conn *g_conns_array;
-
-static pthread_mutex_t g_conns_mutex;
-
-static struct rte_timer g_shutdown_timer;
-
 static int nvmf_allocate_reactor(uint64_t cpumask);
 static void spdk_nvmf_conn_do_work(void *arg);
 
-static struct spdk_nvmf_conn *
-allocate_conn(void)
-{
-	struct spdk_nvmf_conn	*conn;
-	int				i;
-
-	pthread_mutex_lock(&g_conns_mutex);
-	for (i = 0; i < g_max_conns; i++) {
-		conn = &g_conns_array[i];
-		if (!conn->is_valid) {
-			memset(conn, 0, sizeof(*conn));
-			conn->is_valid = 1;
-			pthread_mutex_unlock(&g_conns_mutex);
-			return conn;
-		}
-	}
-	pthread_mutex_unlock(&g_conns_mutex);
-
-	return NULL;
-}
-
-static void
-free_conn(struct spdk_nvmf_conn *conn)
-{
-	conn->sess = NULL;
-	conn->is_valid = 0;
-}
-
-int spdk_initialize_nvmf_conns(int max_connections)
-{
-	int rc;
-
-	rc = pthread_mutex_init(&g_conns_mutex, NULL);
-	if (rc != 0) {
-		SPDK_ERRLOG("mutex_init() failed\n");
-		return -1;
-	}
-
-	g_max_conns = max_connections;
-	g_conns_array = calloc(g_max_conns, sizeof(struct spdk_nvmf_conn));
-
-	return 0;
-}
-
 struct spdk_nvmf_conn *
 spdk_nvmf_allocate_conn(void)
 {
 	struct spdk_nvmf_conn *conn;
 
-	conn = allocate_conn();
+	conn = calloc(1, sizeof(struct spdk_nvmf_conn));
 	if (conn == NULL) {
 		SPDK_ERRLOG("Could not allocate new connection.\n");
-		goto err0;
+		return NULL;
 	}
 
-	/* all new connections initially default as AQ until nvmf connect */
-	conn->type = CONN_TYPE_AQ;
-
-	/* no session association until nvmf connect */
-	conn->sess = NULL;
-
-	conn->state = CONN_STATE_INVALID;
-	conn->sq_head = 0;
-
 	return conn;
-
-err0:
-	return NULL;
 }
 
 /**
@@ -175,100 +112,18 @@ spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn)
 
 	return 0;
 err0:
-	free_conn(conn);
+	free(conn);
 	return -1;
 }
 
-static void
-_conn_destruct(spdk_event_t event)
+void
+spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn)
 {
-	struct spdk_nvmf_conn *conn = spdk_event_get_arg1(event);
+	spdk_poller_unregister(&conn->poller, NULL);
 
-	/*
-	 * Notify NVMf library of the fabric connection
-	 * going away.  If this is the AQ connection then
-	 * set state for other connections to abort.
-	 */
 	nvmf_disconnect(conn->sess, conn);
-
-	if (conn->type == CONN_TYPE_AQ) {
-		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "AQ connection destruct, trigger session closure\n");
-		/* Trigger all I/O connections to shutdown */
-		conn->state = CONN_STATE_FABRIC_DISCONNECT;
-	}
-
 	nvmf_rdma_conn_cleanup(conn);
-
-	pthread_mutex_lock(&g_conns_mutex);
-	free_conn(conn);
-	pthread_mutex_unlock(&g_conns_mutex);
-}
-
-static void spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn)
-{
-	struct spdk_event *event;
-
-	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "conn %p\n", conn);
-	conn->state = CONN_STATE_INVALID;
-
-	event = spdk_event_allocate(rte_lcore_id(), _conn_destruct, conn, NULL, NULL);
-	spdk_poller_unregister(&conn->poller, event);
-}
-
-static int
-spdk_nvmf_get_active_conns(void)
-{
-	struct spdk_nvmf_conn *conn;
-	int num = 0;
-	int i;
-
-	pthread_mutex_lock(&g_conns_mutex);
-	for (i = 0; i < g_max_conns; i++) {
-		conn = &g_conns_array[i];
-		if (!conn->is_valid)
-			continue;
-		num++;
-	}
-	pthread_mutex_unlock(&g_conns_mutex);
-	return num;
-}
-
-static void
-spdk_nvmf_cleanup_conns(void)
-{
-	free(g_conns_array);
-}
-
-static void
-spdk_nvmf_conn_check_shutdown(struct rte_timer *timer, void *arg)
-{
-	if (spdk_nvmf_get_active_conns() == 0) {
-		RTE_VERIFY(timer == &g_shutdown_timer);
-		rte_timer_stop(timer);
-		spdk_nvmf_cleanup_conns();
-		spdk_app_stop(0);
-	}
-}
-
-void spdk_shutdown_nvmf_conns(void)
-{
-	struct spdk_nvmf_conn	*conn;
-	int				i;
-
-	pthread_mutex_lock(&g_conns_mutex);
-
-	for (i = 0; i < g_max_conns; i++) {
-		conn = &g_conns_array[i];
-		if (!conn->is_valid)
-			continue;
-		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Set conn %d state to exiting\n", i);
-		conn->state = CONN_STATE_EXITING;
-	}
-
-	pthread_mutex_unlock(&g_conns_mutex);
-	rte_timer_init(&g_shutdown_timer);
-	rte_timer_reset(&g_shutdown_timer, rte_get_timer_hz() / 1000, PERIODICAL,
-			rte_get_master_lcore(), spdk_nvmf_conn_check_shutdown, NULL);
+	free(conn);
 }
 
 static void

lib/nvmf/conn.h

@@ -55,8 +55,6 @@ enum conn_type {
 };
 
 struct spdk_nvmf_conn {
-	uint32_t			is_valid;
-
 	struct nvmf_session		*sess;
 
 	uint16_t			qid;
@@ -72,13 +70,10 @@ struct spdk_nvmf_conn {
 	struct spdk_poller		poller;
 };
 
-int spdk_initialize_nvmf_conns(int max_connections);
-
-void spdk_shutdown_nvmf_conns(void);
-
 struct spdk_nvmf_conn *
 spdk_nvmf_allocate_conn(void);
 
 int spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn);
+void spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn);
 
 #endif /* NVMF_CONN_H */

lib/nvmf/nvmf.c

@@ -77,10 +77,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF)
 #define SPDK_NVMF_DESC_POOL_SIZE(spdk)	(SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \
 					 SPDK_NVMF_IOQ_POOL_SIZE(spdk))
 
-#define SPDK_NVMF_MAX_CONNECTIONS(spdk)	(MAX_SUBSYSTEMS * \
-					 ((spdk)->MaxSessionsPerSubsystem) * \
-					 ((spdk)->MaxConnectionsPerSession))
-
 struct spdk_nvmf_globals g_nvmf_tgt;
 
 extern struct rte_mempool *request_mempool;
@@ -238,11 +234,7 @@ nvmf_tgt_subsystem_initialize(void)
 		SPDK_ERRLOG("spdk_initialize_nvmf_subsystems failed\n");
 		return rc;
 	}
-	rc = spdk_initialize_nvmf_conns(SPDK_NVMF_MAX_CONNECTIONS(&g_nvmf_tgt));
-	if (rc < 0) {
-		SPDK_ERRLOG("spdk_initialize_nvmf_conns() failed\n");
-		return rc;
-	}
+
 	return rc;
 }
 

lib/nvmf/session.c

@@ -195,10 +195,18 @@ nvmf_create_session(const char *subnqn)
 	return session;
 }
 
-static void
-nvmf_delete_session(struct nvmf_session	*session)
+void
+spdk_nvmf_session_destruct(struct nvmf_session *session)
 {
 	session->subsys->session = NULL;
+
+	while (!TAILQ_EMPTY(&session->connections)) {
+		struct spdk_nvmf_conn *conn = TAILQ_FIRST(&session->connections);
+
+		TAILQ_REMOVE(&session->connections, conn, link);
+		spdk_nvmf_conn_destruct(conn);
+	}
+
 	free(session);
 }
 
@@ -268,15 +276,11 @@ void
 nvmf_disconnect(struct nvmf_session *session,
 		struct spdk_nvmf_conn *conn)
 {
-	if (session) {
-		if (session->num_connections > 0) {
-			session->num_connections--;
-			TAILQ_REMOVE(&session->connections, conn, link);
-		}
+	session->num_connections--;
+	TAILQ_REMOVE(&session->connections, conn, link);
 
-		if (session->num_connections == 0) {
-			nvmf_delete_session(session);
-		}
+	if (session->num_connections == 0 || conn->type == CONN_TYPE_AQ) {
+		spdk_nvmf_session_destruct(session);
 	}
 }
 

lib/nvmf/session.h

@@ -94,4 +94,6 @@ nvmf_check_io_completions(struct nvmf_session *session);
 void
 nvmf_check_admin_completions(struct nvmf_session *session);
 
+void spdk_nvmf_session_destruct(struct nvmf_session *session);
+
 #endif

lib/nvmf/subsystem.c

@@ -101,8 +101,7 @@ nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem)
 	}
 
 	if (subsystem->session) {
-		/* TODO: Call a session function that closes all connections */
-		free(subsystem->session);
+		spdk_nvmf_session_destruct(subsystem->session);
 	}
 
 	TAILQ_REMOVE(&g_subsystems, subsystem, entries);