nvmf: Listen for incoming connections only on addresses specified

Currently the NVMf target listens for new connections on any address.
Instead, listen only on the addresses specified by the user.

Change-Id: Idb6d37c422e442fc70a8673bd3fcfb9c27b57828
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>

lib/nvmf/nvmf.c

@@ -130,11 +130,6 @@ nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess,
 	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max In Capsule Data: %d bytes\n", in_capsule_data_size);
 	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max I/O Size: %d bytes\n", max_io_size);
 
-	/* init nvmf specific config options */
-	if (!g_nvmf_tgt.sin_port) {
-		g_nvmf_tgt.sin_port = htons(SPDK_NVMF_DEFAULT_SIN_PORT);
-	}
-
 	rc = spdk_nvmf_initialize_pools();
 	if (rc != 0) {
 		SPDK_ERRLOG("spdk_nvmf_initialize_pools() failed\n");

lib/nvmf/nvmf_internal.h

@@ -68,8 +68,6 @@ struct spdk_nvmf_globals {
 
 	uint32_t in_capsule_data_size;
 	uint32_t max_io_size;
-
-	uint16_t	   sin_port;
 };
 
 int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess,

lib/nvmf/rdma.c

@@ -136,17 +136,29 @@ struct spdk_nvmf_rdma_session {
 	struct ibv_mr				*buf_mr;
 };
 
+struct spdk_nvmf_rdma_listen_addr {
+	char					*traddr;
+	char					*trsvc;
+	struct rdma_cm_id			*id;
+	TAILQ_ENTRY(spdk_nvmf_rdma_listen_addr)	link;
+};
+
 struct spdk_nvmf_rdma {
-	struct rdma_event_channel	*acceptor_event_channel;
+	struct rdma_event_channel			*acceptor_event_channel;
-	struct rdma_cm_id		*acceptor_listen_id;
 
 	uint16_t max_queue_depth;
 	uint32_t max_io_size;
 	uint32_t in_capsule_data_size;
 	uint32_t num_devices_found;
+
+	pthread_mutex_t lock;
+	TAILQ_HEAD(, spdk_nvmf_rdma_listen_addr)	listen_addrs;
 };
 
-static struct spdk_nvmf_rdma g_rdma = { };
+static struct spdk_nvmf_rdma g_rdma = {
+	.lock = PTHREAD_MUTEX_INITIALIZER,
+	.listen_addrs = TAILQ_HEAD_INITIALIZER(g_rdma.listen_addrs),
+};
 
 static inline struct spdk_nvmf_rdma_conn *
 get_rdma_conn(struct spdk_nvmf_conn *conn)
@@ -975,17 +987,13 @@ static int
 spdk_nvmf_rdma_acceptor_init(void)
 {
 	struct sockaddr_in	addr;
-	uint16_t		sin_port;
 	int			rc;
+	struct spdk_nvmf_rdma_listen_addr *listen_addr, *tmp;
 
 	if (g_rdma.num_devices_found == 0) {
 		return 0;
 	}
 
-	memset(&addr, 0, sizeof(addr));
-	addr.sin_family = AF_INET;
-	addr.sin_port = g_nvmf_tgt.sin_port;
-
 	/* create an event channel with rdmacm to receive
 	   connection oriented requests and notifications */
 	g_rdma.acceptor_event_channel = rdma_create_event_channel();
@@ -999,30 +1007,46 @@ spdk_nvmf_rdma_acceptor_init(void)
 		goto create_id_error;
 	}
 
-	rc = rdma_create_id(g_rdma.acceptor_event_channel, &g_rdma.acceptor_listen_id, NULL, RDMA_PS_TCP);
+	pthread_mutex_lock(&g_rdma.lock);
-	if (rc < 0) {
+	TAILQ_FOREACH_SAFE(listen_addr, &g_rdma.listen_addrs, link, tmp) {
-		SPDK_ERRLOG("rdma_create_id() failed\n");
+		memset(&addr, 0, sizeof(addr));
-		goto create_id_error;
+		addr.sin_family = AF_INET;
+		addr.sin_addr.s_addr = inet_addr(listen_addr->traddr);
+		addr.sin_port = htons((uint16_t)strtoul(listen_addr->trsvc, NULL, 10));
+
+		rc = rdma_create_id(g_rdma.acceptor_event_channel, &listen_addr->id, NULL,
+				    RDMA_PS_TCP);
+		if (rc < 0) {
+			SPDK_ERRLOG("rdma_create_id() failed\n");
+			goto listen_error;
+		}
+
+		rc = rdma_bind_addr(listen_addr->id, (struct sockaddr *)&addr);
+		if (rc < 0) {
+			SPDK_ERRLOG("rdma_bind_addr() failed\n");
+			goto listen_error;
+		}
+
+		rc = rdma_listen(listen_addr->id, 10); /* 10 = backlog */
+		if (rc < 0) {
+			SPDK_ERRLOG("rdma_listen() failed\n");
+			goto listen_error;
+		}
+		SPDK_NOTICELOG("*** NVMf Target Listening on %s port %d ***\n",
+			       listen_addr->traddr, ntohs(rdma_get_src_port(listen_addr->id)));
 	}
 
-	rc = rdma_bind_addr(g_rdma.acceptor_listen_id, (struct sockaddr *)&addr);
+	pthread_mutex_unlock(&g_rdma.lock);
-	if (rc < 0) {
+	return 0;
-		SPDK_ERRLOG("rdma_bind_addr() failed\n");
-		goto listen_error;
-	}
-
-	rc = rdma_listen(g_rdma.acceptor_listen_id, 10); /* 10 = backlog */
-	if (rc < 0) {
-		SPDK_ERRLOG("rdma_listen() failed\n");
-		goto listen_error;
-	}
-	sin_port = ntohs(rdma_get_src_port(g_rdma.acceptor_listen_id));
-	SPDK_NOTICELOG("*** NVMf Target Listening on port %d ***\n", sin_port);
-
-	return rc;
 
 listen_error:
-	rdma_destroy_id(g_rdma.acceptor_listen_id);
+	TAILQ_FOREACH_SAFE(listen_addr, &g_rdma.listen_addrs, link, tmp) {
+		if (listen_addr->id) {
+			rdma_destroy_id(listen_addr->id);
+		}
+	}
+	pthread_mutex_unlock(&g_rdma.lock);
+
 create_id_error:
 	rdma_destroy_event_channel(g_rdma.acceptor_event_channel);
 	return -1;
@@ -1031,6 +1055,14 @@ create_id_error:
 static void
 spdk_nvmf_rdma_acceptor_fini(void)
 {
+	struct spdk_nvmf_rdma_listen_addr *listen_addr, *tmp;
+
+	pthread_mutex_lock(&g_rdma.lock);
+	TAILQ_FOREACH_SAFE(listen_addr, &g_rdma.listen_addrs, link, tmp) {
+		TAILQ_REMOVE(&g_rdma.listen_addrs, listen_addr, link);
+		free(listen_addr);
+	}
+	pthread_mutex_unlock(&g_rdma.lock);
 }
 
 static int
@@ -1430,8 +1462,8 @@ spdk_nvmf_rdma_poll(struct spdk_nvmf_conn *conn)
 }
 
 static void
-nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr,
+spdk_nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr,
-		   struct spdk_nvmf_discovery_log_page_entry *entry)
+			struct spdk_nvmf_discovery_log_page_entry *entry)
 {
 	entry->trtype = SPDK_NVMF_TRTYPE_RDMA;
 	entry->adrfam = SPDK_NVMF_ADRFAM_IPV4;
@@ -1445,6 +1477,35 @@ nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr,
 	entry->tsas.rdma.rdma_cms = SPDK_NVMF_RDMA_CMS_RDMA_CM;
 }
 
+static int
+spdk_nvmf_rdma_listen(struct spdk_nvmf_listen_addr *listen_addr)
+{
+	struct spdk_nvmf_rdma_listen_addr *addr, *tmp;
+
+	pthread_mutex_lock(&g_rdma.lock);
+	TAILQ_FOREACH_SAFE(addr, &g_rdma.listen_addrs, link, tmp) {
+		if ((!strcasecmp(addr->traddr, listen_addr->traddr)) &&
+		    (!strcasecmp(addr->trsvc, listen_addr->trsvc))) {
+			pthread_mutex_unlock(&g_rdma.lock);
+			return 0;
+		}
+	}
+
+	addr = calloc(1, sizeof(*addr));
+	if (!addr) {
+		pthread_mutex_unlock(&g_rdma.lock);
+		return -1;
+	}
+
+	addr->traddr = listen_addr->traddr;
+	addr->trsvc = listen_addr->trsvc;
+
+	TAILQ_INSERT_TAIL(&g_rdma.listen_addrs, addr, link);
+	pthread_mutex_unlock(&g_rdma.lock);
+
+	return 0;
+}
+
 const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = {
 	.name = "rdma",
 	.transport_init = spdk_nvmf_rdma_init,
@@ -1454,6 +1515,9 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = {
 	.acceptor_poll = spdk_nvmf_rdma_acceptor_poll,
 	.acceptor_fini = spdk_nvmf_rdma_acceptor_fini,
 
+	.listen_addr_add = spdk_nvmf_rdma_listen,
+	.listen_addr_discover = spdk_nvmf_rdma_discover,
+
 	.session_init = spdk_nvmf_rdma_session_init,
 	.session_fini = spdk_nvmf_rdma_session_fini,
 
@@ -1463,7 +1527,7 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = {
 	.conn_fini = spdk_nvmf_rdma_close_conn,
 	.conn_poll = spdk_nvmf_rdma_poll,
 
-	.listen_addr_discover = nvmf_rdma_discover,
+
 };
 
 SPDK_LOG_REGISTER_TRACE_FLAG("rdma", SPDK_TRACE_RDMA)

lib/nvmf/subsystem.c

@@ -175,6 +175,7 @@ spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem,
 				 char *traddr, char *trsvc)
 {
 	struct spdk_nvmf_listen_addr *listen_addr;
+	int rc;
 
 	listen_addr = calloc(1, sizeof(*listen_addr));
 	if (!listen_addr) {
@@ -199,6 +200,12 @@ spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem,
 	TAILQ_INSERT_HEAD(&subsystem->listen_addrs, listen_addr, link);
 	subsystem->num_listen_addrs++;
 
+	rc = transport->listen_addr_add(listen_addr);
+	if (rc < 0) {
+		SPDK_ERRLOG("Unable to listen on address '%s'\n", traddr);
+		return -1;
+	}
+
 	return 0;
 }
 

lib/nvmf/transport.h

@@ -75,6 +75,18 @@ struct spdk_nvmf_transport {
 	 */
 	void (*acceptor_fini)(void);
 
+	/**
+	  * Instruct the acceptor to listen on the address provided. This
+	  * may be called multiple times.
+	  */
+	int (*listen_addr_add)(struct spdk_nvmf_listen_addr *listen_addr);
+
+	/**
+	 * Fill out a discovery log entry for a specific listen address.
+	 */
+	void (*listen_addr_discover)(struct spdk_nvmf_listen_addr *listen_addr,
+				     struct spdk_nvmf_discovery_log_page_entry *entry);
+
 	/**
 	 * Initialize the transport for the given session
 	 */
@@ -108,12 +120,6 @@ struct spdk_nvmf_transport {
 	 * Poll a connection for events.
 	 */
 	int (*conn_poll)(struct spdk_nvmf_conn *conn);
-
-	/**
-	 * Fill out a discovery log entry for a specific listen address.
-	 */
-	void (*listen_addr_discover)(struct spdk_nvmf_listen_addr *listen_addr,
-				     struct spdk_nvmf_discovery_log_page_entry *entry);
 };
 
 int spdk_nvmf_transport_init(void);