diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index b13790899..5fd111d2d 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -130,11 +130,6 @@ nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max In Capsule Data: %d bytes\n", in_capsule_data_size); SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max I/O Size: %d bytes\n", max_io_size); - /* init nvmf specific config options */ - if (!g_nvmf_tgt.sin_port) { - g_nvmf_tgt.sin_port = htons(SPDK_NVMF_DEFAULT_SIN_PORT); - } - rc = spdk_nvmf_initialize_pools(); if (rc != 0) { SPDK_ERRLOG("spdk_nvmf_initialize_pools() failed\n"); diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index c41e56e01..a22fd8691 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -68,8 +68,6 @@ struct spdk_nvmf_globals { uint32_t in_capsule_data_size; uint32_t max_io_size; - - uint16_t sin_port; }; int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess, diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 1009f32ae..71fdc91a9 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -136,17 +136,29 @@ struct spdk_nvmf_rdma_session { struct ibv_mr *buf_mr; }; +struct spdk_nvmf_rdma_listen_addr { + char *traddr; + char *trsvc; + struct rdma_cm_id *id; + TAILQ_ENTRY(spdk_nvmf_rdma_listen_addr) link; +}; + struct spdk_nvmf_rdma { - struct rdma_event_channel *acceptor_event_channel; - struct rdma_cm_id *acceptor_listen_id; + struct rdma_event_channel *acceptor_event_channel; uint16_t max_queue_depth; uint32_t max_io_size; uint32_t in_capsule_data_size; uint32_t num_devices_found; + + pthread_mutex_t lock; + TAILQ_HEAD(, spdk_nvmf_rdma_listen_addr) listen_addrs; }; -static struct spdk_nvmf_rdma g_rdma = { }; +static struct spdk_nvmf_rdma g_rdma = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .listen_addrs = TAILQ_HEAD_INITIALIZER(g_rdma.listen_addrs), +}; static inline struct spdk_nvmf_rdma_conn * get_rdma_conn(struct spdk_nvmf_conn *conn) @@ -975,17 +987,13 @@ static int spdk_nvmf_rdma_acceptor_init(void) { struct sockaddr_in addr; - uint16_t sin_port; int rc; + struct spdk_nvmf_rdma_listen_addr *listen_addr, *tmp; if (g_rdma.num_devices_found == 0) { return 0; } - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = g_nvmf_tgt.sin_port; - /* create an event channel with rdmacm to receive connection oriented requests and notifications */ g_rdma.acceptor_event_channel = rdma_create_event_channel(); @@ -999,30 +1007,46 @@ spdk_nvmf_rdma_acceptor_init(void) goto create_id_error; } - rc = rdma_create_id(g_rdma.acceptor_event_channel, &g_rdma.acceptor_listen_id, NULL, RDMA_PS_TCP); - if (rc < 0) { - SPDK_ERRLOG("rdma_create_id() failed\n"); - goto create_id_error; + pthread_mutex_lock(&g_rdma.lock); + TAILQ_FOREACH_SAFE(listen_addr, &g_rdma.listen_addrs, link, tmp) { + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr(listen_addr->traddr); + addr.sin_port = htons((uint16_t)strtoul(listen_addr->trsvc, NULL, 10)); + + rc = rdma_create_id(g_rdma.acceptor_event_channel, &listen_addr->id, NULL, + RDMA_PS_TCP); + if (rc < 0) { + SPDK_ERRLOG("rdma_create_id() failed\n"); + goto listen_error; + } + + rc = rdma_bind_addr(listen_addr->id, (struct sockaddr *)&addr); + if (rc < 0) { + SPDK_ERRLOG("rdma_bind_addr() failed\n"); + goto listen_error; + } + + rc = rdma_listen(listen_addr->id, 10); /* 10 = backlog */ + if (rc < 0) { + SPDK_ERRLOG("rdma_listen() failed\n"); + goto listen_error; + } + SPDK_NOTICELOG("*** NVMf Target Listening on %s port %d ***\n", + listen_addr->traddr, ntohs(rdma_get_src_port(listen_addr->id))); } - rc = rdma_bind_addr(g_rdma.acceptor_listen_id, (struct sockaddr *)&addr); - if (rc < 0) { - SPDK_ERRLOG("rdma_bind_addr() failed\n"); - goto listen_error; - } - - rc = rdma_listen(g_rdma.acceptor_listen_id, 10); /* 10 = backlog */ - if (rc < 0) { - SPDK_ERRLOG("rdma_listen() failed\n"); - goto listen_error; - } - sin_port = ntohs(rdma_get_src_port(g_rdma.acceptor_listen_id)); - SPDK_NOTICELOG("*** NVMf Target Listening on port %d ***\n", sin_port); - - return rc; + pthread_mutex_unlock(&g_rdma.lock); + return 0; listen_error: - rdma_destroy_id(g_rdma.acceptor_listen_id); + TAILQ_FOREACH_SAFE(listen_addr, &g_rdma.listen_addrs, link, tmp) { + if (listen_addr->id) { + rdma_destroy_id(listen_addr->id); + } + } + pthread_mutex_unlock(&g_rdma.lock); + create_id_error: rdma_destroy_event_channel(g_rdma.acceptor_event_channel); return -1; @@ -1031,6 +1055,14 @@ create_id_error: static void spdk_nvmf_rdma_acceptor_fini(void) { + struct spdk_nvmf_rdma_listen_addr *listen_addr, *tmp; + + pthread_mutex_lock(&g_rdma.lock); + TAILQ_FOREACH_SAFE(listen_addr, &g_rdma.listen_addrs, link, tmp) { + TAILQ_REMOVE(&g_rdma.listen_addrs, listen_addr, link); + free(listen_addr); + } + pthread_mutex_unlock(&g_rdma.lock); } static int @@ -1430,8 +1462,8 @@ spdk_nvmf_rdma_poll(struct spdk_nvmf_conn *conn) } static void -nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr, - struct spdk_nvmf_discovery_log_page_entry *entry) +spdk_nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr, + struct spdk_nvmf_discovery_log_page_entry *entry) { entry->trtype = SPDK_NVMF_TRTYPE_RDMA; entry->adrfam = SPDK_NVMF_ADRFAM_IPV4; @@ -1445,6 +1477,35 @@ nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr, entry->tsas.rdma.rdma_cms = SPDK_NVMF_RDMA_CMS_RDMA_CM; } +static int +spdk_nvmf_rdma_listen(struct spdk_nvmf_listen_addr *listen_addr) +{ + struct spdk_nvmf_rdma_listen_addr *addr, *tmp; + + pthread_mutex_lock(&g_rdma.lock); + TAILQ_FOREACH_SAFE(addr, &g_rdma.listen_addrs, link, tmp) { + if ((!strcasecmp(addr->traddr, listen_addr->traddr)) && + (!strcasecmp(addr->trsvc, listen_addr->trsvc))) { + pthread_mutex_unlock(&g_rdma.lock); + return 0; + } + } + + addr = calloc(1, sizeof(*addr)); + if (!addr) { + pthread_mutex_unlock(&g_rdma.lock); + return -1; + } + + addr->traddr = listen_addr->traddr; + addr->trsvc = listen_addr->trsvc; + + TAILQ_INSERT_TAIL(&g_rdma.listen_addrs, addr, link); + pthread_mutex_unlock(&g_rdma.lock); + + return 0; +} + const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = { .name = "rdma", .transport_init = spdk_nvmf_rdma_init, @@ -1454,6 +1515,9 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = { .acceptor_poll = spdk_nvmf_rdma_acceptor_poll, .acceptor_fini = spdk_nvmf_rdma_acceptor_fini, + .listen_addr_add = spdk_nvmf_rdma_listen, + .listen_addr_discover = spdk_nvmf_rdma_discover, + .session_init = spdk_nvmf_rdma_session_init, .session_fini = spdk_nvmf_rdma_session_fini, @@ -1463,7 +1527,7 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = { .conn_fini = spdk_nvmf_rdma_close_conn, .conn_poll = spdk_nvmf_rdma_poll, - .listen_addr_discover = nvmf_rdma_discover, + }; SPDK_LOG_REGISTER_TRACE_FLAG("rdma", SPDK_TRACE_RDMA) diff --git a/lib/nvmf/subsystem.c b/lib/nvmf/subsystem.c index e8a092f9f..296183e2a 100644 --- a/lib/nvmf/subsystem.c +++ b/lib/nvmf/subsystem.c @@ -175,6 +175,7 @@ spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem, char *traddr, char *trsvc) { struct spdk_nvmf_listen_addr *listen_addr; + int rc; listen_addr = calloc(1, sizeof(*listen_addr)); if (!listen_addr) { @@ -199,6 +200,12 @@ spdk_nvmf_subsystem_add_listener(struct spdk_nvmf_subsystem *subsystem, TAILQ_INSERT_HEAD(&subsystem->listen_addrs, listen_addr, link); subsystem->num_listen_addrs++; + rc = transport->listen_addr_add(listen_addr); + if (rc < 0) { + SPDK_ERRLOG("Unable to listen on address '%s'\n", traddr); + return -1; + } + return 0; } diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 48b082806..f352a8f89 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -75,6 +75,18 @@ struct spdk_nvmf_transport { */ void (*acceptor_fini)(void); + /** + * Instruct the acceptor to listen on the address provided. This + * may be called multiple times. + */ + int (*listen_addr_add)(struct spdk_nvmf_listen_addr *listen_addr); + + /** + * Fill out a discovery log entry for a specific listen address. + */ + void (*listen_addr_discover)(struct spdk_nvmf_listen_addr *listen_addr, + struct spdk_nvmf_discovery_log_page_entry *entry); + /** * Initialize the transport for the given session */ @@ -108,12 +120,6 @@ struct spdk_nvmf_transport { * Poll a connection for events. */ int (*conn_poll)(struct spdk_nvmf_conn *conn); - - /** - * Fill out a discovery log entry for a specific listen address. - */ - void (*listen_addr_discover)(struct spdk_nvmf_listen_addr *listen_addr, - struct spdk_nvmf_discovery_log_page_entry *entry); }; int spdk_nvmf_transport_init(void);