From 98e119f7a98fdba07f5d43985d8a8cfe981f98ef Mon Sep 17 00:00:00 2001 From: GangCao Date: Tue, 28 Aug 2018 18:41:16 -0400 Subject: [PATCH] lib/nvmf: add the nvmf qpair to the available poll group In the case that the subsystem in the related poll group has NULL IO channel assigned due to some problem like out of resource, for example, the NVMe SSD hardware itself has limited number of IO qpairs. The subsystems in the particular poll group could have zero valid channels. In this case, the creation of assoicated poll group will fail and when adding the new qpair to the specified poll group, needs to have a check and pick the available poll group. Change-Id: Iedee2a6375e48eb7bf899cfb0542c565c7ebd231 Signed-off-by: GangCao Reviewed-on: https://review.gerrithub.io/423646 Tested-by: SPDK CI Jenkins Chandler-Test-Pool: SPDK Automated Test System Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- lib/event/subsystems/nvmf/nvmf_tgt.c | 54 +++++++++++++++++-- lib/nvmf/nvmf.c | 12 +++-- lib/nvmf/nvmf_internal.h | 6 +-- .../ctrlr_discovery.c/ctrlr_discovery_ut.c | 3 +- test/unit/lib/nvmf/subsystem.c/subsystem_ut.c | 3 +- 5 files changed, 65 insertions(+), 13 deletions(-) diff --git a/lib/event/subsystems/nvmf/nvmf_tgt.c b/lib/event/subsystems/nvmf/nvmf_tgt.c index 940967919..bb35dcce5 100644 --- a/lib/event/subsystems/nvmf/nvmf_tgt.c +++ b/lib/event/subsystems/nvmf/nvmf_tgt.c @@ -137,6 +137,36 @@ spdk_nvmf_get_core_rr(void) return core; } +static void +nvmf_tgt_remove_host_trid(struct spdk_nvmf_qpair *qpair) +{ + struct spdk_nvme_transport_id trid_to_remove; + struct nvmf_tgt_host_trid *trid = NULL, *tmp_trid = NULL; + + if (g_spdk_nvmf_tgt_conf->conn_sched != CONNECT_SCHED_HOST_IP) { + return; + } + + if (spdk_nvmf_qpair_get_peer_trid(qpair, &trid_to_remove) != 0) { + return; + } + + TAILQ_FOREACH_SAFE(trid, &g_nvmf_tgt_host_trids, link, tmp_trid) { + if (trid && !strncmp(trid->host_trid.traddr, + trid_to_remove.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1)) { + trid->ref--; + if (trid->ref == 0) { + TAILQ_REMOVE(&g_nvmf_tgt_host_trids, trid, link); + free(trid); + } + + break; + } + } + + return; +} + static uint32_t nvmf_tgt_get_qpair_core(struct spdk_nvmf_qpair *qpair) { @@ -190,15 +220,28 @@ new_qpair(struct spdk_nvmf_qpair *qpair) struct spdk_event *event; struct nvmf_tgt_poll_group *pg; uint32_t core; + uint32_t attempts; if (g_tgt_state != NVMF_TGT_RUNNING) { spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); return; } - core = nvmf_tgt_get_qpair_core(qpair); + for (attempts = 0; attempts < g_num_poll_groups; attempts++) { + core = nvmf_tgt_get_qpair_core(qpair); + pg = &g_poll_groups[core]; + if (pg->group != NULL) { + break; + } else { + nvmf_tgt_remove_host_trid(qpair); + } + } - pg = &g_poll_groups[core]; + if (attempts == g_num_poll_groups) { + SPDK_ERRLOG("No poll groups exist.\n"); + spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); + return; + } event = spdk_event_allocate(core, nvmf_tgt_poll_group_add, qpair, pg); spdk_event_call(event); @@ -228,8 +271,10 @@ nvmf_tgt_destroy_poll_group(void *ctx) pg = &g_poll_groups[spdk_env_get_current_core()]; - spdk_nvmf_poll_group_destroy(pg->group); - pg->group = NULL; + if (pg->group) { + spdk_nvmf_poll_group_destroy(pg->group); + pg->group = NULL; + } } static void @@ -247,7 +292,6 @@ nvmf_tgt_create_poll_group(void *ctx) pg = &g_poll_groups[spdk_env_get_current_core()]; pg->group = spdk_nvmf_poll_group_create(g_spdk_nvmf_tgt); - assert(pg->group != NULL); } static void diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index 2bd04f8c5..32539f531 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -57,6 +57,7 @@ SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) #define SPDK_NVMF_DEFAULT_IO_UNIT_SIZE 131072 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); +static void spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf); /* supplied to a single call to nvmf_qpair_disconnect */ struct nvmf_qpair_disconnect_ctx { @@ -149,7 +150,10 @@ spdk_nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) continue; } - spdk_nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL); + if (spdk_nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { + spdk_nvmf_tgt_destroy_poll_group(io_device, ctx_buf); + return -1; + } } group->poller = spdk_poller_register(spdk_nvmf_poll_group_poll, group, 0); @@ -963,7 +967,7 @@ spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, return poll_group_update_subsystem(group, subsystem); } -void +int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) @@ -975,7 +979,7 @@ spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, rc = poll_group_update_subsystem(group, subsystem); if (rc) { - sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; + spdk_nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); goto fini; } @@ -984,6 +988,8 @@ fini: if (cb_fn) { cb_fn(cb_arg, rc); } + + return rc; } static void diff --git a/lib/nvmf/nvmf_internal.h b/lib/nvmf/nvmf_internal.h index c505cf150..c9c7bf366 100644 --- a/lib/nvmf/nvmf_internal.h +++ b/lib/nvmf/nvmf_internal.h @@ -263,9 +263,9 @@ int spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, struct spdk_nvmf_transport *transport); int spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, struct spdk_nvmf_subsystem *subsystem); -void spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, - struct spdk_nvmf_subsystem *subsystem, - spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); +int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, + struct spdk_nvmf_subsystem *subsystem, + spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); void spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg); void spdk_nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, diff --git a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c index 3e5c3a04b..f86bdf2bd 100644 --- a/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c +++ b/test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c @@ -179,11 +179,12 @@ spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, return 0; } -void +int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) { + return 0; } void diff --git a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c index 8a821b663..1b92efd26 100644 --- a/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c +++ b/test/unit/lib/nvmf/subsystem.c/subsystem_ut.c @@ -135,11 +135,12 @@ spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, return 0; } -void +int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) { + return 0; } void