From 2de5aabd92c82c0451526ff44f8318c0c1a957dd Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Tue, 22 Dec 2020 21:50:29 +0900 Subject: [PATCH] bdev/nvme: Insert new failover trid before failed trids Add an new variable is_failed to struct nvme_bdev_ctrlr_trid, and set it to true when starting failover or when failover failed, or set it to false when initializing or failover succeeded. Then add an new failover trid before all failed trids. The test log showed that many failover failed because new trid was added after failed trids. Signed-off-by: Shuhei Matsumoto Change-Id: I220839245c6414b5d3ef69a2fa1b97904d88d8bd Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5698 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Changpeng Liu Reviewed-by: Aleksey Marchuk --- module/bdev/nvme/bdev_nvme.c | 21 ++++++++++++++++++++- module/bdev/nvme/common.h | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index 3b2f897de..b086e6897 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -374,6 +374,7 @@ _bdev_nvme_reset_complete(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, int rc) /* we are using the for_each_channel cb_arg like a return code here. */ /* If it's zero, we succeeded, otherwise, the reset failed. */ void *cb_arg = NULL; + struct nvme_bdev_ctrlr_trid *curr_trid; if (rc) { cb_arg = (void *)0x1; @@ -385,6 +386,13 @@ _bdev_nvme_reset_complete(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, int rc) pthread_mutex_lock(&g_bdev_nvme_mutex); nvme_bdev_ctrlr->resetting = false; nvme_bdev_ctrlr->failover_in_progress = false; + + curr_trid = TAILQ_FIRST(&nvme_bdev_ctrlr->trids); + assert(curr_trid != NULL); + assert(&curr_trid->trid == nvme_bdev_ctrlr->connected_trid); + + curr_trid->is_failed = cb_arg != NULL ? true : false; + pthread_mutex_unlock(&g_bdev_nvme_mutex); /* Make sure we clear any pending resets before returning. */ spdk_for_each_channel(nvme_bdev_ctrlr, @@ -545,6 +553,8 @@ bdev_nvme_failover(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, bool remove) } nvme_bdev_ctrlr->resetting = true; + curr_trid->is_failed = true; + if (next_trid) { assert(curr_trid->trid.trtype != SPDK_NVME_TRANSPORT_PCIE); @@ -1802,7 +1812,7 @@ bdev_nvme_add_trid(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct spdk_nvme_ctr uint32_t i, nsid; struct nvme_bdev_ns *nvme_ns; struct spdk_nvme_ns *new_ns; - struct nvme_bdev_ctrlr_trid *new_trid; + struct nvme_bdev_ctrlr_trid *new_trid, *tmp_trid; int rc = 0; assert(nvme_bdev_ctrlr != NULL); @@ -1862,6 +1872,15 @@ bdev_nvme_add_trid(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, struct spdk_nvme_ctr goto exit; } new_trid->trid = *trid; + new_trid->is_failed = false; + + TAILQ_FOREACH(tmp_trid, &nvme_bdev_ctrlr->trids, link) { + if (tmp_trid->is_failed) { + TAILQ_INSERT_BEFORE(tmp_trid, new_trid, link); + goto exit; + } + } + TAILQ_INSERT_TAIL(&nvme_bdev_ctrlr->trids, new_trid, link); exit: diff --git a/module/bdev/nvme/common.h b/module/bdev/nvme/common.h index 6277ad0a2..3b9efce41 100644 --- a/module/bdev/nvme/common.h +++ b/module/bdev/nvme/common.h @@ -72,6 +72,7 @@ struct ocssd_bdev_ctrlr; struct nvme_bdev_ctrlr_trid { struct spdk_nvme_transport_id trid; TAILQ_ENTRY(nvme_bdev_ctrlr_trid) link; + bool is_failed; }; struct nvme_bdev_ctrlr {