From df7c2a225343c91e51bd1a8d71bd9fa9e8dd1b03 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Mon, 21 Feb 2022 11:51:07 +0900 Subject: [PATCH] nvme: Call ctrlr_disconnect_done() after qpair_process_comletions() returns -ENXIO Add a new flag is_disconnecting to struct spdk_nvme_ctrlr. Separate calling nvme_ctrlr_disconnect() and nvme_ctrlr_disconnect_done() by using the flag is_disconnecting. Additionally, change nvme_ctrlr_fail() to skip setting ctrlr->is_failed to true if ctrlr->is_disconnecting is true. Change-Id: Ie2c74ba41f120662a30f6198751d07005d23abcf Signed-off-by: Shuhei Matsumoto Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11000 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Reviewed-by: Aleksey Marchuk Reviewed-by: Ben Walker --- lib/nvme/nvme_ctrlr.c | 29 ++++++++++++++----- lib/nvme/nvme_internal.h | 2 ++ .../lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c | 3 ++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index 0627bb67e..cf7f25d42 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -1040,6 +1040,11 @@ nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove) return; } + if (ctrlr->is_disconnecting) { + NVME_CTRLR_DEBUGLOG(ctrlr, "already disconnecting\n"); + return; + } + ctrlr->is_failed = true; nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); NVME_CTRLR_ERRLOG(ctrlr, "in failed state.\n"); @@ -1640,6 +1645,7 @@ nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) ctrlr->is_resetting = true; ctrlr->is_failed = false; + ctrlr->is_disconnecting = true; NVME_CTRLR_NOTICELOG(ctrlr, "resetting controller\n"); @@ -1665,6 +1671,9 @@ nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) static void nvme_ctrlr_disconnect_done(struct spdk_nvme_ctrlr *ctrlr) { + assert(ctrlr->is_failed == false); + ctrlr->is_disconnecting = false; + /* Doorbell buffer config is invalid during reset */ nvme_ctrlr_free_doorbell_buffer(ctrlr); @@ -1680,13 +1689,9 @@ spdk_nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) int rc; nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); - rc = nvme_ctrlr_disconnect(ctrlr); - if (rc == 0) { - nvme_ctrlr_disconnect_done(ctrlr); - } - nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); + return rc; } @@ -1789,9 +1794,6 @@ spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr) nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); rc = nvme_ctrlr_disconnect(ctrlr); - if (rc == 0) { - nvme_ctrlr_disconnect_done(ctrlr); - } nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); @@ -1802,6 +1804,13 @@ spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr) return rc; } + while (1) { + rc = spdk_nvme_ctrlr_process_admin_completions(ctrlr); + if (rc == -ENXIO) { + break; + } + } + spdk_nvme_ctrlr_reconnect_async(ctrlr); while (true) { @@ -4248,6 +4257,10 @@ spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr *ctrlr) nvme_ctrlr_complete_queued_async_events(ctrlr); } + if (rc == -ENXIO && ctrlr->is_disconnecting) { + nvme_ctrlr_disconnect_done(ctrlr); + } + nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); if (rc < 0) { diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index 5df0ba7fb..fbf9b8614 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -887,6 +887,8 @@ struct spdk_nvme_ctrlr { */ bool prepare_for_reset; + bool is_disconnecting; + uint16_t max_sges; uint16_t cntlid; diff --git a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c index 138d714a7..d7aa4ac61 100644 --- a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c +++ b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c @@ -2794,6 +2794,7 @@ test_nvme_ctrlr_reset(void) g_ut_nvme_regs.cc.raw = 0; g_ut_nvme_regs.csts.raw = 0; g_set_reg_cb = check_en_set_rdy; + g_wait_for_completion_return_val = -ENXIO; CU_ASSERT(spdk_nvme_ctrlr_reset(&ctrlr) == 0); g_set_reg_cb = NULL; CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_READY); @@ -2807,6 +2808,8 @@ test_nvme_ctrlr_reset(void) g_ut_nvme_regs.csts.bits.shst = SPDK_NVME_SHST_COMPLETE; nvme_ctrlr_destruct(&ctrlr); + + g_wait_for_completion_return_val = 0; } static uint32_t g_aer_cb_counter;