From 5e5423de931f6e372710172ea9114033313dafc2 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Wed, 11 May 2022 10:29:24 +0900 Subject: [PATCH] nvme: Add DISABLED to ctrlr's state to show completion of Controller Level Reset In the following patches, nvme_ctrlr_process_init() will be used to disable the controller when disconnecting the admin qpair for PCIe transport. In this case, we will have to exit nvme_ctrlr_process_init() after CSTS.RDY is 0. However, spdk_nvme_ctrlr_reset() and spdk_nvme_ctrlr_reconnect_poll_async() have to continue nvme_ctrlr_process_init() until the controller becomes ready. To differentiate stop and continue clearly, add a new state NVME_CTRLR_STATE_DISABLED to enum nvme_ctrlr_state. Signed-off-by: Shuhei Matsumoto Change-Id: Ic0a5fb7114d4eeb1cefec28bc404184768fb0a96 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12613 Community-CI: Mellanox Build Bot Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Michael Haeuptle Reviewed-by: Ben Walker Reviewed-by: Tomasz Zawadzki --- lib/nvme/nvme_ctrlr.c | 19 ++++--- lib/nvme/nvme_internal.h | 5 ++ .../lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c | 54 +++++++++++++++++++ 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/lib/nvme/nvme_ctrlr.c b/lib/nvme/nvme_ctrlr.c index 4c852e8eb..d458d36a8 100644 --- a/lib/nvme/nvme_ctrlr.c +++ b/lib/nvme/nvme_ctrlr.c @@ -1367,6 +1367,8 @@ nvme_ctrlr_state_string(enum nvme_ctrlr_state state) return "disable and wait for CSTS.RDY = 0"; case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS: return "disable and wait for CSTS.RDY = 0 reg"; + case NVME_CTRLR_STATE_DISABLED: + return "controller is disabled"; case NVME_CTRLR_STATE_ENABLE: return "enable controller by writing CC.EN = 1"; case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC: @@ -3676,13 +3678,8 @@ nvme_ctrlr_process_init_wait_for_ready_0(void *ctx, uint64_t value, const struct csts.raw = (uint32_t)value; if (csts.bits.rdy == 0) { NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 0 && CSTS.RDY = 0\n"); - nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE, + nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLED, nvme_ctrlr_get_ready_timeout(ctrlr)); - /* - * Delay 100us before setting CC.EN = 1. Some NVMe SSDs miss CC.EN getting - * set to 1 if it is too soon after CSTS.RDY is reported as 0. - */ - spdk_delay_us(100); } else { nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, NVME_TIMEOUT_KEEP_EXISTING); @@ -3854,6 +3851,16 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr) rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_0, ctrlr); break; + case NVME_CTRLR_STATE_DISABLED: + nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE, ready_timeout_in_ms); + + /* + * Delay 100us before setting CC.EN = 1. Some NVMe SSDs miss CC.EN getting + * set to 1 if it is too soon after CSTS.RDY is reported as 0. + */ + spdk_delay_us(100); + break; + case NVME_CTRLR_STATE_ENABLE: NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 1\n"); nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC, ready_timeout_in_ms); diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h index 83ea3d7a4..bda62b657 100644 --- a/lib/nvme/nvme_internal.h +++ b/lib/nvme/nvme_internal.h @@ -632,6 +632,11 @@ enum nvme_ctrlr_state { */ NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS, + /** + * The controller is disabled. (CC.EN and CSTS.RDY are 0.) + */ + NVME_CTRLR_STATE_DISABLED, + /** * Enable the controller by writing CC.EN to 1 */ diff --git a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c index d7aa4ac61..149efbcee 100644 --- a/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c +++ b/test/unit/lib/nvme/nvme_ctrlr.c/nvme_ctrlr_ut.c @@ -739,6 +739,12 @@ test_nvme_ctrlr_init_en_1_rdy_0(void) */ g_ut_nvme_regs.csts.bits.rdy = 0; CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + + /* + * Start enabling the controller. + */ + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); /* @@ -796,6 +802,12 @@ test_nvme_ctrlr_init_en_1_rdy_1(void) */ g_ut_nvme_regs.csts.bits.rdy = 0; CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + + /* + * Start enabling the controller. + */ + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); /* @@ -858,6 +870,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_rr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -892,6 +906,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_rr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) != 0); CU_ASSERT(g_ut_nvme_regs.cc.bits.en == 0); @@ -923,6 +939,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_rr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) != 0); CU_ASSERT(g_ut_nvme_regs.cc.bits.en == 0); @@ -954,6 +972,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_rr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) != 0); CU_ASSERT(g_ut_nvme_regs.cc.bits.en == 0); @@ -985,6 +1005,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_rr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -1045,6 +1067,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_wrr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -1079,6 +1103,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_wrr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -1113,6 +1139,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_wrr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) != 0); CU_ASSERT(g_ut_nvme_regs.cc.bits.en == 0); @@ -1144,6 +1172,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_wrr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) != 0); CU_ASSERT(g_ut_nvme_regs.cc.bits.en == 0); @@ -1175,6 +1205,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_wrr(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -1234,6 +1266,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_vs(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -1268,6 +1302,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_vs(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) != 0); CU_ASSERT(g_ut_nvme_regs.cc.bits.en == 0); @@ -1299,6 +1335,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_vs(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -1333,6 +1371,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_vs(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) != 0); CU_ASSERT(g_ut_nvme_regs.cc.bits.en == 0); @@ -1364,6 +1404,8 @@ test_nvme_ctrlr_init_en_0_rdy_0_ams_vs(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1); @@ -1413,6 +1455,9 @@ test_nvme_ctrlr_init_en_0_rdy_0(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); @@ -1466,6 +1511,12 @@ test_nvme_ctrlr_init_en_0_rdy_1(void) */ g_ut_nvme_regs.csts.bits.rdy = 0; CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + + /* + * Start enabling the controller. + */ + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE); /* @@ -2238,6 +2289,9 @@ test_nvme_ctrlr_init_delay(void) CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); + CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_DISABLED); + CU_ASSERT(nvme_ctrlr_process_init(&ctrlr) == 0); CU_ASSERT(ctrlr.state == NVME_CTRLR_STATE_ENABLE);