From 0b32309bf6c9d6adbf28c14cf1d68b7df642b668 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Fri, 28 Jan 2022 07:03:05 +0900 Subject: [PATCH] bdev/nvme: Check not only I/O qpair but also adminq when finding optimal I/O path For RDMA transport, adminq will find transport error first because usually only adminq polls CM events. Change-Id: I7b22cc8883bf02198f1a90d2654c1de6f2e736e6 Signed-off-by: Shuhei Matsumoto Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11331 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Ben Walker --- module/bdev/nvme/bdev_nvme.c | 8 ++++++++ test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c | 14 +++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index e0e465f37..1d1af4551 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -748,6 +748,11 @@ nvme_io_path_is_connected(struct nvme_io_path *io_path) return false; } + if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(io_path->qpair->ctrlr->ctrlr) != + SPDK_NVME_QPAIR_FAILURE_NONE) { + return false; + } + return true; } @@ -1242,6 +1247,9 @@ bdev_nvme_poll_adminq(void *arg) } else { bdev_nvme_failover(nvme_ctrlr, false); } + } else if (spdk_nvme_ctrlr_get_admin_qp_failure_reason(nvme_ctrlr->ctrlr) != + SPDK_NVME_QPAIR_FAILURE_NONE) { + bdev_nvme_clear_io_path_caches(nvme_ctrlr, NULL); } return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY; diff --git a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c index 38df01d5d..95d17d691 100644 --- a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c +++ b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c @@ -797,6 +797,12 @@ spdk_nvme_ctrlr_is_failed(struct spdk_nvme_ctrlr *ctrlr) return ctrlr->is_failed; } +spdk_nvme_qp_failure_reason +spdk_nvme_ctrlr_get_admin_qp_failure_reason(struct spdk_nvme_ctrlr *ctrlr) +{ + return spdk_nvme_qpair_get_failure_reason(&ctrlr->adminq); +} + #define UT_ANA_DESC_SIZE (sizeof(struct spdk_nvme_ana_group_descriptor) + \ sizeof(uint32_t)) static void @@ -3993,9 +3999,11 @@ test_find_io_path(void) .io_path_list = STAILQ_HEAD_INITIALIZER(nbdev_ch.io_path_list), }; struct spdk_nvme_qpair qpair1 = {}, qpair2 = {}; - struct nvme_ctrlr_channel ctrlr_ch1 = {}, ctrlr_ch2 = {}; - struct nvme_qpair nvme_qpair1 = { .ctrlr_ch = &ctrlr_ch1, }; - struct nvme_qpair nvme_qpair2 = { .ctrlr_ch = &ctrlr_ch2, }; + struct spdk_nvme_ctrlr ctrlr1 = {}, ctrlr2 = {}; + struct nvme_ctrlr nvme_ctrlr1 = { .ctrlr = &ctrlr1, }, nvme_ctrlr2 = { .ctrlr = &ctrlr2, }; + struct nvme_ctrlr_channel ctrlr_ch1 = { .ctrlr = &nvme_ctrlr1, }, ctrlr_ch2 = { .ctrlr = &nvme_ctrlr2, }; + struct nvme_qpair nvme_qpair1 = { .ctrlr_ch = &ctrlr_ch1, .ctrlr = &nvme_ctrlr1, }; + struct nvme_qpair nvme_qpair2 = { .ctrlr_ch = &ctrlr_ch2, .ctrlr = &nvme_ctrlr2, }; struct nvme_ns nvme_ns1 = {}, nvme_ns2 = {}; struct nvme_io_path io_path1 = { .qpair = &nvme_qpair1, .nvme_ns = &nvme_ns1, }; struct nvme_io_path io_path2 = { .qpair = &nvme_qpair2, .nvme_ns = &nvme_ns2, };