From fcf52fbff58345f52665dfd627488836df4ef755 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Mon, 9 May 2022 15:48:57 +0900 Subject: [PATCH] bdev/nvme: Reversed orderings for reset between PCIe and NVMe-oF As described in the NVMe specification, a controller level reset includes the following actions: - the controller stops processing any outstanding admin or I/O commands; - all I/O SQs and CQs are deleted. In a full controller reset sequence for a PCIe controller, if we do a controller level reset first, we can abort outstanding commands after the hardware has actually been stopped. For NVMe-oF controller, each I/O qpair is an independent network connection and is disconnected safely. We do not want to change NVMe-oF controller. Fixes the issue #2360 Signed-off-by: Shuhei Matsumoto Change-Id: If05febac74705bfd3df5abd15064c1203126e027 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12447 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Aleksey Marchuk Reviewed-by: Michael Haeuptle Reviewed-by: Ben Walker --- module/bdev/nvme/bdev_nvme.c | 15 ++++++++++----- .../unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index 2551e0c30..e3335d2b0 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -1779,7 +1779,11 @@ bdev_nvme_reset_ctrlr(struct spdk_io_channel_iter *i, int status) assert(status == 0); - nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); + if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { + bdev_nvme_reconnect_ctrlr(nvme_ctrlr); + } else { + nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reconnect_ctrlr); + } } static void @@ -1799,10 +1803,11 @@ _bdev_nvme_reset(void *ctx) assert(nvme_ctrlr->resetting == true); assert(nvme_ctrlr->thread == spdk_get_thread()); - spdk_nvme_ctrlr_prepare_for_reset(nvme_ctrlr->ctrlr); - - /* First, delete all NVMe I/O queue pairs. */ - bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); + if (!spdk_nvme_ctrlr_is_fabrics(nvme_ctrlr->ctrlr)) { + nvme_ctrlr_disconnect(nvme_ctrlr, bdev_nvme_reset_destroy_qpairs); + } else { + bdev_nvme_reset_destroy_qpairs(nvme_ctrlr); + } } static int diff --git a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c index afd12e756..72d388a89 100644 --- a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c +++ b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c @@ -112,6 +112,8 @@ DEFINE_STUB_V(spdk_nvme_ctrlr_register_aer_callback, (struct spdk_nvme_ctrlr *ct DEFINE_STUB_V(spdk_nvme_ctrlr_register_timeout_callback, (struct spdk_nvme_ctrlr *ctrlr, uint64_t timeout_io_us, uint64_t timeout_admin_us, spdk_nvme_timeout_cb cb_fn, void *cb_arg)); +DEFINE_STUB(spdk_nvme_ctrlr_is_fabrics, bool, (struct spdk_nvme_ctrlr *ctrlr), true); + DEFINE_STUB(spdk_nvme_ctrlr_is_ocssd_supported, bool, (struct spdk_nvme_ctrlr *ctrlr), false); DEFINE_STUB(spdk_nvme_ctrlr_cmd_abort, int, (struct spdk_nvme_ctrlr *ctrlr,