From 4ebe3e2b42f39149a684d306687c84b5f689be28 Mon Sep 17 00:00:00 2001
From: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Date: Wed, 3 Nov 2021 11:44:13 +0900
Subject: [PATCH] bdev/nvme: Delete already created qpairs if connect qpair
 failed while resetting ctrlr

bdev_nvme_reset() deletes all qpairs, reset a ctrlr, and then create
all qpairs. Any qpair may fail to be created, and then the reset
request may fail. However, already created qpairs were left.

Let's delete the already created qpairs and then fail the reset request.

This will make us easier to control reconnect, deley reconnect by
a few seconds, or stop reconnect after repeated failures and then
delete ctrlr.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I414e2281b4bf0cbd1cf461d8fc64a22f43d26d13
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9896
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
---
 module/bdev/nvme/bdev_nvme.c | 40 +++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c
index e9940ddf5..7e0ecf5bd 100644
--- a/module/bdev/nvme/bdev_nvme.c
+++ b/module/bdev/nvme/bdev_nvme.c
@@ -1263,12 +1263,39 @@ bdev_nvme_reset_complete(struct nvme_ctrlr *nvme_ctrlr, bool success)
 			      _bdev_nvme_reset_complete);
 }
 
+static void
+bdev_nvme_reset_create_qpairs_failed(struct spdk_io_channel_iter *i, int status)
+{
+	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
+
+	bdev_nvme_reset_complete(nvme_ctrlr, false);
+}
+
+static void
+bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i)
+{
+	struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
+	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
+
+	bdev_nvme_destroy_qpair(ctrlr_ch);
+
+	spdk_for_each_channel_continue(i, 0);
+}
+
 static void
 bdev_nvme_reset_create_qpairs_done(struct spdk_io_channel_iter *i, int status)
 {
 	struct nvme_ctrlr *nvme_ctrlr = spdk_io_channel_iter_get_io_device(i);
 
-	bdev_nvme_reset_complete(nvme_ctrlr, status == 0);
+	if (status == 0) {
+		bdev_nvme_reset_complete(nvme_ctrlr, true);
+	} else {
+		/* Delete the added qpairs and quiesce ctrlr to make the states clean. */
+		spdk_for_each_channel(nvme_ctrlr,
+				      bdev_nvme_reset_destroy_qpair,
+				      NULL,
+				      bdev_nvme_reset_create_qpairs_failed);
+	}
 }
 
 static void
@@ -1329,17 +1356,6 @@ bdev_nvme_reset_ctrlr(struct spdk_io_channel_iter *i, int status)
 					  nvme_ctrlr, 0);
 }
 
-static void
-bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i)
-{
-	struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
-	struct nvme_ctrlr_channel *ctrlr_ch = spdk_io_channel_get_ctx(ch);
-
-	bdev_nvme_destroy_qpair(ctrlr_ch);
-
-	spdk_for_each_channel_continue(i, 0);
-}
-
 static void
 _bdev_nvme_reset(void *ctx)
 {