lib/nvme: Make nvme_ctrlr_destruct() asynchronous

Following the last patch, separate nvme_ctrlr_destruct()
into nvme_ctrlr_destruct_async() and nvme_ctrlr_destruct_poll_async(),
but keep nvme_ctrlr_destruct() by replacing the internal by
nvme_ctrlr_destruct_async() and nvme_ctrlr_destruct_poll_async().

Add shutdown_complete to nvme_ctrlr_detach_ctx. If shutdown_complete is true,
we can destruct the controller. The case that nvme_ctrlr_shutdown_async()
failed sets shutdown_complete to true. The case that nvme_ctrlr_disable()
is called sets shutdown_complete to true unconditionally.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I3994e259f9d3ccf8fede3ac03aadef911eefb9dd
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/4415
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jacek Kalwas <jacek.kalwas@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/nvme/nvme_ctrlr.c

@@ -944,11 +944,13 @@ nvme_ctrlr_shutdown_async(struct spdk_nvme_ctrlr *ctrlr,
 	union spdk_nvme_cc_register	cc;
 
 	if (ctrlr->is_removed) {
+		ctx->shutdown_complete = true;
 		return;
 	}
 
 	if (nvme_ctrlr_get_cc(ctrlr, &cc)) {
 		SPDK_ERRLOG("ctrlr %s get_cc() failed\n", ctrlr->trid.traddr);
+		ctx->shutdown_complete = true;
 		return;
 	}
 
@@ -956,6 +958,7 @@ nvme_ctrlr_shutdown_async(struct spdk_nvme_ctrlr *ctrlr,
 
 	if (nvme_ctrlr_set_cc(ctrlr, &cc)) {
 		SPDK_ERRLOG("ctrlr %s set_cc() failed\n", ctrlr->trid.traddr);
+		ctx->shutdown_complete = true;
 		return;
 	}
 
@@ -3269,11 +3272,10 @@ nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr)
 }
 
 void
-nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
+nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
+			  struct nvme_ctrlr_detach_ctx *ctx)
 {
 	struct spdk_nvme_qpair *qpair, *tmp;
-	struct nvme_ctrlr_detach_ctx ctx = {};
-	int rc;
 
 	SPDK_DEBUGLOG(nvme, "Prepare to destruct SSD: %s\n", ctrlr->trid.traddr);
 
@@ -3295,15 +3297,24 @@ nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
 		SPDK_INFOLOG(nvme, "Disable SSD: %s without shutdown notification\n",
 			     ctrlr->trid.traddr);
 		nvme_ctrlr_disable(ctrlr);
+		ctx->shutdown_complete = true;
 	} else {
-		nvme_ctrlr_shutdown_async(ctrlr, &ctx);
+		nvme_ctrlr_shutdown_async(ctrlr, ctx);
-		while (1) {
+	}
-			rc = nvme_ctrlr_shutdown_poll_async(ctrlr, &ctx);
+}
-			if (rc != -EAGAIN) {
+
-				break;
+int
-			}
+nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
-			nvme_delay(1000);
+			       struct nvme_ctrlr_detach_ctx *ctx)
+{
+	int rc = 0;
+
+	if (!ctx->shutdown_complete) {
+		rc = nvme_ctrlr_shutdown_poll_async(ctrlr, ctx);
+		if (rc == -EAGAIN) {
+			return -EAGAIN;
 		}
+		/* Destruct ctrlr forcefully for any other error. */
 	}
 
 	nvme_ctrlr_destruct_namespaces(ctrlr);
@@ -3315,6 +3326,25 @@ nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
 	ctrlr->ana_log_page_size = 0;
 
 	nvme_transport_ctrlr_destruct(ctrlr);
+
+	return rc;
+}
+
+void
+nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr)
+{
+	struct nvme_ctrlr_detach_ctx ctx = {};
+	int rc;
+
+	nvme_ctrlr_destruct_async(ctrlr, &ctx);
+
+	while (1) {
+		rc = nvme_ctrlr_destruct_poll_async(ctrlr, &ctx);
+		if (rc != -EAGAIN) {
+			break;
+		}
+		nvme_delay(1000);
+	}
 }
 
 int

lib/nvme/nvme_internal.h

@@ -846,6 +846,7 @@ struct spdk_nvme_probe_ctx {
 struct nvme_ctrlr_detach_ctx {
 	uint64_t	shutdown_start_tsc;
 	uint32_t	shutdown_timeout_ms;
+	bool		shutdown_complete;
 };
 
 struct nvme_driver {
@@ -967,6 +968,10 @@ int	nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid,
 int	nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr);
 void	nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr);
 void	nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr);
+void	nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr,
+				  struct nvme_ctrlr_detach_ctx *ctx);
+int	nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr,
+				       struct nvme_ctrlr_detach_ctx *ctx);
 void	nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove);
 int	nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr);
 int	nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr);