nvmf: Allow nvmf_transport_qpair_fini to complete asynchronously.

As part of nvmf_transport_qpair_fini, FC transport needs to cleanup
all the resources used by that QPair on the shared hardware. This
hardware cleanup is asynchronous in nature.

FC transport code to use this functionality will be pushed shortly.

Change-Id: I5606a33dff45971badd74e0cc087b132b56af076
Signed-off-by: Naresh Gottumukkala <raju.gottumukkala@broadcom.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5100
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Anil Veerabhadrappa <anil.veerabhadrappa@broadcom.com>

CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### nvmf
 
+The function `qpair_fini` in the transport interface now accepts a cb_fn and
+cb_arg to call upon completion, and its execution can be asynchronous.
+
 The SPDK nvmf target now supports async event notification for discovery log changes.
 This allows the initiator to create persistent connection to discovery controller and
 be notified of any discovery log changes.

include/spdk/nvmf_transport.h

@@ -197,6 +197,8 @@ struct spdk_nvmf_transport {
 	TAILQ_ENTRY(spdk_nvmf_transport)	link;
 };
 
+typedef void (*spdk_nvmf_transport_qpair_fini_cb)(void *cb_arg);
+
 struct spdk_nvmf_transport_ops {
 	/**
 	 * Transport name
@@ -325,7 +327,9 @@ struct spdk_nvmf_transport_ops {
 	/*
 	 * Deinitialize a connection.
 	 */
-	void (*qpair_fini)(struct spdk_nvmf_qpair *qpair);
+	void (*qpair_fini)(struct spdk_nvmf_qpair *qpair,
+			   spdk_nvmf_transport_qpair_fini_cb cb_fn,
+			   void *cb_args);
 
 	/*
 	 * Get the peer transport ID for the queue pair.

lib/nvmf/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 6
+SO_VER := 7
 SO_MINOR := 0
 
 C_SRCS = ctrlr.c ctrlr_discovery.c ctrlr_bdev.c \

lib/nvmf/fc.c

@@ -2049,12 +2049,13 @@ nvmf_fc_request_free(struct spdk_nvmf_request *req)
 	} else {
 		nvmf_fc_request_abort_complete(fc_req);
 	}
+
 	return 0;
 }
 
-
 static void
-nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair)
+nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair,
+		    spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg)
 {
 	struct spdk_nvmf_fc_conn *fc_conn;
 
@@ -2070,6 +2071,10 @@ nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair)
 		spdk_thread_send_msg(nvmf_fc_get_master_thread(),
 				     nvmf_fc_handle_assoc_deletion, fc_conn);
 	}
+
+	if (cb_fn) {
+		cb_fn(cb_arg);
+	}
 }
 
 static int

lib/nvmf/nvmf.c

@@ -869,12 +869,23 @@ _nvmf_ctrlr_destruct(void *ctx)
 	nvmf_ctrlr_destruct(ctrlr);
 }
 
+static void
+_nvmf_transport_qpair_fini_complete(void *cb_ctx)
+{
+	struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx;
+
+	if (qpair_ctx->cb_fn) {
+		spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx);
+	}
+	free(qpair_ctx);
+}
+
 static void
 _nvmf_transport_qpair_fini(void *ctx)
 {
-	struct spdk_nvmf_qpair *qpair = ctx;
+	struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;
 
-	nvmf_transport_qpair_fini(qpair);
+	nvmf_transport_qpair_fini(qpair_ctx->qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx);
 }
 
 static void
@@ -891,11 +902,7 @@ _nvmf_ctrlr_free_from_qpair(void *ctx)
 		spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr);
 	}
 
-	spdk_thread_send_msg(qpair_ctx->thread, _nvmf_transport_qpair_fini, qpair_ctx->qpair);
+	spdk_thread_send_msg(qpair_ctx->thread, _nvmf_transport_qpair_fini, qpair_ctx);
-	if (qpair_ctx->cb_fn) {
-		spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx);
-	}
-	free(qpair_ctx);
 }
 
 void
@@ -950,11 +957,7 @@ _nvmf_qpair_destroy(void *ctx, int status)
 	spdk_nvmf_poll_group_remove(qpair);
 
 	if (!ctrlr || !ctrlr->thread) {
-		nvmf_transport_qpair_fini(qpair);
+		nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx);
-		if (qpair_ctx->cb_fn) {
-			spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx);
-		}
-		free(qpair_ctx);
 		return;
 	}
 
@@ -985,7 +988,7 @@ spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_
 
 	/* If we get a qpair in the uninitialized state, we can just destroy it immediately */
 	if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) {
-		nvmf_transport_qpair_fini(qpair);
+		nvmf_transport_qpair_fini(qpair, NULL, NULL);
 		if (cb_fn) {
 			cb_fn(ctx);
 		}

lib/nvmf/rdma.c

@@ -3633,7 +3633,8 @@ nvmf_rdma_request_complete(struct spdk_nvmf_request *req)
 }
 
 static void
-nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair)
+nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair,
+		      spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg)
 {
 	struct spdk_nvmf_rdma_qpair *rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair);
 
@@ -3655,6 +3656,10 @@ nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair)
 	}
 
 	nvmf_rdma_destroy_drained_qpair(rqpair);
+
+	if (cb_fn) {
+		cb_fn(cb_arg);
+	}
 }
 
 static struct spdk_nvmf_rdma_qpair *

lib/nvmf/tcp.c

@@ -2565,7 +2565,8 @@ nvmf_tcp_req_complete(struct spdk_nvmf_request *req)
 }
 
 static void
-nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair)
+nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair,
+		     spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg)
 {
 	struct spdk_nvmf_tcp_qpair *tqpair;
 
@@ -2574,6 +2575,10 @@ nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair)
 	tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair);
 	tqpair->state = NVME_TCP_QPAIR_STATE_EXITED;
 	nvmf_tcp_qpair_destroy(tqpair);
+
+	if (cb_fn) {
+		cb_fn(cb_arg);
+	}
 }
 
 static int

lib/nvmf/transport.c

@@ -473,9 +473,11 @@ nvmf_transport_req_complete(struct spdk_nvmf_request *req)
 }
 
 void
-nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair)
+nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair,
+			  spdk_nvmf_transport_qpair_fini_cb cb_fn,
+			  void *cb_arg)
 {
-	qpair->transport->ops->qpair_fini(qpair);
+	qpair->transport->ops->qpair_fini(qpair, cb_fn, cb_arg);
 }
 
 int

lib/nvmf/transport.h

@@ -65,7 +65,8 @@ int nvmf_transport_req_free(struct spdk_nvmf_request *req);
 
 int nvmf_transport_req_complete(struct spdk_nvmf_request *req);
 
-void nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair);
+void nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair,
+			       spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg);
 
 int nvmf_transport_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair,
 				       struct spdk_nvme_transport_id *trid);