From ccd96ead65047706151a938c9d7fa9d79024bd48 Mon Sep 17 00:00:00 2001 From: Naresh Gottumukkala Date: Fri, 13 Nov 2020 07:23:42 +0000 Subject: [PATCH] nvmf: Allow nvmf_transport_qpair_fini to complete asynchronously. As part of nvmf_transport_qpair_fini, FC transport needs to cleanup all the resources used by that QPair on the shared hardware. This hardware cleanup is asynchronous in nature. FC transport code to use this functionality will be pushed shortly. Change-Id: I5606a33dff45971badd74e0cc087b132b56af076 Signed-off-by: Naresh Gottumukkala Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5100 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Aleksey Marchuk Reviewed-by: Jim Harris Reviewed-by: Anil Veerabhadrappa --- CHANGELOG.md | 3 +++ include/spdk/nvmf_transport.h | 6 +++++- lib/nvmf/Makefile | 2 +- lib/nvmf/fc.c | 9 +++++++-- lib/nvmf/nvmf.c | 29 ++++++++++++++++------------- lib/nvmf/rdma.c | 7 ++++++- lib/nvmf/tcp.c | 7 ++++++- lib/nvmf/transport.c | 6 ++++-- lib/nvmf/transport.h | 3 ++- 9 files changed, 50 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b23b68809..937532fdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### nvmf +The function `qpair_fini` in the transport interface now accepts a cb_fn and +cb_arg to call upon completion, and its execution can be asynchronous. + The SPDK nvmf target now supports async event notification for discovery log changes. This allows the initiator to create persistent connection to discovery controller and be notified of any discovery log changes. diff --git a/include/spdk/nvmf_transport.h b/include/spdk/nvmf_transport.h index a4cb0c50a..21c6301a5 100644 --- a/include/spdk/nvmf_transport.h +++ b/include/spdk/nvmf_transport.h @@ -197,6 +197,8 @@ struct spdk_nvmf_transport { TAILQ_ENTRY(spdk_nvmf_transport) link; }; +typedef void (*spdk_nvmf_transport_qpair_fini_cb)(void *cb_arg); + struct spdk_nvmf_transport_ops { /** * Transport name @@ -325,7 +327,9 @@ struct spdk_nvmf_transport_ops { /* * Deinitialize a connection. */ - void (*qpair_fini)(struct spdk_nvmf_qpair *qpair); + void (*qpair_fini)(struct spdk_nvmf_qpair *qpair, + spdk_nvmf_transport_qpair_fini_cb cb_fn, + void *cb_args); /* * Get the peer transport ID for the queue pair. diff --git a/lib/nvmf/Makefile b/lib/nvmf/Makefile index 13f72822c..8585fe4c3 100644 --- a/lib/nvmf/Makefile +++ b/lib/nvmf/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -SO_VER := 6 +SO_VER := 7 SO_MINOR := 0 C_SRCS = ctrlr.c ctrlr_discovery.c ctrlr_bdev.c \ diff --git a/lib/nvmf/fc.c b/lib/nvmf/fc.c index f595bc89f..40028602e 100644 --- a/lib/nvmf/fc.c +++ b/lib/nvmf/fc.c @@ -2049,12 +2049,13 @@ nvmf_fc_request_free(struct spdk_nvmf_request *req) } else { nvmf_fc_request_abort_complete(fc_req); } + return 0; } - static void -nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair) +nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair, + spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg) { struct spdk_nvmf_fc_conn *fc_conn; @@ -2070,6 +2071,10 @@ nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair) spdk_thread_send_msg(nvmf_fc_get_master_thread(), nvmf_fc_handle_assoc_deletion, fc_conn); } + + if (cb_fn) { + cb_fn(cb_arg); + } } static int diff --git a/lib/nvmf/nvmf.c b/lib/nvmf/nvmf.c index b220d2b4b..700c2e62d 100644 --- a/lib/nvmf/nvmf.c +++ b/lib/nvmf/nvmf.c @@ -869,12 +869,23 @@ _nvmf_ctrlr_destruct(void *ctx) nvmf_ctrlr_destruct(ctrlr); } +static void +_nvmf_transport_qpair_fini_complete(void *cb_ctx) +{ + struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; + + if (qpair_ctx->cb_fn) { + spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); + } + free(qpair_ctx); +} + static void _nvmf_transport_qpair_fini(void *ctx) { - struct spdk_nvmf_qpair *qpair = ctx; + struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; - nvmf_transport_qpair_fini(qpair); + nvmf_transport_qpair_fini(qpair_ctx->qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); } static void @@ -891,11 +902,7 @@ _nvmf_ctrlr_free_from_qpair(void *ctx) spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); } - spdk_thread_send_msg(qpair_ctx->thread, _nvmf_transport_qpair_fini, qpair_ctx->qpair); - if (qpair_ctx->cb_fn) { - spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); - } - free(qpair_ctx); + spdk_thread_send_msg(qpair_ctx->thread, _nvmf_transport_qpair_fini, qpair_ctx); } void @@ -950,11 +957,7 @@ _nvmf_qpair_destroy(void *ctx, int status) spdk_nvmf_poll_group_remove(qpair); if (!ctrlr || !ctrlr->thread) { - nvmf_transport_qpair_fini(qpair); - if (qpair_ctx->cb_fn) { - spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); - } - free(qpair_ctx); + nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); return; } @@ -985,7 +988,7 @@ spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_ /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { - nvmf_transport_qpair_fini(qpair); + nvmf_transport_qpair_fini(qpair, NULL, NULL); if (cb_fn) { cb_fn(ctx); } diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 0324a59b1..b7105c1f4 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -3633,7 +3633,8 @@ nvmf_rdma_request_complete(struct spdk_nvmf_request *req) } static void -nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair) +nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair, + spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg) { struct spdk_nvmf_rdma_qpair *rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair); @@ -3655,6 +3656,10 @@ nvmf_rdma_close_qpair(struct spdk_nvmf_qpair *qpair) } nvmf_rdma_destroy_drained_qpair(rqpair); + + if (cb_fn) { + cb_fn(cb_arg); + } } static struct spdk_nvmf_rdma_qpair * diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index 5ed38d58d..4fab57249 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -2565,7 +2565,8 @@ nvmf_tcp_req_complete(struct spdk_nvmf_request *req) } static void -nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair) +nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair, + spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg) { struct spdk_nvmf_tcp_qpair *tqpair; @@ -2574,6 +2575,10 @@ nvmf_tcp_close_qpair(struct spdk_nvmf_qpair *qpair) tqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_tcp_qpair, qpair); tqpair->state = NVME_TCP_QPAIR_STATE_EXITED; nvmf_tcp_qpair_destroy(tqpair); + + if (cb_fn) { + cb_fn(cb_arg); + } } static int diff --git a/lib/nvmf/transport.c b/lib/nvmf/transport.c index 5c2e2b64a..c99a86018 100644 --- a/lib/nvmf/transport.c +++ b/lib/nvmf/transport.c @@ -473,9 +473,11 @@ nvmf_transport_req_complete(struct spdk_nvmf_request *req) } void -nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair) +nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair, + spdk_nvmf_transport_qpair_fini_cb cb_fn, + void *cb_arg) { - qpair->transport->ops->qpair_fini(qpair); + qpair->transport->ops->qpair_fini(qpair, cb_fn, cb_arg); } int diff --git a/lib/nvmf/transport.h b/lib/nvmf/transport.h index 38b5d8db3..9a4ae56ff 100644 --- a/lib/nvmf/transport.h +++ b/lib/nvmf/transport.h @@ -65,7 +65,8 @@ int nvmf_transport_req_free(struct spdk_nvmf_request *req); int nvmf_transport_req_complete(struct spdk_nvmf_request *req); -void nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair); +void nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair, + spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg); int nvmf_transport_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, struct spdk_nvme_transport_id *trid);