From 2c13441ba847bd5aa7b6ced311a11fd7e36a7c1d Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Wed, 2 Mar 2022 08:47:25 +0900 Subject: [PATCH] nvme_rdma: Destroy qpair after qpair is actually disconnected The RDMA transport can disconnect qpair asynchronously now. Previously, we tried to release the resource of the qpair after disconnected. However it did not work because it was done when deleting the qpair. The admin qpair was not deleted in a ctrlr reset sequence. This patch tries to satisfy the same aim again but by a different way. Previously, we released the resource of the qpair before starting actual disconnection process. This patch release the resource of the qpair after the qpair is actually disconnected. The related patches are: b9518a554016ccc65903ce961cd2f8be437ed58c eb09178a59bac1679767eb7bc0d7c05da8d9b3ca Signed-off-by: Shuhei Matsumoto Change-Id: Id6a814895a35b1589b781a91744ef872b42aaa69 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11783 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Ben Walker --- lib/nvme/nvme_rdma.c | 56 +++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index 9b176e497..4dec5c178 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -1950,6 +1950,34 @@ nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr, static void nvme_rdma_qpair_destroy(struct nvme_rdma_qpair *rqpair) { + struct spdk_nvme_qpair *qpair = &rqpair->qpair; + struct nvme_rdma_ctrlr *rctrlr; + struct nvme_rdma_cm_event_entry *entry, *tmp; + + spdk_rdma_free_mem_map(&rqpair->mr_map); + nvme_rdma_unregister_reqs(rqpair); + nvme_rdma_unregister_rsps(rqpair); + + if (rqpair->evt) { + rdma_ack_cm_event(rqpair->evt); + rqpair->evt = NULL; + } + + /* + * This works because we have the controller lock both in + * this function and in the function where we add new events. + */ + if (qpair->ctrlr != NULL) { + rctrlr = nvme_rdma_ctrlr(qpair->ctrlr); + STAILQ_FOREACH_SAFE(entry, &rctrlr->pending_cm_events, link, tmp) { + if (nvme_rdma_qpair(entry->evt->id->context) == rqpair) { + STAILQ_REMOVE(&rctrlr->pending_cm_events, entry, nvme_rdma_cm_event_entry, link); + rdma_ack_cm_event(entry->evt); + STAILQ_INSERT_HEAD(&rctrlr->free_cm_events, entry, link); + } + } + } + if (rqpair->cm_id) { if (rqpair->rdma_qp) { spdk_rdma_qp_destroy(rqpair->rdma_qp); @@ -2021,42 +2049,16 @@ _nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvm nvme_rdma_cm_event_cb disconnected_qpair_cb) { struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair); - struct nvme_rdma_ctrlr *rctrlr = NULL; - struct nvme_rdma_cm_event_entry *entry, *tmp; int rc; assert(disconnected_qpair_cb != NULL); rqpair->state = NVME_RDMA_QPAIR_STATE_EXITING; - spdk_rdma_free_mem_map(&rqpair->mr_map); - nvme_rdma_unregister_reqs(rqpair); - nvme_rdma_unregister_rsps(rqpair); - - if (rqpair->evt) { - rdma_ack_cm_event(rqpair->evt); - rqpair->evt = NULL; - } - - /* - * This works because we have the controller lock both in - * this function and in the function where we add new events. - */ - if (qpair->ctrlr != NULL) { - rctrlr = nvme_rdma_ctrlr(qpair->ctrlr); - STAILQ_FOREACH_SAFE(entry, &rctrlr->pending_cm_events, link, tmp) { - if (nvme_rdma_qpair(entry->evt->id->context) == rqpair) { - STAILQ_REMOVE(&rctrlr->pending_cm_events, entry, nvme_rdma_cm_event_entry, link); - rdma_ack_cm_event(entry->evt); - STAILQ_INSERT_HEAD(&rctrlr->free_cm_events, entry, link); - } - } - } - if (rqpair->cm_id) { if (rqpair->rdma_qp) { rc = spdk_rdma_qp_disconnect(rqpair->rdma_qp); - if ((rctrlr != NULL) && (rc == 0)) { + if ((qpair->ctrlr != NULL) && (rc == 0)) { rc = nvme_rdma_process_event_start(rqpair, RDMA_CM_EVENT_DISCONNECTED, disconnected_qpair_cb); if (rc == 0) {