From d75daea5321079e8d592c35d2ee2ab3f44079e1d Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Mon, 25 Jul 2022 14:48:21 +0900 Subject: [PATCH] nvme_rdma: Use persistent protection domain for qpair Get a PD for the device from the PD pool managed by the RDMA provider when creating a QP, and put the PD when destroying the PD. By this change, PD is managed completely by the RDMA provider or the hooks. nvme_rdma_ctrlr::pd was added long time ago but is not referenced anywhere. Remove nvme_rdma_ctrlr::pd for cleanup and clarification. Signed-off-by: Shuhei Matsumoto Change-Id: If8dc8ad011eed70149012128bd1b33f1a8b7b90b Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13770 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Aleksey Marchuk --- lib/nvme/nvme_rdma.c | 10 +++------- test/common/lib/test_rdma.c | 10 ++++++++++ test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c | 3 +++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index 7bf6e7d51..d86c80677 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -113,8 +113,6 @@ struct nvme_rdma_cm_event_entry { struct nvme_rdma_ctrlr { struct spdk_nvme_ctrlr ctrlr; - struct ibv_pd *pd; - uint16_t max_sge; struct rdma_event_channel *cm_channel; @@ -764,12 +762,11 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) rctrlr = nvme_rdma_ctrlr(rqpair->qpair.ctrlr); if (g_nvme_hooks.get_ibv_pd) { - rctrlr->pd = g_nvme_hooks.get_ibv_pd(&rctrlr->ctrlr.trid, rqpair->cm_id->verbs); + attr.pd = g_nvme_hooks.get_ibv_pd(&rctrlr->ctrlr.trid, rqpair->cm_id->verbs); } else { - rctrlr->pd = NULL; + attr.pd = spdk_rdma_get_pd(rqpair->cm_id->verbs); } - attr.pd = rctrlr->pd; attr.stats = rqpair->poller ? &rqpair->poller->stats.rdma_stats : NULL; attr.send_cq = rqpair->cq; attr.recv_cq = rqpair->cq; @@ -796,8 +793,6 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) rqpair->current_num_recvs = 0; rqpair->current_num_sends = 0; - rctrlr->pd = rqpair->rdma_qp->qp->pd; - rqpair->cm_id->context = rqpair; return 0; @@ -1963,6 +1958,7 @@ nvme_rdma_qpair_destroy(struct nvme_rdma_qpair *rqpair) if (rqpair->cm_id) { if (rqpair->rdma_qp) { + spdk_rdma_put_pd(rqpair->rdma_qp->qp->pd); spdk_rdma_qp_destroy(rqpair->rdma_qp); rqpair->rdma_qp = NULL; } diff --git a/test/common/lib/test_rdma.c b/test/common/lib/test_rdma.c index f240393dc..dba07f87c 100644 --- a/test/common/lib/test_rdma.c +++ b/test/common/lib/test_rdma.c @@ -65,3 +65,13 @@ spdk_rdma_get_translation(struct spdk_rdma_mem_map *map, void *address, return 0; } + +DEFINE_RETURN_MOCK(spdk_rdma_get_pd, struct ibv_pd *); +struct ibv_pd * +spdk_rdma_get_pd(struct ibv_context *context) +{ + HANDLE_RETURN_MOCK(spdk_rdma_get_pd); + return NULL; +} + +DEFINE_STUB_V(spdk_rdma_put_pd, (struct ibv_pd *pd)); diff --git a/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c b/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c index 3e8707beb..a3a8e96e2 100644 --- a/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c +++ b/test/unit/lib/nvme/nvme_rdma.c/nvme_rdma_ut.c @@ -1076,6 +1076,7 @@ test_nvme_rdma_qpair_init(void) rqpair.qpair.poll_group = NULL; rqpair.qpair.ctrlr = &rctrlr.ctrlr; g_spdk_rdma_qp.qp = &qp; + MOCK_SET(spdk_rdma_get_pd, pd); rc = nvme_rdma_qpair_init(&rqpair); CU_ASSERT(rc == 0); @@ -1087,6 +1088,8 @@ test_nvme_rdma_qpair_init(void) CU_ASSERT(rqpair.current_num_recvs == 0); CU_ASSERT(rqpair.cq == (struct ibv_cq *)0xFEEDBEEF); CU_ASSERT(rqpair.memory_domain != NULL); + + MOCK_CLEAR(spdk_rdma_get_pd); } static void