From 549bcdc0a4a83fbfe8bc5c0f438b43fc57bacb0d Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Wed, 6 Oct 2021 17:59:44 +0300 Subject: [PATCH] dma: Update memory domain context structure Instead of a union with domain type specific parameters, store an opaque pointer to user context. Depending on the memory domain type, this context can be cast to a specific struct, e.g. to spdk_memory_domain_rdma_ctx for RDMA memory domains. This change provides more flexibility to applications to create and manage custom memory domains Signed-off-by: Alexey Marchuk Signed-off-by: Max Gurtovoy Change-Id: Ib0a8297de80773d86edc9849beb4cbc693ef5414 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9778 Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- include/spdk/dma.h | 21 ++++++++++++++------- lib/nvme/nvme_rdma.c | 11 +++++++---- test/unit/lib/dma/dma.c/dma_ut.c | 9 +++++---- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/include/spdk/dma.h b/include/spdk/dma.h index f9b27d3f8..545837043 100644 --- a/include/spdk/dma.h +++ b/include/spdk/dma.h @@ -162,15 +162,21 @@ typedef int (*spdk_memory_domain_translate_memory_cb)(struct spdk_memory_domain struct spdk_memory_domain_translation_ctx *dst_domain_ctx, void *addr, size_t len, struct spdk_memory_domain_translation_result *result); +/** Context of memory domain of RDMA type */ +struct spdk_memory_domain_rdma_ctx { + /** size of this structure in bytes */ + size_t size; + /** Opaque handle for ibv_pd */ + void *ibv_pd; +}; + struct spdk_memory_domain_ctx { /** size of this structure in bytes */ size_t size; - union { - struct { - /* Opaque handle for ibv_pd */ - void *ibv_pd; - } rdma; - }; + /** Optional user context + * Depending on memory domain type, this pointer can be cast to a specific structure, + * e.g. to spdk_memory_domain_rdma_ctx structure for RDMA memory domain */ + void *user_ctx; }; /** @@ -183,7 +189,8 @@ struct spdk_memory_domain_ctx { * * \param domain Double pointer to memory domain to be allocated by this function * \param type Type of the DMA device which can access this memory domain - * \param ctx Optional memory domain context + * \param ctx Optional memory domain context to be copied by this function. Later \b ctx can be + * retrieved using \ref spdk_memory_domain_get_context function * \param id String identifier representing the DMA device that can access this memory domain. * \return 0 on success, negated errno on failure */ diff --git a/lib/nvme/nvme_rdma.c b/lib/nvme/nvme_rdma.c index 21bcfaa05..904085ef7 100644 --- a/lib/nvme/nvme_rdma.c +++ b/lib/nvme/nvme_rdma.c @@ -112,6 +112,7 @@ struct nvme_rdma_memory_domain { uint32_t ref; struct ibv_pd *pd; struct spdk_memory_domain *domain; + struct spdk_memory_domain_rdma_ctx rdma_ctx; }; enum nvme_rdma_wr_type { @@ -318,7 +319,7 @@ static struct nvme_rdma_memory_domain * nvme_rdma_get_memory_domain(struct ibv_pd *pd) { struct nvme_rdma_memory_domain *domain = NULL; - struct spdk_memory_domain_ctx dev_ctx; + struct spdk_memory_domain_ctx ctx; int rc; pthread_mutex_lock(&g_memory_domains_lock); @@ -338,10 +339,12 @@ nvme_rdma_get_memory_domain(struct ibv_pd *pd) return NULL; } - dev_ctx.size = sizeof(dev_ctx); - dev_ctx.rdma.ibv_pd = pd; + domain->rdma_ctx.size = sizeof(domain->rdma_ctx); + domain->rdma_ctx.ibv_pd = pd; + ctx.size = sizeof(ctx); + ctx.user_ctx = &domain->rdma_ctx; - rc = spdk_memory_domain_create(&domain->domain, SPDK_DMA_DEVICE_TYPE_RDMA, &dev_ctx, + rc = spdk_memory_domain_create(&domain->domain, SPDK_DMA_DEVICE_TYPE_RDMA, &ctx, SPDK_RDMA_DMA_DEVICE); if (rc) { SPDK_ERRLOG("Failed to create memory domain\n"); diff --git a/test/unit/lib/dma/dma.c/dma_ut.c b/test/unit/lib/dma/dma.c/dma_ut.c index fa93b0732..059982bee 100644 --- a/test/unit/lib/dma/dma.c/dma_ut.c +++ b/test/unit/lib/dma/dma.c/dma_ut.c @@ -81,9 +81,8 @@ test_dma(void) void *test_ibv_pd = (void *)0xdeadbeaf; struct iovec src_iov = {}, dst_iov = {}; struct spdk_memory_domain *domain = NULL, *domain_2 = NULL, *domain_3 = NULL; - struct spdk_memory_domain_ctx memory_domain_ctx = { - .rdma = { .ibv_pd = test_ibv_pd } - }; + struct spdk_memory_domain_rdma_ctx rdma_ctx = { .ibv_pd = test_ibv_pd }; + struct spdk_memory_domain_ctx memory_domain_ctx = { .user_ctx = &rdma_ctx }; struct spdk_memory_domain_ctx *stored_memory_domain_ctx; struct spdk_memory_domain_translation_result translation_result; const char *id; @@ -107,7 +106,9 @@ test_dma(void) /* Get context. Expect pass */ stored_memory_domain_ctx = spdk_memory_domain_get_context(domain); SPDK_CU_ASSERT_FATAL(stored_memory_domain_ctx != NULL); - CU_ASSERT(stored_memory_domain_ctx->rdma.ibv_pd == test_ibv_pd); + CU_ASSERT(stored_memory_domain_ctx->user_ctx == &rdma_ctx); + CU_ASSERT(((struct spdk_memory_domain_rdma_ctx *)stored_memory_domain_ctx->user_ctx)->ibv_pd == + rdma_ctx.ibv_pd); /* Get DMA device type. Expect pass */ CU_ASSERT(spdk_memory_domain_get_dma_device_type(domain) == SPDK_DMA_DEVICE_TYPE_RDMA);