From d3ac42caa416f8797b8771134c11f9d7c8736537 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Fri, 21 Oct 2022 16:09:30 +0200 Subject: [PATCH] dma: add "virtual" accel memory domain This domain is meant to represent data being transformed by accel engine. Users will be able to allocate buffers from that memory domain and use them when appending operations to an accel sequence. Since these buffers are only meant to be used as placeholders for actual buffers, none of the push/pull/translate callbacks are implemented. To access the data after it was transformed by accel, users should make sure that the final command's destination buffer isn't allocated from accel memory domain. Signed-off-by: Konrad Sztyber Change-Id: Ia031c7b205e98792d0a93f01513101b86afa9faa Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15744 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- include/spdk/dma.h | 2 ++ lib/accel/accel.c | 22 ++++++++++++++++++++-- mk/spdk.lib_deps.mk | 2 +- test/unit/lib/accel/accel.c/accel_ut.c | 4 ++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/include/spdk/dma.h b/include/spdk/dma.h index 5a85f0734..161ce6616 100644 --- a/include/spdk/dma.h +++ b/include/spdk/dma.h @@ -30,6 +30,8 @@ enum spdk_dma_device_type { /** DMA devices are capable of performing DMA operations on memory domains using physical or * I/O virtual addresses. */ SPDK_DMA_DEVICE_TYPE_DMA, + /** Virtual memory domain representing memory being transformed by accel framework */ + SPDK_DMA_DEVICE_TYPE_ACCEL, /** * Start of the range of vendor-specific DMA device types */ diff --git a/lib/accel/accel.c b/lib/accel/accel.c index a0340e64f..42cdc4330 100644 --- a/lib/accel/accel.c +++ b/lib/accel/accel.c @@ -10,6 +10,7 @@ #include "accel_internal.h" +#include "spdk/dma.h" #include "spdk/env.h" #include "spdk/likely.h" #include "spdk/log.h" @@ -35,6 +36,7 @@ static struct spdk_accel_module_if *g_accel_module = NULL; static spdk_accel_fini_cb g_fini_cb_fn = NULL; static void *g_fini_cb_arg = NULL; static bool g_modules_started = false; +static struct spdk_memory_domain *g_accel_domain; /* Global list of registered accelerator modules */ static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list = @@ -1080,6 +1082,14 @@ spdk_accel_initialize(void) { enum accel_opcode op; struct spdk_accel_module_if *accel_module = NULL; + int rc; + + rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL, + "SPDK_ACCEL_DMA_DEVICE"); + if (rc != 0) { + SPDK_ERRLOG("Failed to create accel memory domain\n"); + return rc; + } g_modules_started = true; accel_module_initialize(); @@ -1105,11 +1115,13 @@ spdk_accel_initialize(void) accel_module = _module_find_by_name(g_modules_opc_override[op]); if (accel_module == NULL) { SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]); - return -EINVAL; + rc = -EINVAL; + goto error; } if (accel_module->supports_opcode(op) == false) { SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op); - return -EINVAL; + rc = -EINVAL; + goto error; } g_modules_opc[op] = accel_module; } @@ -1128,6 +1140,10 @@ spdk_accel_initialize(void) sizeof(struct accel_io_channel), "accel"); return 0; +error: + spdk_memory_domain_destroy(g_accel_domain); + + return rc; } static void @@ -1135,6 +1151,8 @@ accel_module_finish_cb(void) { spdk_accel_fini_cb cb_fn = g_fini_cb_fn; + spdk_memory_domain_destroy(g_accel_domain); + cb_fn(g_fini_cb_arg); g_fini_cb_fn = NULL; g_fini_cb_arg = NULL; diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index b1e6b66f6..915993451 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -47,7 +47,7 @@ DEPDIRS-nvme += rdma dma endif DEPDIRS-blob := log util thread dma -DEPDIRS-accel := log util thread json rpc jsonrpc +DEPDIRS-accel := log util thread json rpc jsonrpc dma DEPDIRS-jsonrpc := log util json DEPDIRS-virtio := log util json thread vfio_user diff --git a/test/unit/lib/accel/accel.c/accel_ut.c b/test/unit/lib/accel/accel.c/accel_ut.c index 1c6c45407..129921282 100644 --- a/test/unit/lib/accel/accel.c/accel_ut.c +++ b/test/unit/lib/accel/accel.c/accel_ut.c @@ -19,6 +19,10 @@ DEFINE_STUB(pmem_memcpy_persist, void *, (void *pmemdest, const void *src, size_ DEFINE_STUB(pmem_is_pmem, int, (const void *addr, size_t len), 0); DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL); #endif +DEFINE_STUB(spdk_memory_domain_create, int, + (struct spdk_memory_domain **domain, enum spdk_dma_device_type type, + struct spdk_memory_domain_ctx *ctx, const char *id), 0); +DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); /* global vars and setup/cleanup functions used for all test functions */ struct spdk_accel_module_if g_module = {};