dma: add "virtual" accel memory domain

This domain is meant to represent data being transformed by accel
engine.  Users will be able to allocate buffers from that memory domain
and use them when appending operations to an accel sequence.

Since these buffers are only meant to be used as placeholders for actual
buffers, none of the push/pull/translate callbacks are implemented. To
access the data after it was transformed by accel, users should make
sure that the final command's destination buffer isn't allocated from
accel memory domain.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Ia031c7b205e98792d0a93f01513101b86afa9faa
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15744
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

include/spdk/dma.h

@@ -30,6 +30,8 @@ enum spdk_dma_device_type {
 	/** DMA devices are capable of performing DMA operations on memory domains using physical or
 	 *  I/O virtual addresses. */
 	SPDK_DMA_DEVICE_TYPE_DMA,
+	/** Virtual memory domain representing memory being transformed by accel framework */
+	SPDK_DMA_DEVICE_TYPE_ACCEL,
 	/**
 	 * Start of the range of vendor-specific DMA device types
 	 */

lib/accel/accel.c

@@ -10,6 +10,7 @@
 
 #include "accel_internal.h"
 
+#include "spdk/dma.h"
 #include "spdk/env.h"
 #include "spdk/likely.h"
 #include "spdk/log.h"
@@ -35,6 +36,7 @@ static struct spdk_accel_module_if *g_accel_module = NULL;
 static spdk_accel_fini_cb g_fini_cb_fn = NULL;
 static void *g_fini_cb_arg = NULL;
 static bool g_modules_started = false;
+static struct spdk_memory_domain *g_accel_domain;
 
 /* Global list of registered accelerator modules */
 static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
@@ -1080,6 +1082,14 @@ spdk_accel_initialize(void)
 {
 	enum accel_opcode op;
 	struct spdk_accel_module_if *accel_module = NULL;
+	int rc;
+
+	rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL,
+				       "SPDK_ACCEL_DMA_DEVICE");
+	if (rc != 0) {
+		SPDK_ERRLOG("Failed to create accel memory domain\n");
+		return rc;
+	}
 
 	g_modules_started = true;
 	accel_module_initialize();
@@ -1105,11 +1115,13 @@ spdk_accel_initialize(void)
 			accel_module = _module_find_by_name(g_modules_opc_override[op]);
 			if (accel_module == NULL) {
 				SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]);
-				return -EINVAL;
+				rc = -EINVAL;
+				goto error;
 			}
 			if (accel_module->supports_opcode(op) == false) {
 				SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op);
-				return -EINVAL;
+				rc = -EINVAL;
+				goto error;
 			}
 			g_modules_opc[op] = accel_module;
 		}
@@ -1128,6 +1140,10 @@ spdk_accel_initialize(void)
 				sizeof(struct accel_io_channel), "accel");
 
 	return 0;
+error:
+	spdk_memory_domain_destroy(g_accel_domain);
+
+	return rc;
 }
 
 static void
@@ -1135,6 +1151,8 @@ accel_module_finish_cb(void)
 {
 	spdk_accel_fini_cb cb_fn = g_fini_cb_fn;
 
+	spdk_memory_domain_destroy(g_accel_domain);
+
 	cb_fn(g_fini_cb_arg);
 	g_fini_cb_fn = NULL;
 	g_fini_cb_arg = NULL;

mk/spdk.lib_deps.mk

@@ -47,7 +47,7 @@ DEPDIRS-nvme += rdma dma
 endif
 
 DEPDIRS-blob := log util thread dma
-DEPDIRS-accel := log util thread json rpc jsonrpc
+DEPDIRS-accel := log util thread json rpc jsonrpc dma
 DEPDIRS-jsonrpc := log util json
 DEPDIRS-virtio := log util json thread vfio_user
 

test/unit/lib/accel/accel.c/accel_ut.c

@@ -19,6 +19,10 @@ DEFINE_STUB(pmem_memcpy_persist, void *, (void *pmemdest, const void *src, size_
 DEFINE_STUB(pmem_is_pmem, int, (const void *addr, size_t len), 0);
 DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL);
 #endif
+DEFINE_STUB(spdk_memory_domain_create, int,
+	    (struct spdk_memory_domain **domain, enum spdk_dma_device_type type,
+	     struct spdk_memory_domain_ctx *ctx, const char *id), 0);
+DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain));
 
 /* global vars and setup/cleanup functions used for all test functions */
 struct spdk_accel_module_if g_module = {};