copy_engine: add memory copy offload engine layer

Change-Id: I1aee004d6e0938972ece16eba6317d02ba3b0042
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>

include/spdk/copy_engine.h

@@ -0,0 +1,103 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * Memory copy offload engine abstraction layer
+ */
+
+#ifndef SPDK_COPY_ENGINE_H
+#define SPDK_COPY_ENGINE_H
+
+#include <stdio.h>
+#include <stdint.h>
+
+#include "spdk/queue.h"
+
+typedef void (*copy_completion_cb)(void *ref, int status);
+
+struct copy_task {
+	copy_completion_cb	cb;
+	uint8_t			offload_ctx[0];
+};
+
+struct spdk_copy_engine {
+	int64_t	(*copy)(void *cb_arg, void *dst, void *src,
+			uint64_t nbytes, copy_completion_cb cb);
+	void	(*check_io)(void);
+};
+
+struct spdk_copy_module_if {
+	/** Initialization function for the module.  Called by the spdk
+	 *   application during startup.
+	 *
+	 *  Modules are required to define this function.
+	 */
+	int	(*module_init)(void);
+
+	/** Finish function for the module.  Called by the spdk application
+	 *   before the spdk application exits to perform any necessary cleanup.
+	 *
+	 *  Modules are not required to define this function.
+	 */
+	void	(*module_fini)(void);
+
+	/** Function called to return a text string representing the
+	 *   module's configuration options for inclusion in an
+	 *   spdk configuration file.
+	 */
+	void	(*config_text)(FILE *fp);
+
+	int	(*get_ctx_size)(void);
+	TAILQ_ENTRY(spdk_copy_module_if)	tailq;
+};
+
+void spdk_copy_engine_register(struct spdk_copy_engine *copy_engine);
+int64_t spdk_copy_submit(struct copy_task *copy_req, void *dst, void *src,
+			 uint64_t nbytes, copy_completion_cb cb);
+int spdk_copy_check_io(void);
+int spdk_copy_module_get_max_ctx_size(void);
+void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module);
+
+#define SPDK_COPY_MODULE_REGISTER(init_fn, fini_fn, config_fn, ctx_size_fn)				\
+	static struct spdk_copy_module_if init_fn ## _if = {						\
+	.module_init 	= init_fn,									\
+	.module_fini	= fini_fn,									\
+	.config_text	= config_fn,									\
+	.get_ctx_size	= ctx_size_fn,                                					\
+	};  												\
+	__attribute__((constructor)) static void init_fn ## _init(void)  				\
+	{                                                           					\
+	    spdk_copy_module_list_add(&init_fn ## _if);                  				\
+	}
+
+#endif

lib/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-DIRS-y += conf cunit event json jsonrpc log memory rpc trace util nvme nvmf ioat
+DIRS-y += conf copy cunit event json jsonrpc log memory rpc trace util nvme nvmf ioat
 
 .PHONY: all clean $(DIRS-y)
 

lib/copy/Makefile

@@ -0,0 +1,41 @@
+#
+#  BSD LICENSE
+#
+#  Copyright (c) Intel Corporation.
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in
+#      the documentation and/or other materials provided with the
+#      distribution.
+#    * Neither the name of Intel Corporation nor the names of its
+#      contributors may be used to endorse or promote products derived
+#      from this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
+include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
+
+CFLAGS += $(DPDK_INC)
+LIBNAME = copy
+C_SRCS = copy_engine.c
+
+include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/copy/copy_engine.c

@@ -0,0 +1,233 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/copy_engine.h"
+
+#include <stdio.h>
+#include <errno.h>
+#include <rte_config.h>
+#include <rte_debug.h>
+#include <rte_malloc.h>
+#include <rte_memcpy.h>
+#include <rte_lcore.h>
+
+#include "spdk/log.h"
+#include "spdk/event.h"
+
+struct mem_request {
+	struct mem_request	*next;
+	copy_completion_cb	cb;
+};
+
+struct mem_request *copy_engine_req_head[RTE_MAX_LCORE];
+
+static struct spdk_copy_engine *hw_copy_engine = NULL;
+/* Memcpy engine always exist */
+static struct spdk_copy_engine *mem_copy_engine = NULL;
+
+TAILQ_HEAD(, spdk_copy_module_if) spdk_copy_module_list =
+	TAILQ_HEAD_INITIALIZER(spdk_copy_module_list);
+
+void
+spdk_copy_engine_register(struct spdk_copy_engine *copy_engine)
+{
+	RTE_VERIFY(hw_copy_engine == NULL);
+	hw_copy_engine = copy_engine;
+}
+
+static void
+spdk_memcpy_register(struct spdk_copy_engine *copy_engine)
+{
+	RTE_VERIFY(mem_copy_engine == NULL);
+	mem_copy_engine = copy_engine;
+}
+
+static int
+spdk_has_copy_engine(void)
+{
+	return (hw_copy_engine == NULL) ? 0 : 1;
+}
+
+int
+spdk_copy_check_io(void)
+{
+	if (spdk_has_copy_engine())
+		hw_copy_engine->check_io();
+	else
+		mem_copy_engine->check_io();
+
+	return 0;
+}
+
+static void
+copy_engine_done(void *ref, int status)
+{
+	struct copy_task *req = (struct copy_task *)ref;
+
+	req->cb(req, status);
+}
+
+int64_t
+spdk_copy_submit(struct copy_task *copy_req, void *dst, void *src,
+		 uint64_t nbytes, copy_completion_cb cb)
+{
+	struct copy_task *req = copy_req;
+
+	req->cb = cb;
+
+	if (spdk_has_copy_engine())
+		return hw_copy_engine->copy(req->offload_ctx, dst, src, nbytes,
+					    copy_engine_done);
+
+	return mem_copy_engine->copy(req->offload_ctx, dst, src, nbytes,
+				     copy_engine_done);
+}
+
+/* memcpy default copy engine */
+static void
+mem_copy_check_io(void)
+{
+	struct mem_request **req_head = &copy_engine_req_head[rte_lcore_id()];
+	struct mem_request *req = *req_head;
+	struct mem_request *req_next;
+	struct copy_task *copy_req;
+
+	*req_head = NULL;
+
+	while (req != NULL) {
+		req_next = req->next;
+		copy_req = (struct copy_task *)((uintptr_t)req -
+						offsetof(struct copy_task, offload_ctx));
+		req->cb((void *)copy_req, 0);
+		req = req_next;
+	}
+
+}
+
+static int64_t
+mem_copy_submit(void *cb_arg, void *dst, void *src, uint64_t nbytes,
+		copy_completion_cb cb)
+{
+	struct mem_request **req_head = &copy_engine_req_head[rte_lcore_id()];
+	struct mem_request *req = (struct mem_request *)cb_arg;
+
+	req->next = *req_head;
+	*req_head = req;
+	req->cb = cb;
+
+	rte_memcpy(dst, src, (size_t)nbytes);
+
+	return nbytes;
+}
+
+static struct spdk_copy_engine memcpy_copy_engine = {
+	.copy		= mem_copy_submit,
+	.check_io	= mem_copy_check_io,
+};
+
+static int
+copy_engine_mem_get_ctx_size(void)
+{
+	return sizeof(struct mem_request) + sizeof(struct copy_task);
+}
+
+int spdk_copy_module_get_max_ctx_size(void)
+{
+	struct spdk_copy_module_if *copy_engine;
+	int max_copy_module_size = 0;
+
+	TAILQ_FOREACH(copy_engine, &spdk_copy_module_list, tailq) {
+		if (copy_engine->get_ctx_size && copy_engine->get_ctx_size() > max_copy_module_size) {
+			max_copy_module_size = copy_engine->get_ctx_size();
+		}
+	}
+	return max_copy_module_size;
+}
+
+void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module)
+{
+	TAILQ_INSERT_TAIL(&spdk_copy_module_list, copy_module, tailq);
+}
+
+static int
+copy_engine_mem_init(void)
+{
+	int i;
+
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		copy_engine_req_head[i] = NULL;
+	}
+
+	/* Anyway, We will register memcpy engine */
+	spdk_memcpy_register(&memcpy_copy_engine);
+
+	return 0;
+}
+
+static void
+spdk_copy_engine_module_initialize(void)
+{
+	struct spdk_copy_module_if *copy_engine_module;
+
+	TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) {
+		copy_engine_module->module_init();
+	}
+}
+
+static void
+spdk_copy_engine_module_finish(void)
+{
+	struct spdk_copy_module_if *copy_engine_module;
+
+	TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) {
+		if (copy_engine_module->module_fini)
+			copy_engine_module->module_fini();
+	}
+}
+
+static int
+spdk_copy_engine_initialize(void)
+{
+	spdk_copy_engine_module_initialize();
+	return 0;
+}
+
+static int
+spdk_copy_engine_finish(void)
+{
+	spdk_copy_engine_module_finish();
+	return 0;
+}
+
+SPDK_COPY_MODULE_REGISTER(copy_engine_mem_init, NULL, NULL, copy_engine_mem_get_ctx_size)
+SPDK_SUBSYSTEM_REGISTER(copy, spdk_copy_engine_initialize, spdk_copy_engine_finish, NULL)