copy_engine: add memory copy offload engine layer
Change-Id: I1aee004d6e0938972ece16eba6317d02ba3b0042 Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp
parent
07697ff1d2
commit
746dc47d62
103
include/spdk/copy_engine.h
Normal file
103
include/spdk/copy_engine.h
Normal file
@ -0,0 +1,103 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
/** \file
|
||||
* Memory copy offload engine abstraction layer
|
||||
*/
|
||||
|
||||
#ifndef SPDK_COPY_ENGINE_H
|
||||
#define SPDK_COPY_ENGINE_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "spdk/queue.h"
|
||||
|
||||
typedef void (*copy_completion_cb)(void *ref, int status);
|
||||
|
||||
struct copy_task {
|
||||
copy_completion_cb cb;
|
||||
uint8_t offload_ctx[0];
|
||||
};
|
||||
|
||||
struct spdk_copy_engine {
|
||||
int64_t (*copy)(void *cb_arg, void *dst, void *src,
|
||||
uint64_t nbytes, copy_completion_cb cb);
|
||||
void (*check_io)(void);
|
||||
};
|
||||
|
||||
struct spdk_copy_module_if {
|
||||
/** Initialization function for the module. Called by the spdk
|
||||
* application during startup.
|
||||
*
|
||||
* Modules are required to define this function.
|
||||
*/
|
||||
int (*module_init)(void);
|
||||
|
||||
/** Finish function for the module. Called by the spdk application
|
||||
* before the spdk application exits to perform any necessary cleanup.
|
||||
*
|
||||
* Modules are not required to define this function.
|
||||
*/
|
||||
void (*module_fini)(void);
|
||||
|
||||
/** Function called to return a text string representing the
|
||||
* module's configuration options for inclusion in an
|
||||
* spdk configuration file.
|
||||
*/
|
||||
void (*config_text)(FILE *fp);
|
||||
|
||||
int (*get_ctx_size)(void);
|
||||
TAILQ_ENTRY(spdk_copy_module_if) tailq;
|
||||
};
|
||||
|
||||
void spdk_copy_engine_register(struct spdk_copy_engine *copy_engine);
|
||||
int64_t spdk_copy_submit(struct copy_task *copy_req, void *dst, void *src,
|
||||
uint64_t nbytes, copy_completion_cb cb);
|
||||
int spdk_copy_check_io(void);
|
||||
int spdk_copy_module_get_max_ctx_size(void);
|
||||
void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module);
|
||||
|
||||
#define SPDK_COPY_MODULE_REGISTER(init_fn, fini_fn, config_fn, ctx_size_fn) \
|
||||
static struct spdk_copy_module_if init_fn ## _if = { \
|
||||
.module_init = init_fn, \
|
||||
.module_fini = fini_fn, \
|
||||
.config_text = config_fn, \
|
||||
.get_ctx_size = ctx_size_fn, \
|
||||
}; \
|
||||
__attribute__((constructor)) static void init_fn ## _init(void) \
|
||||
{ \
|
||||
spdk_copy_module_list_add(&init_fn ## _if); \
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -34,7 +34,7 @@
|
||||
SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
|
||||
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
|
||||
|
||||
DIRS-y += conf cunit event json jsonrpc log memory rpc trace util nvme nvmf ioat
|
||||
DIRS-y += conf copy cunit event json jsonrpc log memory rpc trace util nvme nvmf ioat
|
||||
|
||||
.PHONY: all clean $(DIRS-y)
|
||||
|
||||
|
||||
41
lib/copy/Makefile
Normal file
41
lib/copy/Makefile
Normal file
@ -0,0 +1,41 @@
|
||||
#
|
||||
# BSD LICENSE
|
||||
#
|
||||
# Copyright (c) Intel Corporation.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in
|
||||
# the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
# * Neither the name of Intel Corporation nor the names of its
|
||||
# contributors may be used to endorse or promote products derived
|
||||
# from this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
|
||||
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
|
||||
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
|
||||
|
||||
CFLAGS += $(DPDK_INC)
|
||||
LIBNAME = copy
|
||||
C_SRCS = copy_engine.c
|
||||
|
||||
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk
|
||||
233
lib/copy/copy_engine.c
Normal file
233
lib/copy/copy_engine.c
Normal file
@ -0,0 +1,233 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "spdk/copy_engine.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
#include <rte_config.h>
|
||||
#include <rte_debug.h>
|
||||
#include <rte_malloc.h>
|
||||
#include <rte_memcpy.h>
|
||||
#include <rte_lcore.h>
|
||||
|
||||
#include "spdk/log.h"
|
||||
#include "spdk/event.h"
|
||||
|
||||
struct mem_request {
|
||||
struct mem_request *next;
|
||||
copy_completion_cb cb;
|
||||
};
|
||||
|
||||
struct mem_request *copy_engine_req_head[RTE_MAX_LCORE];
|
||||
|
||||
static struct spdk_copy_engine *hw_copy_engine = NULL;
|
||||
/* Memcpy engine always exist */
|
||||
static struct spdk_copy_engine *mem_copy_engine = NULL;
|
||||
|
||||
TAILQ_HEAD(, spdk_copy_module_if) spdk_copy_module_list =
|
||||
TAILQ_HEAD_INITIALIZER(spdk_copy_module_list);
|
||||
|
||||
void
|
||||
spdk_copy_engine_register(struct spdk_copy_engine *copy_engine)
|
||||
{
|
||||
RTE_VERIFY(hw_copy_engine == NULL);
|
||||
hw_copy_engine = copy_engine;
|
||||
}
|
||||
|
||||
static void
|
||||
spdk_memcpy_register(struct spdk_copy_engine *copy_engine)
|
||||
{
|
||||
RTE_VERIFY(mem_copy_engine == NULL);
|
||||
mem_copy_engine = copy_engine;
|
||||
}
|
||||
|
||||
static int
|
||||
spdk_has_copy_engine(void)
|
||||
{
|
||||
return (hw_copy_engine == NULL) ? 0 : 1;
|
||||
}
|
||||
|
||||
int
|
||||
spdk_copy_check_io(void)
|
||||
{
|
||||
if (spdk_has_copy_engine())
|
||||
hw_copy_engine->check_io();
|
||||
else
|
||||
mem_copy_engine->check_io();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
copy_engine_done(void *ref, int status)
|
||||
{
|
||||
struct copy_task *req = (struct copy_task *)ref;
|
||||
|
||||
req->cb(req, status);
|
||||
}
|
||||
|
||||
int64_t
|
||||
spdk_copy_submit(struct copy_task *copy_req, void *dst, void *src,
|
||||
uint64_t nbytes, copy_completion_cb cb)
|
||||
{
|
||||
struct copy_task *req = copy_req;
|
||||
|
||||
req->cb = cb;
|
||||
|
||||
if (spdk_has_copy_engine())
|
||||
return hw_copy_engine->copy(req->offload_ctx, dst, src, nbytes,
|
||||
copy_engine_done);
|
||||
|
||||
return mem_copy_engine->copy(req->offload_ctx, dst, src, nbytes,
|
||||
copy_engine_done);
|
||||
}
|
||||
|
||||
/* memcpy default copy engine */
|
||||
static void
|
||||
mem_copy_check_io(void)
|
||||
{
|
||||
struct mem_request **req_head = ©_engine_req_head[rte_lcore_id()];
|
||||
struct mem_request *req = *req_head;
|
||||
struct mem_request *req_next;
|
||||
struct copy_task *copy_req;
|
||||
|
||||
*req_head = NULL;
|
||||
|
||||
while (req != NULL) {
|
||||
req_next = req->next;
|
||||
copy_req = (struct copy_task *)((uintptr_t)req -
|
||||
offsetof(struct copy_task, offload_ctx));
|
||||
req->cb((void *)copy_req, 0);
|
||||
req = req_next;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static int64_t
|
||||
mem_copy_submit(void *cb_arg, void *dst, void *src, uint64_t nbytes,
|
||||
copy_completion_cb cb)
|
||||
{
|
||||
struct mem_request **req_head = ©_engine_req_head[rte_lcore_id()];
|
||||
struct mem_request *req = (struct mem_request *)cb_arg;
|
||||
|
||||
req->next = *req_head;
|
||||
*req_head = req;
|
||||
req->cb = cb;
|
||||
|
||||
rte_memcpy(dst, src, (size_t)nbytes);
|
||||
|
||||
return nbytes;
|
||||
}
|
||||
|
||||
static struct spdk_copy_engine memcpy_copy_engine = {
|
||||
.copy = mem_copy_submit,
|
||||
.check_io = mem_copy_check_io,
|
||||
};
|
||||
|
||||
static int
|
||||
copy_engine_mem_get_ctx_size(void)
|
||||
{
|
||||
return sizeof(struct mem_request) + sizeof(struct copy_task);
|
||||
}
|
||||
|
||||
int spdk_copy_module_get_max_ctx_size(void)
|
||||
{
|
||||
struct spdk_copy_module_if *copy_engine;
|
||||
int max_copy_module_size = 0;
|
||||
|
||||
TAILQ_FOREACH(copy_engine, &spdk_copy_module_list, tailq) {
|
||||
if (copy_engine->get_ctx_size && copy_engine->get_ctx_size() > max_copy_module_size) {
|
||||
max_copy_module_size = copy_engine->get_ctx_size();
|
||||
}
|
||||
}
|
||||
return max_copy_module_size;
|
||||
}
|
||||
|
||||
void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module)
|
||||
{
|
||||
TAILQ_INSERT_TAIL(&spdk_copy_module_list, copy_module, tailq);
|
||||
}
|
||||
|
||||
static int
|
||||
copy_engine_mem_init(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < RTE_MAX_LCORE; i++) {
|
||||
copy_engine_req_head[i] = NULL;
|
||||
}
|
||||
|
||||
/* Anyway, We will register memcpy engine */
|
||||
spdk_memcpy_register(&memcpy_copy_engine);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
spdk_copy_engine_module_initialize(void)
|
||||
{
|
||||
struct spdk_copy_module_if *copy_engine_module;
|
||||
|
||||
TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) {
|
||||
copy_engine_module->module_init();
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
spdk_copy_engine_module_finish(void)
|
||||
{
|
||||
struct spdk_copy_module_if *copy_engine_module;
|
||||
|
||||
TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) {
|
||||
if (copy_engine_module->module_fini)
|
||||
copy_engine_module->module_fini();
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
spdk_copy_engine_initialize(void)
|
||||
{
|
||||
spdk_copy_engine_module_initialize();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
spdk_copy_engine_finish(void)
|
||||
{
|
||||
spdk_copy_engine_module_finish();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SPDK_COPY_MODULE_REGISTER(copy_engine_mem_init, NULL, NULL, copy_engine_mem_get_ctx_size)
|
||||
SPDK_SUBSYSTEM_REGISTER(copy, spdk_copy_engine_initialize, spdk_copy_engine_finish, NULL)
|
||||
Loading…
Reference in New Issue
Block a user