diff --git a/include/spdk/copy_engine.h b/include/spdk/copy_engine.h new file mode 100644 index 000000000..a80cb0038 --- /dev/null +++ b/include/spdk/copy_engine.h @@ -0,0 +1,103 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * Memory copy offload engine abstraction layer + */ + +#ifndef SPDK_COPY_ENGINE_H +#define SPDK_COPY_ENGINE_H + +#include +#include + +#include "spdk/queue.h" + +typedef void (*copy_completion_cb)(void *ref, int status); + +struct copy_task { + copy_completion_cb cb; + uint8_t offload_ctx[0]; +}; + +struct spdk_copy_engine { + int64_t (*copy)(void *cb_arg, void *dst, void *src, + uint64_t nbytes, copy_completion_cb cb); + void (*check_io)(void); +}; + +struct spdk_copy_module_if { + /** Initialization function for the module. Called by the spdk + * application during startup. + * + * Modules are required to define this function. + */ + int (*module_init)(void); + + /** Finish function for the module. Called by the spdk application + * before the spdk application exits to perform any necessary cleanup. + * + * Modules are not required to define this function. + */ + void (*module_fini)(void); + + /** Function called to return a text string representing the + * module's configuration options for inclusion in an + * spdk configuration file. + */ + void (*config_text)(FILE *fp); + + int (*get_ctx_size)(void); + TAILQ_ENTRY(spdk_copy_module_if) tailq; +}; + +void spdk_copy_engine_register(struct spdk_copy_engine *copy_engine); +int64_t spdk_copy_submit(struct copy_task *copy_req, void *dst, void *src, + uint64_t nbytes, copy_completion_cb cb); +int spdk_copy_check_io(void); +int spdk_copy_module_get_max_ctx_size(void); +void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module); + +#define SPDK_COPY_MODULE_REGISTER(init_fn, fini_fn, config_fn, ctx_size_fn) \ + static struct spdk_copy_module_if init_fn ## _if = { \ + .module_init = init_fn, \ + .module_fini = fini_fn, \ + .config_text = config_fn, \ + .get_ctx_size = ctx_size_fn, \ + }; \ + __attribute__((constructor)) static void init_fn ## _init(void) \ + { \ + spdk_copy_module_list_add(&init_fn ## _if); \ + } + +#endif diff --git a/lib/Makefile b/lib/Makefile index eda0b8bc6..a16399749 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -DIRS-y += conf cunit event json jsonrpc log memory rpc trace util nvme nvmf ioat +DIRS-y += conf copy cunit event json jsonrpc log memory rpc trace util nvme nvmf ioat .PHONY: all clean $(DIRS-y) diff --git a/lib/copy/Makefile b/lib/copy/Makefile new file mode 100644 index 000000000..dd48f29f8 --- /dev/null +++ b/lib/copy/Makefile @@ -0,0 +1,41 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +CFLAGS += $(DPDK_INC) +LIBNAME = copy +C_SRCS = copy_engine.c + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/lib/copy/copy_engine.c b/lib/copy/copy_engine.c new file mode 100644 index 000000000..f317cd71e --- /dev/null +++ b/lib/copy/copy_engine.c @@ -0,0 +1,233 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "spdk/copy_engine.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "spdk/log.h" +#include "spdk/event.h" + +struct mem_request { + struct mem_request *next; + copy_completion_cb cb; +}; + +struct mem_request *copy_engine_req_head[RTE_MAX_LCORE]; + +static struct spdk_copy_engine *hw_copy_engine = NULL; +/* Memcpy engine always exist */ +static struct spdk_copy_engine *mem_copy_engine = NULL; + +TAILQ_HEAD(, spdk_copy_module_if) spdk_copy_module_list = + TAILQ_HEAD_INITIALIZER(spdk_copy_module_list); + +void +spdk_copy_engine_register(struct spdk_copy_engine *copy_engine) +{ + RTE_VERIFY(hw_copy_engine == NULL); + hw_copy_engine = copy_engine; +} + +static void +spdk_memcpy_register(struct spdk_copy_engine *copy_engine) +{ + RTE_VERIFY(mem_copy_engine == NULL); + mem_copy_engine = copy_engine; +} + +static int +spdk_has_copy_engine(void) +{ + return (hw_copy_engine == NULL) ? 0 : 1; +} + +int +spdk_copy_check_io(void) +{ + if (spdk_has_copy_engine()) + hw_copy_engine->check_io(); + else + mem_copy_engine->check_io(); + + return 0; +} + +static void +copy_engine_done(void *ref, int status) +{ + struct copy_task *req = (struct copy_task *)ref; + + req->cb(req, status); +} + +int64_t +spdk_copy_submit(struct copy_task *copy_req, void *dst, void *src, + uint64_t nbytes, copy_completion_cb cb) +{ + struct copy_task *req = copy_req; + + req->cb = cb; + + if (spdk_has_copy_engine()) + return hw_copy_engine->copy(req->offload_ctx, dst, src, nbytes, + copy_engine_done); + + return mem_copy_engine->copy(req->offload_ctx, dst, src, nbytes, + copy_engine_done); +} + +/* memcpy default copy engine */ +static void +mem_copy_check_io(void) +{ + struct mem_request **req_head = ©_engine_req_head[rte_lcore_id()]; + struct mem_request *req = *req_head; + struct mem_request *req_next; + struct copy_task *copy_req; + + *req_head = NULL; + + while (req != NULL) { + req_next = req->next; + copy_req = (struct copy_task *)((uintptr_t)req - + offsetof(struct copy_task, offload_ctx)); + req->cb((void *)copy_req, 0); + req = req_next; + } + +} + +static int64_t +mem_copy_submit(void *cb_arg, void *dst, void *src, uint64_t nbytes, + copy_completion_cb cb) +{ + struct mem_request **req_head = ©_engine_req_head[rte_lcore_id()]; + struct mem_request *req = (struct mem_request *)cb_arg; + + req->next = *req_head; + *req_head = req; + req->cb = cb; + + rte_memcpy(dst, src, (size_t)nbytes); + + return nbytes; +} + +static struct spdk_copy_engine memcpy_copy_engine = { + .copy = mem_copy_submit, + .check_io = mem_copy_check_io, +}; + +static int +copy_engine_mem_get_ctx_size(void) +{ + return sizeof(struct mem_request) + sizeof(struct copy_task); +} + +int spdk_copy_module_get_max_ctx_size(void) +{ + struct spdk_copy_module_if *copy_engine; + int max_copy_module_size = 0; + + TAILQ_FOREACH(copy_engine, &spdk_copy_module_list, tailq) { + if (copy_engine->get_ctx_size && copy_engine->get_ctx_size() > max_copy_module_size) { + max_copy_module_size = copy_engine->get_ctx_size(); + } + } + return max_copy_module_size; +} + +void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module) +{ + TAILQ_INSERT_TAIL(&spdk_copy_module_list, copy_module, tailq); +} + +static int +copy_engine_mem_init(void) +{ + int i; + + for (i = 0; i < RTE_MAX_LCORE; i++) { + copy_engine_req_head[i] = NULL; + } + + /* Anyway, We will register memcpy engine */ + spdk_memcpy_register(&memcpy_copy_engine); + + return 0; +} + +static void +spdk_copy_engine_module_initialize(void) +{ + struct spdk_copy_module_if *copy_engine_module; + + TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) { + copy_engine_module->module_init(); + } +} + +static void +spdk_copy_engine_module_finish(void) +{ + struct spdk_copy_module_if *copy_engine_module; + + TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) { + if (copy_engine_module->module_fini) + copy_engine_module->module_fini(); + } +} + +static int +spdk_copy_engine_initialize(void) +{ + spdk_copy_engine_module_initialize(); + return 0; +} + +static int +spdk_copy_engine_finish(void) +{ + spdk_copy_engine_module_finish(); + return 0; +} + +SPDK_COPY_MODULE_REGISTER(copy_engine_mem_init, NULL, NULL, copy_engine_mem_get_ctx_size) +SPDK_SUBSYSTEM_REGISTER(copy, spdk_copy_engine_initialize, spdk_copy_engine_finish, NULL)