ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/accel: support multiple accel modules (aka engines) at once

Browse Source 
We enable multiple engines by:

* getting rid of the globals that point to the one available HW
and one available SW engine

* adding a submit_tasks() entry point for the SW engine so that
it is treated like any other engine allowing us to just call
submit_tasks() to the assigned engine for the opcode instead of
checking what is supported

* changing the definition of engine capabilities from
"HW accelerated" to simply "supported"

* during init, use a global (g_engines_opc) that contains engines
and is indexed by opcode so we know what the best engine is for each
op code

* future patches will add RPC's to override engine priorities or
specifically assign an opcode(s) to an engine.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I9b9f3d5a2e499124aa7ccf71f0da83c8ee3dd9f9
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11870
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
This commit is contained in:
paul luse 2022-03-09 22:21:42 +00:00 committed by Tomasz Zawadzki
parent 8f9b977504
commit d58a2f6cc5
8 changed files with 352 additions and 536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/spdk_internal/accel_engine.h
View File

@ -44,9 +44,7 @@ struct spdk_accel_task;
void spdk_accel_task_complete(struct spdk_accel_task *task, int status);
struct accel_io_channel {
struct spdk_accel_engine *engine;
struct spdk_io_channel *engine_ch;
struct spdk_io_channel *sw_engine_ch;
struct spdk_io_channel *engine_ch[ACCEL_OPC_LAST];
void *task_pool_base;
TAILQ_HEAD(, spdk_accel_task) task_pool;
};
@ -85,9 +83,11 @@ struct spdk_accel_task {
};
struct spdk_accel_engine {
const char *name;
bool (*supports_opcode)(enum accel_opcode);
struct spdk_io_channel *(*get_io_channel)(void);
int (*submit_tasks)(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task);
TAILQ_ENTRY(spdk_accel_engine) tailq;
};
struct spdk_accel_module_if {
@ -118,7 +118,7 @@ struct spdk_accel_module_if {
TAILQ_ENTRY(spdk_accel_module_if) tailq;
};
void spdk_accel_hw_engine_register(struct spdk_accel_engine *accel_engine);
void spdk_accel_engine_register(struct spdk_accel_engine *accel_engine);
void spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module);
#define SPDK_ACCEL_MODULE_REGISTER(init_fn, fini_fn, config_json, ctx_size_fn) \

391
lib/accel/accel_engine.c
View File

@ -60,8 +60,6 @@
/* Largest context size for all accel modules */
static size_t g_max_accel_module_size = 0;
static struct spdk_accel_engine *g_hw_accel_engine = NULL;
static struct spdk_accel_engine *g_sw_accel_engine = NULL;
static struct spdk_accel_module_if *g_accel_engine_module = NULL;
static spdk_accel_fini_cb g_fini_cb_fn = NULL;
static void *g_fini_cb_arg = NULL;
@ -70,46 +68,48 @@ static void *g_fini_cb_arg = NULL;
static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);
static void _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags);
static void _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags);
static void _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags);
static int _sw_accel_compare(void *src1, void *src2, size_t nbytes);
static void _sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags);
static void _sw_accel_crc32c(uint32_t *dst, void *src, uint32_t seed, size_t nbytes);
static void _sw_accel_crc32cv(uint32_t *dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed);
/* Global list of registered engines */
static TAILQ_HEAD(, spdk_accel_engine) g_engine_list =
TAILQ_HEAD_INITIALIZER(g_engine_list);
/* Registration of hw modules (currently supports only 1 at a time) */
/* Global array mapping capabilities to engines */
static struct spdk_accel_engine *g_engines_opc[ACCEL_OPC_LAST] = {};
static int sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task);
static struct spdk_accel_engine *
_engine_find_by_name(const char *name)
{
struct spdk_accel_engine *accel_engine = NULL;
TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) {
if (strcmp(name, accel_engine->name) == 0) {
break;
}
}
return accel_engine;
}
/* Registration of all engines */
void
spdk_accel_hw_engine_register(struct spdk_accel_engine *accel_engine)
spdk_accel_engine_register(struct spdk_accel_engine *engine)
{
if (g_hw_accel_engine == NULL) {
g_hw_accel_engine = accel_engine;
} else {
SPDK_NOTICELOG("Hardware offload engine already enabled\n");
}
if (_engine_find_by_name(engine->name)) {
SPDK_NOTICELOG("Accel engine %s already registered\n", engine->name);
assert(false);
return;
}
/* Registration of sw modules (currently supports only 1) */
static void
accel_sw_register(struct spdk_accel_engine *accel_engine)
{
assert(g_sw_accel_engine == NULL);
g_sw_accel_engine = accel_engine;
}
static void
accel_sw_unregister(void)
{
g_sw_accel_engine = NULL;
}
/* Used to determine whether a command is sent to an engine/module or done here
* via SW implementation.
/* Make sure that the software engine is at the head of the list, this
* will assure that all opcodes are later assigned to software first and
* then udpated to HW engines as they are registered.
*/
inline static bool
_is_supported(struct spdk_accel_engine *engine, enum accel_opcode operation)
{
return (engine->supports_opcode(operation));
if (strcmp(engine->name, "software") == 0) {
TAILQ_INSERT_HEAD(&g_engine_list, engine, tailq);
} else {
TAILQ_INSERT_TAIL(&g_engine_list, engine, tailq);
}
}
void
@ -152,10 +152,8 @@ _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, voi
/* Post SW completions to a list and complete in a poller as we don't want to
* complete them on the caller's stack as they'll likely submit another. */
inline static void
_add_to_comp_list(struct accel_io_channel *accel_ch, struct spdk_accel_task *accel_task, int status)
_add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status)
{
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(accel_ch->sw_engine_ch);
accel_task->status = status;
TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link);
}
@ -176,12 +174,13 @@ _check_flags(int flags)
/* Accel framework public API for copy function */
int
spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -194,27 +193,19 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_
accel_task->nbytes = nbytes;
accel_task->flags = flags;
if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_copy(dst, src, (size_t)nbytes, flags);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Accel framework public API for dual cast copy function */
int
spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,
void *dst2, void *src, uint64_t nbytes, int flags,
spdk_accel_completion_cb cb_fn, void *cb_arg)
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_DUALCAST];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_DUALCAST];
if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
@ -233,27 +224,19 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, v
accel_task->flags = flags;
accel_task->op_code = ACCEL_OPC_DUALCAST;
if (_is_supported(accel_ch->engine, ACCEL_OPC_DUALCAST)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_dualcast(dst1, dst2, src, (size_t)nbytes, flags);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Accel framework public API for compare function */
int
spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes,
spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,
void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
void *cb_arg)
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COMPARE];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COMPARE];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -265,23 +248,19 @@ spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, ui
accel_task->nbytes = nbytes;
accel_task->op_code = ACCEL_OPC_COMPARE;
if (_is_supported(accel_ch->engine, ACCEL_OPC_COMPARE)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
rc = _sw_accel_compare(src1, src2, (size_t)nbytes);
_add_to_comp_list(accel_ch, accel_task, rc);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Accel framework public API for fill function */
int
spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes,
int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,
uint8_t fill, uint64_t nbytes, int flags,
spdk_accel_completion_cb cb_fn, void *cb_arg)
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_FILL];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_FILL];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -294,26 +273,19 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint
accel_task->flags = flags;
accel_task->op_code = ACCEL_OPC_FILL;
if (_is_supported(accel_ch->engine, ACCEL_OPC_FILL)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_fill(dst, fill, (size_t)nbytes, flags);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Accel framework public API for CRC-32C function */
int
spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *src, uint32_t seed,
uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,
void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
void *cb_arg)
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_CRC32C];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_CRC32C];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -327,22 +299,19 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *sr
accel_task->nbytes = nbytes;
accel_task->op_code = ACCEL_OPC_CRC32C;
if (_is_supported(accel_ch->engine, ACCEL_OPC_CRC32C)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Accel framework public API for chained CRC-32C function */
int
spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct iovec *iov,
uint32_t iov_cnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,
struct iovec *iov, uint32_t iov_cnt, uint32_t seed,
spdk_accel_completion_cb cb_fn, void *cb_arg)
{
struct accel_io_channel *accel_ch;
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_CRC32C];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_CRC32C];
if (iov == NULL) {
SPDK_ERRLOG("iov should not be NULL");
@ -354,7 +323,6 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct
return -EINVAL;
}
accel_ch = spdk_io_channel_get_ctx(ch);
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
SPDK_ERRLOG("no memory\n");
@ -368,24 +336,19 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct
accel_task->seed = seed;
accel_task->op_code = ACCEL_OPC_CRC32C;
if (_is_supported(accel_ch->engine, ACCEL_OPC_CRC32C)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
_sw_accel_crc32cv(crc_dst, iov, iov_cnt, seed);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Accel framework public API for copy with CRC-32C function */
int
spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,
uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, int flags,
spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,
void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY_CRC32C];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY_CRC32C];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
@ -401,29 +364,19 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,
accel_task->flags = flags;
accel_task->op_code = ACCEL_OPC_COPY_CRC32C;
if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY_CRC32C)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_copy(dst, src, (size_t)nbytes, flags);
_sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Accel framework public API for chained copy + CRC-32C function */
int
spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iovec *src_iovs,
uint32_t iov_cnt, uint32_t *crc_dst, uint32_t seed, int flags,
spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,
struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst,
uint32_t seed, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)
{
struct accel_io_channel *accel_ch;
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
int rc;
struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY_CRC32C];
struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY_CRC32C];
if (src_iovs == NULL) {
SPDK_ERRLOG("iov should not be NULL");
@ -435,7 +388,6 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov
return -EINVAL;
}
accel_ch = spdk_io_channel_get_ctx(ch);
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
if (accel_task == NULL) {
SPDK_ERRLOG("no memory\n");
@ -451,18 +403,7 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov
accel_task->flags = flags;
accel_task->op_code = ACCEL_OPC_COPY_CRC32C;
if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY_CRC32C)) {
return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);
} else {
rc = _check_flags(flags);
if (rc) {
return rc;
}
_sw_accel_copyv(dst, src_iovs, iov_cnt, flags);
_sw_accel_crc32cv(crc_dst, src_iovs, iov_cnt, seed);
_add_to_comp_list(accel_ch, accel_task, 0);
return 0;
}
return engine->submit_tasks(engine_ch, accel_task);
}
/* Helper function when when accel modules register with the framework. */
@ -481,7 +422,7 @@ accel_engine_create_cb(void *io_device, void *ctx_buf)
struct accel_io_channel *accel_ch = ctx_buf;
struct spdk_accel_task *accel_task;
uint8_t *task_mem;
int i;
int i, j;
accel_ch->task_pool_base = calloc(MAX_TASKS_PER_CHANNEL, g_max_accel_module_size);
if (accel_ch->task_pool_base == NULL) {
@ -496,21 +437,21 @@ accel_engine_create_cb(void *io_device, void *ctx_buf)
task_mem += g_max_accel_module_size;
}
/* Set sw engine channel for operations where hw engine does not support. */
accel_ch->sw_engine_ch = g_sw_accel_engine->get_io_channel();
assert(accel_ch->sw_engine_ch != NULL);
if (g_hw_accel_engine != NULL) {
accel_ch->engine_ch = g_hw_accel_engine->get_io_channel();
accel_ch->engine = g_hw_accel_engine;
} else {
/* No hw engine enabled, use sw. */
accel_ch->engine_ch = accel_ch->sw_engine_ch;
accel_ch->engine = g_sw_accel_engine;
/* Assign engines and get IO channels for each */
for (i = 0; i < ACCEL_OPC_LAST; i++) {
accel_ch->engine_ch[i] = g_engines_opc[i]->get_io_channel();
/* This can happen if idxd runs out of channels. */
if (accel_ch->engine_ch[i] == NULL) {
goto err;
}
}
assert(accel_ch->engine_ch != NULL);
return 0;
err:
for (j = 0; j < i; j++) {
spdk_put_io_channel(accel_ch->engine_ch[j]);
}
return -EINVAL;
}
/* Framework level channel destroy callback. */
@ -518,11 +459,15 @@ static void
accel_engine_destroy_cb(void *io_device, void *ctx_buf)
{
struct accel_io_channel *accel_ch = ctx_buf;
int i;
if (accel_ch->sw_engine_ch != accel_ch->engine_ch) {
spdk_put_io_channel(accel_ch->sw_engine_ch);
for (i = 0; i < ACCEL_OPC_LAST; i++) {
if (accel_ch->engine_ch[i]) {
spdk_put_io_channel(accel_ch->engine_ch[i]);
accel_ch->engine_ch[i] = NULL;
}
spdk_put_io_channel(accel_ch->engine_ch);
}
free(accel_ch->task_pool_base);
}
@ -545,9 +490,30 @@ accel_engine_module_initialize(void)
int
spdk_accel_engine_initialize(void)
{
SPDK_NOTICELOG("Accel engine initialized to use software engine.\n");
enum accel_opcode op;
struct spdk_accel_engine *accel_engine = NULL;
accel_engine_module_initialize();
/* Create our priority global map of opcodes to engines, we populate starting
* with the software engine (guaranteed to be first on the list) and then
* updating opcodes with HW engines that have been initilaized.
* NOTE: all opcodes must be suported by software in the event that no HW
* engines are initilaized to support the operation.
*/
TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) {
for (op = 0; op < ACCEL_OPC_LAST; op++) {
if (accel_engine->supports_opcode(op)) {
g_engines_opc[op] = accel_engine;
SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_engine->name);
}
}
}
#ifdef DEBUG
for (op = 0; op < ACCEL_OPC_LAST; op++) {
assert(g_engines_opc[op] != NULL);
}
#endif
/*
* We need a unique identifier for the accel engine framework, so use the
* spdk_accel_module_list address for this purpose.
@ -625,8 +591,18 @@ spdk_accel_engine_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
static bool
sw_accel_supports_opcode(enum accel_opcode opc)
{
switch (opc) {
case ACCEL_OPC_COPY:
case ACCEL_OPC_FILL:
case ACCEL_OPC_DUALCAST:
case ACCEL_OPC_COMPARE:
case ACCEL_OPC_CRC32C:
case ACCEL_OPC_COPY_CRC32C:
return true;
default:
return false;
}
}
static inline void
_pmem_memcpy(void *dst, const void *src, size_t len)
@ -725,12 +701,87 @@ _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_
*crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed);
}
static int
sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task)
{
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *tmp;
int rc = 0;
do {
switch (accel_task->op_code) {
case ACCEL_OPC_COPY:
rc = _check_flags(accel_task->flags);
if (rc == 0) {
_sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags);
}
_add_to_comp_list(sw_ch, accel_task, rc);
break;
case ACCEL_OPC_FILL:
rc = _check_flags(accel_task->flags);
if (rc == 0) {
_sw_accel_fill(accel_task->dst, accel_task->fill_pattern, accel_task->nbytes, accel_task->flags);
}
_add_to_comp_list(sw_ch, accel_task, rc);
break;
case ACCEL_OPC_DUALCAST:
rc = _check_flags(accel_task->flags);
if (rc == 0) {
_sw_accel_dualcast(accel_task->dst, accel_task->dst2, accel_task->src, accel_task->nbytes,
accel_task->flags);
}
_add_to_comp_list(sw_ch, accel_task, rc);
break;
case ACCEL_OPC_COMPARE:
rc = _sw_accel_compare(accel_task->src, accel_task->src2, accel_task->nbytes);
_add_to_comp_list(sw_ch, accel_task, rc);
break;
case ACCEL_OPC_CRC32C:
if (accel_task->v.iovcnt == 0) {
_sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes);
} else {
_sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed);
}
_add_to_comp_list(sw_ch, accel_task, 0);
break;
case ACCEL_OPC_COPY_CRC32C:
rc = _check_flags(accel_task->flags);
if (rc == 0) {
if (accel_task->v.iovcnt == 0) {
_sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags);
_sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes);
} else {
_sw_accel_copyv(accel_task->dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->flags);
_sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed);
}
}
_add_to_comp_list(sw_ch, accel_task, rc);
break;
default:
assert(false);
break;
}
tmp = TAILQ_NEXT(accel_task, link);
/* Report any build errors via the callback now. */
if (rc) {
spdk_accel_task_complete(accel_task, rc);
}
accel_task = tmp;
} while (accel_task);
return 0;
}
static struct spdk_io_channel *sw_accel_get_io_channel(void);
static struct spdk_accel_engine sw_accel_engine = {
.name = "software",
.supports_opcode = sw_accel_supports_opcode,
.get_io_channel = sw_accel_get_io_channel,
.submit_tasks = sw_accel_submit_tasks,
};
static int
@ -788,7 +839,8 @@ sw_accel_engine_get_ctx_size(void)
static int
sw_accel_engine_init(void)
{
accel_sw_register(&sw_accel_engine);
SPDK_NOTICELOG("Accel framework software engine initialized.\n");
spdk_accel_engine_register(&sw_accel_engine);
spdk_io_device_register(&sw_accel_engine, sw_accel_create_cb, sw_accel_destroy_cb,
sizeof(struct sw_accel_io_channel), "sw_accel_engine");
@ -798,8 +850,17 @@ sw_accel_engine_init(void)
static void
sw_accel_engine_fini(void *ctxt)
{
struct spdk_accel_engine *accel_engine;
spdk_io_device_unregister(&sw_accel_engine, NULL);
accel_sw_unregister();
/* unregister the software engine */
TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) {
if (strcmp(accel_engine->name, "software") == 0) {
TAILQ_REMOVE(&g_engine_list, accel_engine, tailq);
break;
}
}
spdk_accel_engine_module_finish();
}

2
lib/accel/spdk_accel.map
View File

@ -17,7 +17,7 @@
spdk_accel_write_config_json;
# functions needed by modules
spdk_accel_hw_engine_register;
spdk_accel_engine_register;
spdk_accel_module_list_add;
spdk_accel_task_complete;

11
module/accel/idxd/accel_engine_idxd.c
View File

@ -130,7 +130,9 @@ static void
idxd_done(void *cb_arg, int status)
{
struct spdk_accel_task *accel_task = cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(accel_task->accel_ch->engine_ch);
struct idxd_io_channel *chan;
chan = spdk_io_channel_get_ctx(accel_task->accel_ch->engine_ch[accel_task->op_code]);
assert(chan->num_outstanding > 0);
spdk_trace_record(TRACE_IDXD_OP_COMPLETE, 0, 0, 0, chan->num_outstanding - 1);
@ -302,7 +304,7 @@ idxd_poll(void *arg)
TAILQ_INIT(&chan->queued_tasks);
idxd_submit_tasks(task->accel_ch->engine_ch, task);
idxd_submit_tasks(task->accel_ch->engine_ch[task->op_code], task);
}
}
@ -332,6 +334,7 @@ idxd_supports_opcode(enum accel_opcode opc)
}
static struct spdk_accel_engine idxd_accel_engine = {
.name = "idxd",
.supports_opcode = idxd_supports_opcode,
.get_io_channel = idxd_get_io_channel,
.submit_tasks = idxd_submit_tasks,
@ -419,8 +422,8 @@ accel_engine_idxd_init(void)
}
g_idxd_initialized = true;
SPDK_NOTICELOG("Accel engine updated to use IDXD DSA engine.\n");
spdk_accel_hw_engine_register(&idxd_accel_engine);
SPDK_NOTICELOG("Accel framework IDXD engine initialized.\n");
spdk_accel_engine_register(&idxd_accel_engine);
spdk_io_device_register(&idxd_accel_engine, idxd_create_cb, idxd_destroy_cb,
sizeof(struct idxd_io_channel), "idxd_accel_engine");
return 0;

5
module/accel/ioat/accel_engine_ioat.c
View File

@ -182,6 +182,7 @@ ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task
}
static struct spdk_accel_engine ioat_accel_engine = {
.name = "ioat",
.supports_opcode = ioat_supports_opcode,
.get_io_channel = ioat_get_io_channel,
.submit_tasks = ioat_submit_tasks,
@ -289,8 +290,8 @@ accel_engine_ioat_init(void)
}
g_ioat_initialized = true;
SPDK_NOTICELOG("Accel engine updated to use IOAT engine.\n");
spdk_accel_hw_engine_register(&ioat_accel_engine);
SPDK_NOTICELOG("Accel framework IOAT engine initialized.\n");
spdk_accel_engine_register(&ioat_accel_engine);
spdk_io_device_register(&ioat_accel_engine, ioat_create_cb, ioat_destroy_cb,
sizeof(struct ioat_io_channel), "ioat_accel_engine");
return 0;

1
module/accel/ioat/accel_engine_ioat_rpc.c
View File

@ -47,6 +47,7 @@ rpc_ioat_scan_accel_engine(struct spdk_jsonrpc_request *request,
return;
}
SPDK_NOTICELOG("Enabling IOAT\n");
accel_engine_ioat_enable_probe();
spdk_jsonrpc_send_bool_response(request, true);

466
test/unit/lib/accel/accel.c/accel_engine_ut.c
View File

@ -73,6 +73,8 @@ _supports_opcode(enum accel_opcode opc)
static int
test_setup(void)
{
int i;
g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel));
if (g_ch == NULL) {
/* for some reason the assert fatal macro doesn't work in the setup function. */
@ -85,9 +87,13 @@ test_setup(void)
CU_ASSERT(false);
return -1;
}
g_accel_ch->engine_ch = g_engine_ch;
g_accel_ch->sw_engine_ch = g_engine_ch;
g_sw_ch = (struct sw_accel_io_channel *)((char *)g_accel_ch->sw_engine_ch + sizeof(
g_accel_engine.submit_tasks = sw_accel_submit_tasks;
for (i = 0; i < ACCEL_OPC_LAST; i++) {
g_accel_ch->engine_ch[i] = g_engine_ch;
g_engines_opc[i] = &g_accel_engine;
}
g_sw_ch = (struct sw_accel_io_channel *)((char *)g_engine_ch + sizeof(
struct spdk_io_channel));
TAILQ_INIT(&g_sw_ch->tasks_to_complete);
g_accel_engine.supports_opcode = _supports_opcode;
@ -103,52 +109,6 @@ test_cleanup(void)
return 0;
}
static void
test_spdk_accel_hw_engine_register(void)
{
/* Run once with no engine assigned, assign it. */
g_hw_accel_engine = NULL;
spdk_accel_hw_engine_register(&g_accel_engine);
CU_ASSERT(g_hw_accel_engine == &g_accel_engine);
/* Run with one assigned, should not change. */
spdk_accel_hw_engine_register(&g_accel_engine);
CU_ASSERT(g_hw_accel_engine == &g_accel_engine);
}
static int
test_accel_sw_register(void)
{
/* Run once with no engine assigned, assign it. */
g_sw_accel_engine = NULL;
accel_sw_register(&g_accel_engine);
CU_ASSERT(g_sw_accel_engine == &g_accel_engine);
return 0;
}
static void
test_accel_sw_unregister(void)
{
/* Run once engine assigned, make sure it gets unassigned. */
g_sw_accel_engine = &g_accel_engine;
accel_sw_unregister();
CU_ASSERT(g_sw_accel_engine == NULL);
}
static void
test_is_supported(void)
{
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY) | _accel_op_to_bit(
ACCEL_OPC_DUALCAST) |
_accel_op_to_bit(ACCEL_OPC_CRC32C);
CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_COPY) == true);
CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_FILL) == false);
CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_DUALCAST) == true);
CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_COMPARE) == false);
CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_CRC32C) == true);
}
#define DUMMY_ARG 0xDEADBEEF
static bool g_dummy_cb_called = false;
static void
@ -206,22 +166,6 @@ test_get_task(void)
CU_ASSERT(_task.accel_ch == g_accel_ch);
}
static bool g_dummy_submit_called = false;
static int
dummy_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task)
{
g_dummy_submit_called = true;
return 0;
}
static bool g_dummy_submit_cb_called = false;
static void
dummy_submit_cb_fn(void *cb_arg, int status)
{
g_dummy_submit_cb_called = true;
CU_ASSERT(status == 0);
}
#define TEST_SUBMIT_SIZE 64
static void
test_spdk_accel_submit_copy(void)
@ -238,48 +182,21 @@ test_spdk_accel_submit_copy(void)
TAILQ_INIT(&g_accel_ch->task_pool);
/* Fail with no tasks on _get_task() */
rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, dummy_submit_cb_fn, cb_arg);
rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == -ENOMEM);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
task.flags = 1;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* HW accel submission OK. */
rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, dummy_submit_cb_fn, cb_arg);
/* submission OK. */
rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.dst == dst);
CU_ASSERT(task.src == src);
CU_ASSERT(task.op_code == ACCEL_OPC_COPY);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(g_dummy_submit_called == true);
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
/* reset values before next case */
g_dummy_submit_called = false;
g_opc_mask = 0;
task.dst = 0;
task.src = 0;
task.op_code = 0xff;
task.nbytes = 0;
task.flags = 1;
/* SW engine does copy. */
rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, dummy_submit_cb_fn, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.dst == dst);
CU_ASSERT(task.src == src);
CU_ASSERT(task.op_code == ACCEL_OPC_COPY);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(g_dummy_submit_cb_called == false);
CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0);
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link);
@ -300,6 +217,8 @@ test_spdk_accel_submit_dualcast(void)
struct spdk_accel_task *expected_accel_task = NULL;
int flags = 0;
TAILQ_INIT(&g_accel_ch->task_pool);
/* Dualcast requires 4K alignment on dst addresses,
* hence using the hard coded address to test the buffer alignment
*/
@ -309,69 +228,33 @@ test_spdk_accel_submit_dualcast(void)
SPDK_CU_ASSERT_FATAL(src != NULL);
memset(src, 0x5A, TEST_SUBMIT_SIZE);
TAILQ_INIT(&g_accel_ch->task_pool);
/* This should fail since dst2 is not 4k aligned */
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn,
cb_arg);
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == -EINVAL);
dst1 = (void *)0x7010;
dst2 = (void *)0x6000;
/* This should fail since dst1 is not 4k aligned */
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn,
cb_arg);
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == -EINVAL);
/* Dualcast requires 4K alignment on dst addresses */
dst1 = (void *)0x7000;
dst2 = (void *)0x6000;
/* Fail with no tasks on _get_task() */
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn,
cb_arg);
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == -ENOMEM);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_DUALCAST);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* HW accel submission OK. */
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn,
cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.dst == dst1);
CU_ASSERT(task.dst2 == dst2);
CU_ASSERT(task.src == src);
CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(g_dummy_submit_called == true);
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
/* Reset values before next case */
g_dummy_submit_called = false;
g_opc_mask = 0;
task.dst = 0;
task.dst2 = 0;
task.src = 0;
task.op_code = 0xff;
task.nbytes = 0;
task.flags = 1;
/* Since we test the SW path next, need to use valid memory addresses
* cannot hardcode them anymore
*/
/* accel submission OK., since we test the SW path , need to use valid memory addresses
* cannot hardcode them anymore */
dst1 = spdk_dma_zmalloc(nbytes, align, NULL);
SPDK_CU_ASSERT_FATAL(dst1 != NULL);
dst2 = spdk_dma_zmalloc(nbytes, align, NULL);
SPDK_CU_ASSERT_FATAL(dst2 != NULL);
/* SW engine does the dualcast. */
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn,
cb_arg);
rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.dst == dst1);
CU_ASSERT(task.dst2 == dst2);
@ -379,7 +262,6 @@ test_spdk_accel_submit_dualcast(void)
CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(g_dummy_submit_cb_called == false);
CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0);
CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0);
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
@ -402,54 +284,26 @@ test_spdk_accel_submit_compare(void)
struct spdk_accel_task task;
struct spdk_accel_task *expected_accel_task = NULL;
TAILQ_INIT(&g_accel_ch->task_pool);
src1 = calloc(1, TEST_SUBMIT_SIZE);
SPDK_CU_ASSERT_FATAL(src1 != NULL);
src2 = calloc(1, TEST_SUBMIT_SIZE);
SPDK_CU_ASSERT_FATAL(src2 != NULL);
/* Fail with no tasks on _get_task() */
rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, dummy_submit_cb_fn, cb_arg);
rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg);
CU_ASSERT(rc == -ENOMEM);
TAILQ_INIT(&g_accel_ch->task_pool);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COMPARE);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* HW accel submission OK. */
rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, dummy_submit_cb_fn, cb_arg);
/* accel submission OK. */
rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.src == src1);
CU_ASSERT(task.src2 == src2);
CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(g_dummy_submit_called == true);
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
/* Reset values before next case */
g_dummy_submit_called = false;
g_opc_mask = 0;
task.src = 0;
task.src2 = 0;
task.op_code = 0xff;
task.nbytes = 0;
memset(src1, 0x5A, TEST_SUBMIT_SIZE);
memset(src2, 0x5A, TEST_SUBMIT_SIZE);
/* SW engine does compare. */
rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, dummy_submit_cb_fn, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.src == src1);
CU_ASSERT(task.src2 == src2);
CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(g_dummy_submit_cb_called == false);
CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0);
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link);
@ -473,6 +327,8 @@ test_spdk_accel_submit_fill(void)
struct spdk_accel_task *expected_accel_task = NULL;
int flags = 0;
TAILQ_INIT(&g_accel_ch->task_pool);
dst = calloc(1, TEST_SUBMIT_SIZE);
SPDK_CU_ASSERT_FATAL(dst != NULL);
src = calloc(1, TEST_SUBMIT_SIZE);
@ -481,48 +337,20 @@ test_spdk_accel_submit_fill(void)
memset(&fill64, fill, sizeof(uint64_t));
/* Fail with no tasks on _get_task() */
rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, dummy_submit_cb_fn, cb_arg);
rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == -ENOMEM);
TAILQ_INIT(&g_accel_ch->task_pool);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_FILL);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* HW accel submission OK. */
rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, dummy_submit_cb_fn, cb_arg);
/* accel submission OK. */
rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.dst == dst);
CU_ASSERT(task.fill_pattern == fill64);
CU_ASSERT(task.op_code == ACCEL_OPC_FILL);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(g_dummy_submit_called == true);
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
/* Reset values before next case */
g_dummy_submit_called = false;
g_opc_mask = 0;
task.dst = 0;
task.fill_pattern = 0;
task.op_code = 0xff;
task.nbytes = 0;
task.flags = 1;
/* SW engine does the fill. */
rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, dummy_submit_cb_fn, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.dst == dst);
CU_ASSERT(task.fill_pattern == fill64);
CU_ASSERT(task.op_code == ACCEL_OPC_FILL);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(g_dummy_submit_cb_called == false);
CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0);
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link);
@ -544,85 +372,16 @@ test_spdk_accel_submit_crc32c(void)
struct spdk_accel_task task;
struct spdk_accel_task *expected_accel_task = NULL;
/* Fail with no tasks on _get_task() */
rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg);
CU_ASSERT(rc == -ENOMEM);
TAILQ_INIT(&g_accel_ch->task_pool);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_CRC32C);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* HW accel submission OK. */
rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.crc_dst == &crc_dst);
CU_ASSERT(task.src == src);
CU_ASSERT(task.v.iovcnt == 0);
CU_ASSERT(task.seed == seed);
CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(g_dummy_submit_called == true);
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
/* Reset values before next case */
g_dummy_submit_called = false;
g_opc_mask = 0;
task.crc_dst = 0;
task.src = 0;
task.seed = 0;
task.op_code = 0xff;
task.nbytes = 0;
/* SW engine does crc. */
rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.crc_dst == &crc_dst);
CU_ASSERT(task.src == src);
CU_ASSERT(task.v.iovcnt == 0);
CU_ASSERT(task.seed == seed);
CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(g_dummy_submit_cb_called == false);
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link);
CU_ASSERT(expected_accel_task == &task);
}
static void
test_spdk_accel_submit_crc32c_hw_engine_unsupported(void)
{
const uint64_t nbytes = TEST_SUBMIT_SIZE;
uint32_t crc_dst;
uint8_t src[TEST_SUBMIT_SIZE];
uint32_t seed = 1;
void *cb_arg = NULL;
int rc;
struct spdk_accel_task task;
struct spdk_accel_task *expected_accel_task = NULL;
/* Fail with no tasks on _get_task() */
rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg);
rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg);
CU_ASSERT(rc == -ENOMEM);
TAILQ_INIT(&g_accel_ch->task_pool);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
/* HW engine only supports COPY and does not support CRC */
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* Summit to HW engine while eventually handled by SW engine. */
rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg);
/* accel submission OK. */
rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.crc_dst == &crc_dst);
CU_ASSERT(task.src == src);
@ -630,10 +389,6 @@ test_spdk_accel_submit_crc32c_hw_engine_unsupported(void)
CU_ASSERT(task.seed == seed);
CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C);
CU_ASSERT(task.nbytes == nbytes);
/* Not set in HW engine callback while handled by SW engine instead. */
CU_ASSERT(g_dummy_submit_called == false);
/* SW engine does crc. */
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link);
CU_ASSERT(expected_accel_task == &task);
@ -652,55 +407,27 @@ test_spdk_accel_submit_crc32cv(void)
struct iovec iov[32];
struct spdk_accel_task *expected_accel_task = NULL;
TAILQ_INIT(&g_accel_ch->task_pool);
for (i = 0; i < iov_cnt; i++) {
iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE);
SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL);
iov[i].iov_len = TEST_SUBMIT_SIZE;
}
TAILQ_INIT(&g_accel_ch->task_pool);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
task.nbytes = TEST_SUBMIT_SIZE;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_CRC32C);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* HW accel submission OK. */
rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, dummy_submit_cb_fn, cb_arg);
/* accel submission OK. */
rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.v.iovs == iov);
CU_ASSERT(task.v.iovcnt == iov_cnt);
CU_ASSERT(task.crc_dst == &crc_dst);
CU_ASSERT(task.seed == seed);
CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C);
CU_ASSERT(g_dummy_submit_called == true);
CU_ASSERT(task.cb_fn == dummy_submit_cb_fn);
CU_ASSERT(task.cb_arg == cb_arg);
CU_ASSERT(task.nbytes == iov[0].iov_len);
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_dummy_submit_called = false;
g_opc_mask = 0;
task.v.iovs = 0;
task.v.iovcnt = 0;
task.crc_dst = 0;
task.seed = 0;
task.op_code = 0xff;
/* SW engine submit crc. */
rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, dummy_submit_cb_fn, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.v.iovs == iov);
CU_ASSERT(task.v.iovcnt == iov_cnt);
CU_ASSERT(task.crc_dst == &crc_dst);
CU_ASSERT(task.seed == seed);
CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C);
CU_ASSERT(g_dummy_submit_cb_called == false);
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link);
CU_ASSERT(expected_accel_task == &task);
@ -724,26 +451,18 @@ test_spdk_accel_submit_copy_crc32c(void)
struct spdk_accel_task *expected_accel_task = NULL;
int flags = 0;
TAILQ_INIT(&g_accel_ch->task_pool);
/* Fail with no tasks on _get_task() */
rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags,
dummy_submit_cb_fn,
cb_arg);
NULL, cb_arg);
CU_ASSERT(rc == -ENOMEM);
TAILQ_INIT(&g_accel_ch->task_pool);
task.cb_fn = dummy_submit_cb_fn;
task.cb_arg = cb_arg;
task.accel_ch = g_accel_ch;
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_accel_ch->engine = &g_accel_engine;
g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY_CRC32C);
g_accel_ch->engine->submit_tasks = dummy_submit_tasks;
/* HW accel submission OK. */
/* accel submission OK. */
rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags,
dummy_submit_cb_fn,
cb_arg);
NULL, cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(task.dst == dst);
CU_ASSERT(task.src == src);
@ -753,41 +472,75 @@ test_spdk_accel_submit_copy_crc32c(void)
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C);
CU_ASSERT(g_dummy_submit_called == true);
TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link);
g_dummy_submit_called = false;
g_opc_mask = 0;
task.dst = 0;
task.src = 0;
task.crc_dst = 0;
task.v.iovcnt = 0;
task.seed = 0;
task.nbytes = 0;
task.flags = 1;
task.op_code = 0xff;
memset(src, 0x5A, TEST_SUBMIT_SIZE);
/* SW engine does copy crc. */
rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags,
dummy_submit_cb_fn,
cb_arg);
CU_ASSERT(rc == 0);
CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0);
CU_ASSERT(task.dst == dst);
CU_ASSERT(task.src == src);
CU_ASSERT(task.crc_dst == &crc_dst);
CU_ASSERT(task.v.iovcnt == 0);
CU_ASSERT(task.seed == seed);
CU_ASSERT(task.nbytes == nbytes);
CU_ASSERT(task.flags == 0);
CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C);
CU_ASSERT(g_dummy_submit_cb_called == false);
expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete);
TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link);
CU_ASSERT(expected_accel_task == &task);
}
static void
test_engine_find_by_name(void)
{
struct spdk_accel_engine eng1, eng2, eng3;
struct spdk_accel_engine *accel_engine = NULL;
eng1.name = "ioat";
eng2.name = "idxd";
eng3.name = "software";
TAILQ_INIT(&g_engine_list);
TAILQ_INSERT_TAIL(&g_engine_list, &eng1, tailq);
TAILQ_INSERT_TAIL(&g_engine_list, &eng2, tailq);
TAILQ_INSERT_TAIL(&g_engine_list, &eng3, tailq);
/* Now let's find a valid engine */
accel_engine = _engine_find_by_name("ioat");
CU_ASSERT(accel_engine != NULL);
/* Try to find one that doesn't exist */
accel_engine = _engine_find_by_name("XXX");
CU_ASSERT(accel_engine == NULL);
}
static void
test_spdk_accel_engine_register(void)
{
struct spdk_accel_engine eng1, eng2, eng3, eng4;
struct spdk_accel_engine *accel_engine = NULL;
int i = 0;
eng1.name = "ioat";
eng2.name = "idxd";
eng3.name = "software";
eng4.name = "nothing";
spdk_accel_engine_register(&eng1);
spdk_accel_engine_register(&eng2);
spdk_accel_engine_register(&eng3);
spdk_accel_engine_register(&eng4);
/* Now confirm they're in the right order. */
TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) {
switch (i++) {
case 0:
CU_ASSERT(strcmp(accel_engine->name, "software") == 0);
break;
case 1:
CU_ASSERT(strcmp(accel_engine->name, "ioat") == 0);
break;
case 2:
CU_ASSERT(strcmp(accel_engine->name, "idxd") == 0);
break;
case 3:
CU_ASSERT(strcmp(accel_engine->name, "nothing") == 0);
break;
default:
CU_ASSERT(false);
break;
}
}
CU_ASSERT(i == 4);
}
int main(int argc, char **argv)
{
CU_pSuite suite = NULL;
@ -798,10 +551,7 @@ int main(int argc, char **argv)
suite = CU_add_suite("accel", test_setup, test_cleanup);
CU_ADD_TEST(suite, test_spdk_accel_hw_engine_register);
CU_ADD_TEST(suite, test_accel_sw_register);
CU_ADD_TEST(suite, test_accel_sw_unregister);
CU_ADD_TEST(suite, test_is_supported);
CU_ADD_TEST(suite, test_spdk_accel_engine_register);
CU_ADD_TEST(suite, test_spdk_accel_task_complete);
CU_ADD_TEST(suite, test_get_task);
CU_ADD_TEST(suite, test_spdk_accel_submit_copy);
@ -809,9 +559,9 @@ int main(int argc, char **argv)
CU_ADD_TEST(suite, test_spdk_accel_submit_compare);
CU_ADD_TEST(suite, test_spdk_accel_submit_fill);
CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c);
CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c_hw_engine_unsupported);
CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv);
CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c);
CU_ADD_TEST(suite, test_engine_find_by_name);
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();