From 6293ac87594b49f48bf54a163f025f97c56b2534 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Wed, 16 Nov 2022 08:22:55 +0100 Subject: [PATCH] accel: initial operation chaining support This patch introduces the concept of chaining multiple accel operations and executing them all at once in a single step. This means that it will be possible to schedule accel operations at different layers of the stack (e.g. copy in NVMe-oF transport, crypto in bdev_crypto), but execute them all in a single place. Thanks to this, we can take advantage of hardware accelerators that supports executing multiple operations as a single operation (e.g. copy + crypto). This operation group is called spdk_accel_sequence and operations can be appended to that object via one of the spdk_accel_append_* functions. New operations are always added at the end of a sequence. Users can specify a callback to be notified when a particular operation in a sequence is completed, but they don't receive the status of whether it was successful or not. This is by design, as they shouldn't care about the status of an individual operation and should rely on other means to receive the status of the whole sequence. It's also important to note that any intermediate steps within a sequence may not produce observable results. For instance, appending a copy from A to B and then a copy from B to C, it's indeterminate whether A's data will be in B after a sequence is executed. It is only guaranteed that A's data will be in C. A sequence can also be reversed using spdk_accel_sequence_reverse(), meaning that the first operation becomes last and vice versa. It's especially useful in read paths, as it makes it possible to build the sequence during submission, then, once the data is read from storage, reverse the sequence and execute it. Finally, there are two ways to terminate a sequence: aborting or executing. It can be aborted via spdk_accel_sequence_abort() which will execute individual operations' callbacks and free any allocated resources. To execute it, one must use spdk_accel_sequence_finish(). For now, each operation is executed one by one and is submitted to the appropriate accel module. Executing multiple operations as a single one will be added in the future. Also, currently, only fill and copy operations can be appended to a sequence. Support for more operations will be added in subsequent patches. Signed-off-by: Konrad Sztyber Change-Id: Id35d093e14feb59b996f780ef77e000e10bfcd20 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15529 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris Reviewed-by: Aleksey Marchuk --- include/spdk/accel.h | 87 ++++ include/spdk_internal/accel_module.h | 7 + lib/accel/Makefile | 2 +- lib/accel/accel.c | 363 +++++++++++++++- lib/accel/spdk_accel.map | 5 + test/unit/lib/accel/accel.c/accel_ut.c | 568 ++++++++++++++++++++++++- 6 files changed, 1024 insertions(+), 8 deletions(-) diff --git a/include/spdk/accel.h b/include/spdk/accel.h index 9ac8f7c39..482a77bc7 100644 --- a/include/spdk/accel.h +++ b/include/spdk/accel.h @@ -11,6 +11,7 @@ #define SPDK_ACCEL_H #include "spdk/stdinc.h" +#include "spdk/dma.h" #ifdef __cplusplus extern "C" { @@ -260,6 +261,92 @@ int spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_i size_t src_iovcnt, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg); +/** Object grouping multiple accel operations to be executed at the same point in time */ +struct spdk_accel_sequence; + +/** + * Completion callback of a single operation within a sequence. After it's executed, the sequence + * object might be freed, so users should not touch it. + */ +typedef void (*spdk_accel_step_cb)(void *cb_arg); + +/** + * Append a copy operation to a sequence. Copy operation in a sequence is special, as it is not + * guaranteed that the data will be actually copied. If it's possible, it will only change + * source / destination buffers of some of the operations in a sequence. + * + * \param seq Sequence object. If NULL, a new sequence object will be created. + * \param ch I/O channel. + * \param dst_iovs Destination I/O vector array. + * \param dst_iovcnt Size of the `dst_iovs` array. + * \param dst_domain Memory domain to which the destination buffers belong. + * \param dst_domain_ctx Destination buffer domain context. + * \param src_iovs Source I/O vector array. + * \param src_iovcnt Size of the `src_iovs` array. + * \param src_domain Memory domain to which the source buffers belong. + * \param src_domain_ctx Source buffer domain context. + * \param flags Accel operation flags. + * \param cb_fn Callback to be executed once this operation is completed. + * \param cb_arg Argument to be passed to `cb_fn`. + * + * \return 0 if operation was successfully added to the sequence, negative errno otherwise. + */ +int spdk_accel_append_copy(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch, + struct iovec *dst_iovs, uint32_t dst_iovcnt, + struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, + struct iovec *src_iovs, uint32_t src_iovcnt, + struct spdk_memory_domain *src_domain, void *src_domain_ctx, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg); + +/** + * Append a fill operation to a sequence. + * + * \param seq Sequence object. If NULL, a new sequence object will be created. + * \param ch I/O channel. + * \param buf Data buffer. + * \param len Length of the data buffer. + * \param domain Memory domain to which the data buffer belongs. + * \param domain_ctx Buffer domain context. + * \param pattern Pattern to fill the buffer with. + * \param flags Accel operation flags. + * \param cb_fn Callback to be executed once this operation is completed. + * \param cb_arg Argument to be passed to `cb_fn`. + * + * \return 0 if operation was successfully added to the sequence, negative errno otherwise. + */ +int spdk_accel_append_fill(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch, + void *buf, uint64_t len, + struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg); + +/** + * Finish a sequence and execute all its operations. After the completion callback is executed, the + * sequence object is automatically freed. + * + * \param seq Sequence to finish. + * \param cb_fn Completion callback to be executed once all operations are executed. + * \param cb_arg Argument to be passed to `cb_fn`. + * + * \return 0 on success, negative errno otherwise. + */ +int spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, + spdk_accel_completion_cb cb_fn, void *cb_arg); + +/** + * Reverse a sequence, so that the last operation becomes the first and vice versa. + * + * \param seq Sequence to reverse. + */ +void spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq); + +/** + * Abort a sequence. This will execute the completion callbacks of all operations that were added + * to the sequence and will then free the sequence object. + * + * \param seq Sequence to abort. + */ +void spdk_accel_sequence_abort(struct spdk_accel_sequence *seq); + /** * Return the name of the module assigned to a specific opcode. * diff --git a/include/spdk_internal/accel_module.h b/include/spdk_internal/accel_module.h index a4cde32fa..2bf6188b6 100644 --- a/include/spdk_internal/accel_module.h +++ b/include/spdk_internal/accel_module.h @@ -20,6 +20,12 @@ struct spdk_accel_task { struct accel_io_channel *accel_ch; spdk_accel_completion_cb cb_fn; void *cb_arg; + spdk_accel_step_cb step_cb_fn; + void *step_cb_arg; + struct spdk_memory_domain *src_domain; + void *src_domain_ctx; + struct spdk_memory_domain *dst_domain; + void *dst_domain_ctx; union { struct { struct iovec *iovs; /* iovs passed by the caller */ @@ -50,6 +56,7 @@ struct spdk_accel_task { int flags; int status; TAILQ_ENTRY(spdk_accel_task) link; + TAILQ_ENTRY(spdk_accel_task) seq_link; }; struct spdk_accel_module_if { diff --git a/lib/accel/Makefile b/lib/accel/Makefile index 2013e5dd5..6265b24f8 100644 --- a/lib/accel/Makefile +++ b/lib/accel/Makefile @@ -6,7 +6,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -SO_VER := 10 +SO_VER := 11 SO_MINOR := 0 SO_SUFFIX := $(SO_VER).$(SO_MINOR) diff --git a/lib/accel/accel.c b/lib/accel/accel.c index ac26d6e19..a6ec13fe4 100644 --- a/lib/accel/accel.c +++ b/lib/accel/accel.c @@ -50,9 +50,22 @@ static const char *g_opcode_strings[ACCEL_OPC_LAST] = { }; struct accel_io_channel { - struct spdk_io_channel *module_ch[ACCEL_OPC_LAST]; - void *task_pool_base; - TAILQ_HEAD(, spdk_accel_task) task_pool; + struct spdk_io_channel *module_ch[ACCEL_OPC_LAST]; + void *task_pool_base; + struct spdk_accel_sequence *seq_pool_base; + TAILQ_HEAD(, spdk_accel_task) task_pool; + TAILQ_HEAD(, spdk_accel_sequence) seq_pool; +}; + +TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task); + +struct spdk_accel_sequence { + struct accel_io_channel *ch; + struct accel_sequence_tasks tasks; + struct accel_sequence_tasks completed; + spdk_accel_completion_cb cb_fn; + void *cb_arg; + TAILQ_ENTRY(spdk_accel_sequence) link; }; int @@ -186,6 +199,9 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, accel_task->op_code = ACCEL_OPC_COPY; accel_task->nbytes = nbytes; accel_task->flags = flags; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -217,6 +233,9 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_DUALCAST; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -241,6 +260,9 @@ spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, accel_task->src2 = src2; accel_task->nbytes = nbytes; accel_task->op_code = ACCEL_OPC_COMPARE; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -266,6 +288,9 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_FILL; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -292,6 +317,9 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, accel_task->seed = seed; accel_task->nbytes = nbytes; accel_task->op_code = ACCEL_OPC_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -329,6 +357,9 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, accel_task->crc_dst = crc_dst; accel_task->seed = seed; accel_task->op_code = ACCEL_OPC_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -357,6 +388,9 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COPY_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -404,6 +438,9 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COPY_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -436,6 +473,9 @@ spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbyte accel_task->nbytes_dst = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COMPRESS; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); @@ -463,12 +503,316 @@ spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs, accel_task->d.iovcnt = dst_iovcnt; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_DECOMPRESS; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); return 0; } +static inline struct spdk_accel_sequence * +accel_sequence_get(struct accel_io_channel *ch) +{ + struct spdk_accel_sequence *seq; + + seq = TAILQ_FIRST(&ch->seq_pool); + if (seq == NULL) { + return NULL; + } + + TAILQ_REMOVE(&ch->seq_pool, seq, link); + + TAILQ_INIT(&seq->tasks); + TAILQ_INIT(&seq->completed); + + seq->ch = ch; + + return seq; +} + +static inline void +accel_sequence_put(struct spdk_accel_sequence *seq) +{ + struct accel_io_channel *ch = seq->ch; + + assert(TAILQ_EMPTY(&seq->tasks)); + assert(TAILQ_EMPTY(&seq->completed)); + seq->ch = NULL; + + TAILQ_INSERT_HEAD(&ch->seq_pool, seq, link); +} + +static void accel_sequence_task_cb(void *cb_arg, int status); + +static inline struct spdk_accel_task * +accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq, + spdk_accel_step_cb cb_fn, void *cb_arg) +{ + struct spdk_accel_task *task; + + task = _get_task(ch, accel_sequence_task_cb, seq); + if (task == NULL) { + return task; + } + + task->step_cb_fn = cb_fn; + task->step_cb_arg = cb_arg; + + return task; +} + +int +spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, + struct iovec *dst_iovs, uint32_t dst_iovcnt, + struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, + struct iovec *src_iovs, uint32_t src_iovcnt, + struct spdk_memory_domain *src_domain, void *src_domain_ctx, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg) +{ + struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); + struct spdk_accel_task *task; + struct spdk_accel_sequence *seq = *pseq; + + if (dst_domain != NULL || src_domain != NULL) { + SPDK_ERRLOG("Memory domains are currently unsupported\n"); + return -EINVAL; + } + + if (seq == NULL) { + seq = accel_sequence_get(accel_ch); + if (spdk_unlikely(seq == NULL)) { + return -ENOMEM; + } + } + + assert(seq->ch == accel_ch); + task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); + if (spdk_unlikely(task == NULL)) { + if (*pseq == NULL) { + accel_sequence_put(seq); + } + + return -ENOMEM; + } + + task->dst_domain = dst_domain; + task->dst_domain_ctx = dst_domain_ctx; + task->d.iovs = dst_iovs; + task->d.iovcnt = dst_iovcnt; + task->src_domain = src_domain; + task->src_domain_ctx = src_domain_ctx; + task->s.iovs = src_iovs; + task->s.iovcnt = src_iovcnt; + task->flags = flags; + task->op_code = ACCEL_OPC_COPY; + + TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); + *pseq = seq; + + return 0; +} + +int +spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, + void *buf, uint64_t len, + struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg) +{ + struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); + struct spdk_accel_task *task; + struct spdk_accel_sequence *seq = *pseq; + + if (domain != NULL) { + SPDK_ERRLOG("Memory domains are currently unsupported\n"); + return -EINVAL; + } + + if (seq == NULL) { + seq = accel_sequence_get(accel_ch); + if (spdk_unlikely(seq == NULL)) { + return -ENOMEM; + } + } + + assert(seq->ch == accel_ch); + task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); + if (spdk_unlikely(task == NULL)) { + if (*pseq == NULL) { + accel_sequence_put(seq); + } + + return -ENOMEM; + } + + memset(&task->fill_pattern, pattern, sizeof(uint64_t)); + + task->src_domain = NULL; + task->dst_domain = domain; + task->dst_domain_ctx = domain_ctx; + task->dst = buf; + task->nbytes = len; + task->flags = flags; + task->op_code = ACCEL_OPC_FILL; + + TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); + *pseq = seq; + + return 0; +} + +static void +accel_sequence_complete_tasks(struct spdk_accel_sequence *seq) +{ + struct spdk_accel_task *task; + struct accel_io_channel *ch = seq->ch; + spdk_accel_step_cb cb_fn; + void *cb_arg; + + while (!TAILQ_EMPTY(&seq->completed)) { + task = TAILQ_FIRST(&seq->completed); + TAILQ_REMOVE(&seq->completed, task, seq_link); + cb_fn = task->step_cb_fn; + cb_arg = task->step_cb_arg; + TAILQ_INSERT_HEAD(&ch->task_pool, task, link); + if (cb_fn != NULL) { + cb_fn(cb_arg); + } + } + + while (!TAILQ_EMPTY(&seq->tasks)) { + task = TAILQ_FIRST(&seq->tasks); + TAILQ_REMOVE(&seq->tasks, task, seq_link); + cb_fn = task->step_cb_fn; + cb_arg = task->step_cb_arg; + TAILQ_INSERT_HEAD(&ch->task_pool, task, link); + if (cb_fn != NULL) { + cb_fn(cb_arg); + } + } +} + +static void +accel_sequence_complete(struct spdk_accel_sequence *seq, int status) +{ + SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, status); + + /* First notify all users that appended operations to this sequence */ + accel_sequence_complete_tasks(seq); + + /* Then notify the user that finished the sequence */ + seq->cb_fn(seq->cb_arg, status); + + accel_sequence_put(seq); +} + +static void +accel_process_sequence(struct spdk_accel_sequence *seq) +{ + struct accel_io_channel *accel_ch = seq->ch; + struct spdk_accel_module_if *module; + struct spdk_io_channel *module_ch; + struct spdk_accel_task *task; + int rc; + + task = TAILQ_FIRST(&seq->tasks); + if (task == NULL) { + /* We've processed all tasks */ + accel_sequence_complete(seq, 0); + return; + } + + SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n", + g_opcode_strings[task->op_code], seq); + + module = g_modules_opc[task->op_code]; + module_ch = accel_ch->module_ch[task->op_code]; + + rc = module->submit_tasks(module_ch, task); + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n", + g_opcode_strings[task->op_code], seq); + accel_sequence_complete(seq, rc); + } +} + +static void +accel_sequence_task_cb(void *cb_arg, int status) +{ + struct spdk_accel_sequence *seq = cb_arg; + struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks); + struct accel_io_channel *accel_ch = seq->ch; + + /* spdk_accel_task_complete() puts the task back to the task pool, but we don't want to do + * that if a task is part of a sequence. Removing the task from that pool here is the + * easiest way to prevent this, even though it is a bit hacky. + */ + assert(task != NULL); + TAILQ_REMOVE(&accel_ch->task_pool, task, link); + + TAILQ_REMOVE(&seq->tasks, task, seq_link); + TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); + + if (spdk_unlikely(status != 0)) { + SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n", + g_opcode_strings[task->op_code], seq); + accel_sequence_complete(seq, status); + return; + } + + accel_process_sequence(seq); +} + +int +spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, + spdk_accel_completion_cb cb_fn, void *cb_arg) +{ + struct spdk_accel_task *task; + + /* Since we store copy operations' buffers as iovecs, we need to convert them to scalar + * buffers, as that's what accel modules expect + */ + TAILQ_FOREACH(task, &seq->tasks, seq_link) { + if (task->op_code != ACCEL_OPC_COPY) { + continue; + } + + if (spdk_unlikely(task->s.iovcnt != 1 || task->d.iovcnt != 1)) { + SPDK_ERRLOG("Unable to set buffer of a copy operation due " + "to iovcnt!=1\n"); + return -EINVAL; + } + task->nbytes = spdk_min(task->s.iovs[0].iov_len, + task->d.iovs[0].iov_len); + task->src = task->s.iovs[0].iov_base; + task->dst = task->d.iovs[0].iov_base; + } + + seq->cb_fn = cb_fn; + seq->cb_arg = cb_arg; + + accel_process_sequence(seq); + + return 0; +} + +void +spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq) +{ + assert(0 && "unsupported"); +} + +void +spdk_accel_sequence_abort(struct spdk_accel_sequence *seq) +{ + if (seq == NULL) { + return; + } + + accel_sequence_complete_tasks(seq); + accel_sequence_put(seq); +} static struct spdk_accel_module_if * _module_find_by_name(const char *name) @@ -515,19 +859,28 @@ accel_create_channel(void *io_device, void *ctx_buf) { struct accel_io_channel *accel_ch = ctx_buf; struct spdk_accel_task *accel_task; + struct spdk_accel_sequence *seq; uint8_t *task_mem; - int i, j; + int i = 0, j; accel_ch->task_pool_base = calloc(MAX_TASKS_PER_CHANNEL, g_max_accel_module_size); if (accel_ch->task_pool_base == NULL) { return -ENOMEM; } + accel_ch->seq_pool_base = calloc(MAX_TASKS_PER_CHANNEL, sizeof(struct spdk_accel_sequence)); + if (accel_ch->seq_pool_base == NULL) { + goto err; + } + TAILQ_INIT(&accel_ch->task_pool); + TAILQ_INIT(&accel_ch->seq_pool); task_mem = accel_ch->task_pool_base; for (i = 0 ; i < MAX_TASKS_PER_CHANNEL; i++) { accel_task = (struct spdk_accel_task *)task_mem; + seq = &accel_ch->seq_pool_base[i]; TAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link); + TAILQ_INSERT_TAIL(&accel_ch->seq_pool, seq, link); task_mem += g_max_accel_module_size; } @@ -546,6 +899,7 @@ err: spdk_put_io_channel(accel_ch->module_ch[j]); } free(accel_ch->task_pool_base); + free(accel_ch->seq_pool_base); return -ENOMEM; } @@ -563,6 +917,7 @@ accel_destroy_channel(void *io_device, void *ctx_buf) } free(accel_ch->task_pool_base); + free(accel_ch->seq_pool_base); } struct spdk_io_channel * diff --git a/lib/accel/spdk_accel.map b/lib/accel/spdk_accel.map index f8e63c1c1..1862c33d1 100644 --- a/lib/accel/spdk_accel.map +++ b/lib/accel/spdk_accel.map @@ -18,6 +18,11 @@ spdk_accel_get_opc_module_name; spdk_accel_assign_opc; spdk_accel_write_config_json; + spdk_accel_append_copy; + spdk_accel_append_fill; + spdk_accel_sequence_finish; + spdk_accel_sequence_abort; + spdk_accel_sequence_reverse; # functions needed by modules spdk_accel_module_list_add; diff --git a/test/unit/lib/accel/accel.c/accel_ut.c b/test/unit/lib/accel/accel.c/accel_ut.c index 4c3c5c3b9..9e1779d4a 100644 --- a/test/unit/lib/accel/accel.c/accel_ut.c +++ b/test/unit/lib/accel/accel.c/accel_ut.c @@ -8,7 +8,7 @@ #include "spdk_internal/mock.h" #include "spdk_internal/accel_module.h" #include "thread/thread_internal.h" -#include "common/lib/test_env.c" +#include "common/lib/ut_multithread.c" #include "accel/accel.c" #include "accel/accel_sw.c" #include "unit/lib/json_mock.c" @@ -523,17 +523,579 @@ test_spdk_accel_module_register(void) CU_ASSERT(i == 4); } +struct ut_sequence { + bool complete; + int status; +}; + +static void +ut_sequence_step_cb(void *cb_arg) +{ + int *completed = cb_arg; + + (*completed)++; +} + +static void +ut_sequence_complete_cb(void *cb_arg, int status) +{ + struct ut_sequence *seq = cb_arg; + + seq->complete = true; + seq->status = status; +} + +static void +test_sequence_fill_copy(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + struct ut_sequence ut_seq; + char buf[4096], tmp[2][4096], expected[4096]; + struct iovec src_iovs[2], dst_iovs[2]; + int rc, completed; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + + /* First check the simplest case - single task in a sequence */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0xa5, sizeof(expected)); + completed = 0; + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + CU_ASSERT_EQUAL(completed, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 1); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check a single copy operation */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0xa5, sizeof(tmp[0])); + memset(expected, 0xa5, sizeof(expected)); + completed = 0; + seq = NULL; + + dst_iovs[0].iov_base = buf; + dst_iovs[0].iov_len = sizeof(buf); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 1); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check multiple fill operations */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0xfe, 4096); + memset(expected, 0xde, 2048); + memset(expected, 0xa5, 1024); + seq = NULL; + completed = 0; + rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 3); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check multiple copy operations */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0, sizeof(tmp[0])); + memset(tmp[1], 0, sizeof(tmp[1])); + memset(expected, 0xa5, sizeof(expected)); + seq = NULL; + completed = 0; + + rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[0].iov_base = tmp[1]; + dst_iovs[0].iov_len = sizeof(tmp[1]); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[1].iov_base = buf; + dst_iovs[1].iov_len = sizeof(buf); + src_iovs[1].iov_base = tmp[1]; + src_iovs[1].iov_len = sizeof(tmp[1]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, + &src_iovs[1], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 3); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check that adding a copy operation at the end will change destination buffer */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0, sizeof(tmp[0])); + memset(expected, 0xa5, sizeof(buf)); + seq = NULL; + completed = 0; + rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[0].iov_base = buf; + dst_iovs[0].iov_len = sizeof(buf); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check that it's also possible to add copy operation at the beginning */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0xde, sizeof(tmp[0])); + memset(tmp[1], 0, sizeof(tmp[1])); + memset(expected, 0xa5, sizeof(expected)); + seq = NULL; + completed = 0; + + dst_iovs[0].iov_base = tmp[1]; + dst_iovs[0].iov_len = sizeof(tmp[1]); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[1].iov_base = buf; + dst_iovs[1].iov_len = sizeof(buf); + src_iovs[1].iov_base = tmp[1]; + src_iovs[1].iov_len = sizeof(tmp[1]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, + &src_iovs[1], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 3); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + spdk_put_io_channel(ioch); + poll_threads(); +} + +static void +test_sequence_abort(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + char buf[4096], tmp[2][4096], expected[4096]; + struct iovec src_iovs[2], dst_iovs[2]; + int rc, completed; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + + /* Check that aborting a sequence calls operation's callback, the operation is not executed + * and the sequence is freed + */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0, sizeof(buf)); + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + spdk_accel_sequence_abort(seq); + CU_ASSERT_EQUAL(completed, 1); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check sequence with multiple operations */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0, sizeof(buf)); + completed = 0; + seq = NULL; + + dst_iovs[0].iov_base = tmp[1]; + dst_iovs[0].iov_len = sizeof(tmp[1]); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[1].iov_base = buf; + dst_iovs[1].iov_len = sizeof(buf); + src_iovs[1].iov_base = tmp[1]; + src_iovs[1].iov_len = sizeof(tmp[1]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, + &src_iovs[1], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + spdk_accel_sequence_abort(seq); + CU_ASSERT_EQUAL(completed, 4); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* This should be a no-op */ + spdk_accel_sequence_abort(NULL); + + spdk_put_io_channel(ioch); + poll_threads(); +} + +static void +test_sequence_append_error(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + struct accel_io_channel *accel_ch; + struct iovec src_iovs, dst_iovs; + char buf[4096]; + TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); + TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); + int rc; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + accel_ch = spdk_io_channel_get_ctx(ioch); + + /* Check that append fails and no sequence object is allocated when there are no more free + * tasks */ + TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); + + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = 2048; + src_iovs.iov_base = &buf[2048]; + src_iovs.iov_len = 2048; + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + /* Check that the same happens when the sequence queue is empty */ + TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); + TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); + + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = 2048; + src_iovs.iov_base = &buf[2048]; + src_iovs.iov_len = 2048; + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); + + spdk_put_io_channel(ioch); + poll_threads(); +} + +struct ut_sequence_operation { + int complete_status; + int submit_status; +}; + +static struct ut_sequence_operation g_seq_operations[ACCEL_OPC_LAST]; + +static int +ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) +{ + struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; + + if (op->submit_status != 0) { + return op->submit_status; + } + + spdk_accel_task_complete(task, op->complete_status); + + return 0; +} + +static void +test_sequence_completion_error(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + struct ut_sequence ut_seq; + struct iovec src_iovs, dst_iovs; + char buf[4096], tmp[4096]; + struct spdk_accel_module_if *modules[ACCEL_OPC_LAST]; + int i, rc, completed; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + + /* Override the submit_tasks function */ + g_module.submit_tasks = ut_sequnce_submit_tasks; + for (i = 0; i < ACCEL_OPC_LAST; ++i) { + modules[i] = g_modules_opc[i]; + g_modules_opc[i] = &g_module; + } + + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + + /* Check that if the first operation completes with an error, the whole sequence is + * completed with that error and that all operations' completion callbacks are executed + */ + g_seq_operations[ACCEL_OPC_FILL].complete_status = -E2BIG; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); + + /* Check the same with a second operation in the sequence */ + g_seq_operations[ACCEL_OPC_COPY].complete_status = -EACCES; + g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -EACCES); + + g_seq_operations[ACCEL_OPC_COPY].complete_status = 0; + g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; + + /* Check submission failure of the first operation */ + g_seq_operations[ACCEL_OPC_FILL].submit_status = -EADDRINUSE; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); + + /* Check the same with a second operation */ + g_seq_operations[ACCEL_OPC_COPY].submit_status = -EADDRNOTAVAIL; + g_seq_operations[ACCEL_OPC_FILL].submit_status = 0; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); + + /* Cleanup module pointers to make subsequent tests work correctly */ + for (i = 0; i < ACCEL_OPC_LAST; ++i) { + g_modules_opc[i] = modules[i]; + } + + spdk_put_io_channel(ioch); + poll_threads(); +} + +static int +test_sequence_setup(void) +{ + int rc; + + allocate_cores(1); + allocate_threads(1); + set_thread(0); + + rc = spdk_accel_initialize(); + if (rc != 0) { + CU_ASSERT(false); + return -1; + } + + return 0; +} + +static void +accel_finish_cb(void *cb_arg) +{ + bool *done = cb_arg; + + *done = true; +} + +static int +test_sequence_cleanup(void) +{ + bool done = false; + + spdk_accel_finish(accel_finish_cb, &done); + + while (!done) { + poll_threads(); + } + + free_threads(); + free_cores(); + + return 0; +} + int main(int argc, char **argv) { - CU_pSuite suite = NULL; + CU_pSuite suite = NULL, seq_suite; unsigned int num_failures; CU_set_error_action(CUEA_ABORT); CU_initialize_registry(); - suite = CU_add_suite("accel", test_setup, test_cleanup); + /* Sequence tests require accel to be initialized normally, so run them before the other + * tests which register accel modules which aren't fully implemented, causing accel + * initialization to fail. + */ + seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); + CU_ADD_TEST(seq_suite, test_sequence_fill_copy); + CU_ADD_TEST(seq_suite, test_sequence_abort); + CU_ADD_TEST(seq_suite, test_sequence_append_error); + CU_ADD_TEST(seq_suite, test_sequence_completion_error); + suite = CU_add_suite("accel", test_setup, test_cleanup); CU_ADD_TEST(suite, test_spdk_accel_task_complete); CU_ADD_TEST(suite, test_get_task); CU_ADD_TEST(suite, test_spdk_accel_submit_copy);