diff --git a/include/spdk/accel.h b/include/spdk/accel.h index 9ac8f7c39..482a77bc7 100644 --- a/include/spdk/accel.h +++ b/include/spdk/accel.h @@ -11,6 +11,7 @@ #define SPDK_ACCEL_H #include "spdk/stdinc.h" +#include "spdk/dma.h" #ifdef __cplusplus extern "C" { @@ -260,6 +261,92 @@ int spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_i size_t src_iovcnt, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg); +/** Object grouping multiple accel operations to be executed at the same point in time */ +struct spdk_accel_sequence; + +/** + * Completion callback of a single operation within a sequence. After it's executed, the sequence + * object might be freed, so users should not touch it. + */ +typedef void (*spdk_accel_step_cb)(void *cb_arg); + +/** + * Append a copy operation to a sequence. Copy operation in a sequence is special, as it is not + * guaranteed that the data will be actually copied. If it's possible, it will only change + * source / destination buffers of some of the operations in a sequence. + * + * \param seq Sequence object. If NULL, a new sequence object will be created. + * \param ch I/O channel. + * \param dst_iovs Destination I/O vector array. + * \param dst_iovcnt Size of the `dst_iovs` array. + * \param dst_domain Memory domain to which the destination buffers belong. + * \param dst_domain_ctx Destination buffer domain context. + * \param src_iovs Source I/O vector array. + * \param src_iovcnt Size of the `src_iovs` array. + * \param src_domain Memory domain to which the source buffers belong. + * \param src_domain_ctx Source buffer domain context. + * \param flags Accel operation flags. + * \param cb_fn Callback to be executed once this operation is completed. + * \param cb_arg Argument to be passed to `cb_fn`. + * + * \return 0 if operation was successfully added to the sequence, negative errno otherwise. + */ +int spdk_accel_append_copy(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch, + struct iovec *dst_iovs, uint32_t dst_iovcnt, + struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, + struct iovec *src_iovs, uint32_t src_iovcnt, + struct spdk_memory_domain *src_domain, void *src_domain_ctx, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg); + +/** + * Append a fill operation to a sequence. + * + * \param seq Sequence object. If NULL, a new sequence object will be created. + * \param ch I/O channel. + * \param buf Data buffer. + * \param len Length of the data buffer. + * \param domain Memory domain to which the data buffer belongs. + * \param domain_ctx Buffer domain context. + * \param pattern Pattern to fill the buffer with. + * \param flags Accel operation flags. + * \param cb_fn Callback to be executed once this operation is completed. + * \param cb_arg Argument to be passed to `cb_fn`. + * + * \return 0 if operation was successfully added to the sequence, negative errno otherwise. + */ +int spdk_accel_append_fill(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch, + void *buf, uint64_t len, + struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg); + +/** + * Finish a sequence and execute all its operations. After the completion callback is executed, the + * sequence object is automatically freed. + * + * \param seq Sequence to finish. + * \param cb_fn Completion callback to be executed once all operations are executed. + * \param cb_arg Argument to be passed to `cb_fn`. + * + * \return 0 on success, negative errno otherwise. + */ +int spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, + spdk_accel_completion_cb cb_fn, void *cb_arg); + +/** + * Reverse a sequence, so that the last operation becomes the first and vice versa. + * + * \param seq Sequence to reverse. + */ +void spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq); + +/** + * Abort a sequence. This will execute the completion callbacks of all operations that were added + * to the sequence and will then free the sequence object. + * + * \param seq Sequence to abort. + */ +void spdk_accel_sequence_abort(struct spdk_accel_sequence *seq); + /** * Return the name of the module assigned to a specific opcode. * diff --git a/include/spdk_internal/accel_module.h b/include/spdk_internal/accel_module.h index a4cde32fa..2bf6188b6 100644 --- a/include/spdk_internal/accel_module.h +++ b/include/spdk_internal/accel_module.h @@ -20,6 +20,12 @@ struct spdk_accel_task { struct accel_io_channel *accel_ch; spdk_accel_completion_cb cb_fn; void *cb_arg; + spdk_accel_step_cb step_cb_fn; + void *step_cb_arg; + struct spdk_memory_domain *src_domain; + void *src_domain_ctx; + struct spdk_memory_domain *dst_domain; + void *dst_domain_ctx; union { struct { struct iovec *iovs; /* iovs passed by the caller */ @@ -50,6 +56,7 @@ struct spdk_accel_task { int flags; int status; TAILQ_ENTRY(spdk_accel_task) link; + TAILQ_ENTRY(spdk_accel_task) seq_link; }; struct spdk_accel_module_if { diff --git a/lib/accel/Makefile b/lib/accel/Makefile index 2013e5dd5..6265b24f8 100644 --- a/lib/accel/Makefile +++ b/lib/accel/Makefile @@ -6,7 +6,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -SO_VER := 10 +SO_VER := 11 SO_MINOR := 0 SO_SUFFIX := $(SO_VER).$(SO_MINOR) diff --git a/lib/accel/accel.c b/lib/accel/accel.c index ac26d6e19..a6ec13fe4 100644 --- a/lib/accel/accel.c +++ b/lib/accel/accel.c @@ -50,9 +50,22 @@ static const char *g_opcode_strings[ACCEL_OPC_LAST] = { }; struct accel_io_channel { - struct spdk_io_channel *module_ch[ACCEL_OPC_LAST]; - void *task_pool_base; - TAILQ_HEAD(, spdk_accel_task) task_pool; + struct spdk_io_channel *module_ch[ACCEL_OPC_LAST]; + void *task_pool_base; + struct spdk_accel_sequence *seq_pool_base; + TAILQ_HEAD(, spdk_accel_task) task_pool; + TAILQ_HEAD(, spdk_accel_sequence) seq_pool; +}; + +TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task); + +struct spdk_accel_sequence { + struct accel_io_channel *ch; + struct accel_sequence_tasks tasks; + struct accel_sequence_tasks completed; + spdk_accel_completion_cb cb_fn; + void *cb_arg; + TAILQ_ENTRY(spdk_accel_sequence) link; }; int @@ -186,6 +199,9 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, accel_task->op_code = ACCEL_OPC_COPY; accel_task->nbytes = nbytes; accel_task->flags = flags; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -217,6 +233,9 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_DUALCAST; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -241,6 +260,9 @@ spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, accel_task->src2 = src2; accel_task->nbytes = nbytes; accel_task->op_code = ACCEL_OPC_COMPARE; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -266,6 +288,9 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_FILL; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -292,6 +317,9 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, accel_task->seed = seed; accel_task->nbytes = nbytes; accel_task->op_code = ACCEL_OPC_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -329,6 +357,9 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, accel_task->crc_dst = crc_dst; accel_task->seed = seed; accel_task->op_code = ACCEL_OPC_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -357,6 +388,9 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COPY_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -404,6 +438,9 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, accel_task->nbytes = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COPY_CRC32C; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); } @@ -436,6 +473,9 @@ spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbyte accel_task->nbytes_dst = nbytes; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COMPRESS; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); @@ -463,12 +503,316 @@ spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs, accel_task->d.iovcnt = dst_iovcnt; accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_DECOMPRESS; + accel_task->src_domain = NULL; + accel_task->dst_domain = NULL; + accel_task->step_cb_fn = NULL; return module->submit_tasks(module_ch, accel_task); return 0; } +static inline struct spdk_accel_sequence * +accel_sequence_get(struct accel_io_channel *ch) +{ + struct spdk_accel_sequence *seq; + + seq = TAILQ_FIRST(&ch->seq_pool); + if (seq == NULL) { + return NULL; + } + + TAILQ_REMOVE(&ch->seq_pool, seq, link); + + TAILQ_INIT(&seq->tasks); + TAILQ_INIT(&seq->completed); + + seq->ch = ch; + + return seq; +} + +static inline void +accel_sequence_put(struct spdk_accel_sequence *seq) +{ + struct accel_io_channel *ch = seq->ch; + + assert(TAILQ_EMPTY(&seq->tasks)); + assert(TAILQ_EMPTY(&seq->completed)); + seq->ch = NULL; + + TAILQ_INSERT_HEAD(&ch->seq_pool, seq, link); +} + +static void accel_sequence_task_cb(void *cb_arg, int status); + +static inline struct spdk_accel_task * +accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq, + spdk_accel_step_cb cb_fn, void *cb_arg) +{ + struct spdk_accel_task *task; + + task = _get_task(ch, accel_sequence_task_cb, seq); + if (task == NULL) { + return task; + } + + task->step_cb_fn = cb_fn; + task->step_cb_arg = cb_arg; + + return task; +} + +int +spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, + struct iovec *dst_iovs, uint32_t dst_iovcnt, + struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, + struct iovec *src_iovs, uint32_t src_iovcnt, + struct spdk_memory_domain *src_domain, void *src_domain_ctx, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg) +{ + struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); + struct spdk_accel_task *task; + struct spdk_accel_sequence *seq = *pseq; + + if (dst_domain != NULL || src_domain != NULL) { + SPDK_ERRLOG("Memory domains are currently unsupported\n"); + return -EINVAL; + } + + if (seq == NULL) { + seq = accel_sequence_get(accel_ch); + if (spdk_unlikely(seq == NULL)) { + return -ENOMEM; + } + } + + assert(seq->ch == accel_ch); + task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); + if (spdk_unlikely(task == NULL)) { + if (*pseq == NULL) { + accel_sequence_put(seq); + } + + return -ENOMEM; + } + + task->dst_domain = dst_domain; + task->dst_domain_ctx = dst_domain_ctx; + task->d.iovs = dst_iovs; + task->d.iovcnt = dst_iovcnt; + task->src_domain = src_domain; + task->src_domain_ctx = src_domain_ctx; + task->s.iovs = src_iovs; + task->s.iovcnt = src_iovcnt; + task->flags = flags; + task->op_code = ACCEL_OPC_COPY; + + TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); + *pseq = seq; + + return 0; +} + +int +spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, + void *buf, uint64_t len, + struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, + int flags, spdk_accel_step_cb cb_fn, void *cb_arg) +{ + struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); + struct spdk_accel_task *task; + struct spdk_accel_sequence *seq = *pseq; + + if (domain != NULL) { + SPDK_ERRLOG("Memory domains are currently unsupported\n"); + return -EINVAL; + } + + if (seq == NULL) { + seq = accel_sequence_get(accel_ch); + if (spdk_unlikely(seq == NULL)) { + return -ENOMEM; + } + } + + assert(seq->ch == accel_ch); + task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); + if (spdk_unlikely(task == NULL)) { + if (*pseq == NULL) { + accel_sequence_put(seq); + } + + return -ENOMEM; + } + + memset(&task->fill_pattern, pattern, sizeof(uint64_t)); + + task->src_domain = NULL; + task->dst_domain = domain; + task->dst_domain_ctx = domain_ctx; + task->dst = buf; + task->nbytes = len; + task->flags = flags; + task->op_code = ACCEL_OPC_FILL; + + TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); + *pseq = seq; + + return 0; +} + +static void +accel_sequence_complete_tasks(struct spdk_accel_sequence *seq) +{ + struct spdk_accel_task *task; + struct accel_io_channel *ch = seq->ch; + spdk_accel_step_cb cb_fn; + void *cb_arg; + + while (!TAILQ_EMPTY(&seq->completed)) { + task = TAILQ_FIRST(&seq->completed); + TAILQ_REMOVE(&seq->completed, task, seq_link); + cb_fn = task->step_cb_fn; + cb_arg = task->step_cb_arg; + TAILQ_INSERT_HEAD(&ch->task_pool, task, link); + if (cb_fn != NULL) { + cb_fn(cb_arg); + } + } + + while (!TAILQ_EMPTY(&seq->tasks)) { + task = TAILQ_FIRST(&seq->tasks); + TAILQ_REMOVE(&seq->tasks, task, seq_link); + cb_fn = task->step_cb_fn; + cb_arg = task->step_cb_arg; + TAILQ_INSERT_HEAD(&ch->task_pool, task, link); + if (cb_fn != NULL) { + cb_fn(cb_arg); + } + } +} + +static void +accel_sequence_complete(struct spdk_accel_sequence *seq, int status) +{ + SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, status); + + /* First notify all users that appended operations to this sequence */ + accel_sequence_complete_tasks(seq); + + /* Then notify the user that finished the sequence */ + seq->cb_fn(seq->cb_arg, status); + + accel_sequence_put(seq); +} + +static void +accel_process_sequence(struct spdk_accel_sequence *seq) +{ + struct accel_io_channel *accel_ch = seq->ch; + struct spdk_accel_module_if *module; + struct spdk_io_channel *module_ch; + struct spdk_accel_task *task; + int rc; + + task = TAILQ_FIRST(&seq->tasks); + if (task == NULL) { + /* We've processed all tasks */ + accel_sequence_complete(seq, 0); + return; + } + + SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n", + g_opcode_strings[task->op_code], seq); + + module = g_modules_opc[task->op_code]; + module_ch = accel_ch->module_ch[task->op_code]; + + rc = module->submit_tasks(module_ch, task); + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n", + g_opcode_strings[task->op_code], seq); + accel_sequence_complete(seq, rc); + } +} + +static void +accel_sequence_task_cb(void *cb_arg, int status) +{ + struct spdk_accel_sequence *seq = cb_arg; + struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks); + struct accel_io_channel *accel_ch = seq->ch; + + /* spdk_accel_task_complete() puts the task back to the task pool, but we don't want to do + * that if a task is part of a sequence. Removing the task from that pool here is the + * easiest way to prevent this, even though it is a bit hacky. + */ + assert(task != NULL); + TAILQ_REMOVE(&accel_ch->task_pool, task, link); + + TAILQ_REMOVE(&seq->tasks, task, seq_link); + TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); + + if (spdk_unlikely(status != 0)) { + SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n", + g_opcode_strings[task->op_code], seq); + accel_sequence_complete(seq, status); + return; + } + + accel_process_sequence(seq); +} + +int +spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, + spdk_accel_completion_cb cb_fn, void *cb_arg) +{ + struct spdk_accel_task *task; + + /* Since we store copy operations' buffers as iovecs, we need to convert them to scalar + * buffers, as that's what accel modules expect + */ + TAILQ_FOREACH(task, &seq->tasks, seq_link) { + if (task->op_code != ACCEL_OPC_COPY) { + continue; + } + + if (spdk_unlikely(task->s.iovcnt != 1 || task->d.iovcnt != 1)) { + SPDK_ERRLOG("Unable to set buffer of a copy operation due " + "to iovcnt!=1\n"); + return -EINVAL; + } + task->nbytes = spdk_min(task->s.iovs[0].iov_len, + task->d.iovs[0].iov_len); + task->src = task->s.iovs[0].iov_base; + task->dst = task->d.iovs[0].iov_base; + } + + seq->cb_fn = cb_fn; + seq->cb_arg = cb_arg; + + accel_process_sequence(seq); + + return 0; +} + +void +spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq) +{ + assert(0 && "unsupported"); +} + +void +spdk_accel_sequence_abort(struct spdk_accel_sequence *seq) +{ + if (seq == NULL) { + return; + } + + accel_sequence_complete_tasks(seq); + accel_sequence_put(seq); +} static struct spdk_accel_module_if * _module_find_by_name(const char *name) @@ -515,19 +859,28 @@ accel_create_channel(void *io_device, void *ctx_buf) { struct accel_io_channel *accel_ch = ctx_buf; struct spdk_accel_task *accel_task; + struct spdk_accel_sequence *seq; uint8_t *task_mem; - int i, j; + int i = 0, j; accel_ch->task_pool_base = calloc(MAX_TASKS_PER_CHANNEL, g_max_accel_module_size); if (accel_ch->task_pool_base == NULL) { return -ENOMEM; } + accel_ch->seq_pool_base = calloc(MAX_TASKS_PER_CHANNEL, sizeof(struct spdk_accel_sequence)); + if (accel_ch->seq_pool_base == NULL) { + goto err; + } + TAILQ_INIT(&accel_ch->task_pool); + TAILQ_INIT(&accel_ch->seq_pool); task_mem = accel_ch->task_pool_base; for (i = 0 ; i < MAX_TASKS_PER_CHANNEL; i++) { accel_task = (struct spdk_accel_task *)task_mem; + seq = &accel_ch->seq_pool_base[i]; TAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link); + TAILQ_INSERT_TAIL(&accel_ch->seq_pool, seq, link); task_mem += g_max_accel_module_size; } @@ -546,6 +899,7 @@ err: spdk_put_io_channel(accel_ch->module_ch[j]); } free(accel_ch->task_pool_base); + free(accel_ch->seq_pool_base); return -ENOMEM; } @@ -563,6 +917,7 @@ accel_destroy_channel(void *io_device, void *ctx_buf) } free(accel_ch->task_pool_base); + free(accel_ch->seq_pool_base); } struct spdk_io_channel * diff --git a/lib/accel/spdk_accel.map b/lib/accel/spdk_accel.map index f8e63c1c1..1862c33d1 100644 --- a/lib/accel/spdk_accel.map +++ b/lib/accel/spdk_accel.map @@ -18,6 +18,11 @@ spdk_accel_get_opc_module_name; spdk_accel_assign_opc; spdk_accel_write_config_json; + spdk_accel_append_copy; + spdk_accel_append_fill; + spdk_accel_sequence_finish; + spdk_accel_sequence_abort; + spdk_accel_sequence_reverse; # functions needed by modules spdk_accel_module_list_add; diff --git a/test/unit/lib/accel/accel.c/accel_ut.c b/test/unit/lib/accel/accel.c/accel_ut.c index 4c3c5c3b9..9e1779d4a 100644 --- a/test/unit/lib/accel/accel.c/accel_ut.c +++ b/test/unit/lib/accel/accel.c/accel_ut.c @@ -8,7 +8,7 @@ #include "spdk_internal/mock.h" #include "spdk_internal/accel_module.h" #include "thread/thread_internal.h" -#include "common/lib/test_env.c" +#include "common/lib/ut_multithread.c" #include "accel/accel.c" #include "accel/accel_sw.c" #include "unit/lib/json_mock.c" @@ -523,17 +523,579 @@ test_spdk_accel_module_register(void) CU_ASSERT(i == 4); } +struct ut_sequence { + bool complete; + int status; +}; + +static void +ut_sequence_step_cb(void *cb_arg) +{ + int *completed = cb_arg; + + (*completed)++; +} + +static void +ut_sequence_complete_cb(void *cb_arg, int status) +{ + struct ut_sequence *seq = cb_arg; + + seq->complete = true; + seq->status = status; +} + +static void +test_sequence_fill_copy(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + struct ut_sequence ut_seq; + char buf[4096], tmp[2][4096], expected[4096]; + struct iovec src_iovs[2], dst_iovs[2]; + int rc, completed; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + + /* First check the simplest case - single task in a sequence */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0xa5, sizeof(expected)); + completed = 0; + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + CU_ASSERT_EQUAL(completed, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 1); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check a single copy operation */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0xa5, sizeof(tmp[0])); + memset(expected, 0xa5, sizeof(expected)); + completed = 0; + seq = NULL; + + dst_iovs[0].iov_base = buf; + dst_iovs[0].iov_len = sizeof(buf); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 1); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check multiple fill operations */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0xfe, 4096); + memset(expected, 0xde, 2048); + memset(expected, 0xa5, 1024); + seq = NULL; + completed = 0; + rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 3); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check multiple copy operations */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0, sizeof(tmp[0])); + memset(tmp[1], 0, sizeof(tmp[1])); + memset(expected, 0xa5, sizeof(expected)); + seq = NULL; + completed = 0; + + rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[0].iov_base = tmp[1]; + dst_iovs[0].iov_len = sizeof(tmp[1]); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[1].iov_base = buf; + dst_iovs[1].iov_len = sizeof(buf); + src_iovs[1].iov_base = tmp[1]; + src_iovs[1].iov_len = sizeof(tmp[1]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, + &src_iovs[1], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 3); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check that adding a copy operation at the end will change destination buffer */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0, sizeof(tmp[0])); + memset(expected, 0xa5, sizeof(buf)); + seq = NULL; + completed = 0; + rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[0].iov_base = buf; + dst_iovs[0].iov_len = sizeof(buf); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check that it's also possible to add copy operation at the beginning */ + memset(buf, 0, sizeof(buf)); + memset(tmp[0], 0xde, sizeof(tmp[0])); + memset(tmp[1], 0, sizeof(tmp[1])); + memset(expected, 0xa5, sizeof(expected)); + seq = NULL; + completed = 0; + + dst_iovs[0].iov_base = tmp[1]; + dst_iovs[0].iov_len = sizeof(tmp[1]); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[1].iov_base = buf; + dst_iovs[1].iov_len = sizeof(buf); + src_iovs[1].iov_base = tmp[1]; + src_iovs[1].iov_len = sizeof(tmp[1]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, + &src_iovs[1], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 3); + CU_ASSERT(ut_seq.complete); + CU_ASSERT_EQUAL(ut_seq.status, 0); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + spdk_put_io_channel(ioch); + poll_threads(); +} + +static void +test_sequence_abort(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + char buf[4096], tmp[2][4096], expected[4096]; + struct iovec src_iovs[2], dst_iovs[2]; + int rc, completed; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + + /* Check that aborting a sequence calls operation's callback, the operation is not executed + * and the sequence is freed + */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0, sizeof(buf)); + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + spdk_accel_sequence_abort(seq); + CU_ASSERT_EQUAL(completed, 1); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* Check sequence with multiple operations */ + memset(buf, 0, sizeof(buf)); + memset(expected, 0, sizeof(buf)); + completed = 0; + seq = NULL; + + dst_iovs[0].iov_base = tmp[1]; + dst_iovs[0].iov_len = sizeof(tmp[1]); + src_iovs[0].iov_base = tmp[0]; + src_iovs[0].iov_len = sizeof(tmp[0]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, + &src_iovs[0], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs[1].iov_base = buf; + dst_iovs[1].iov_len = sizeof(buf); + src_iovs[1].iov_base = tmp[1]; + src_iovs[1].iov_len = sizeof(tmp[1]); + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, + &src_iovs[1], 1, NULL, NULL, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + spdk_accel_sequence_abort(seq); + CU_ASSERT_EQUAL(completed, 4); + CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); + + /* This should be a no-op */ + spdk_accel_sequence_abort(NULL); + + spdk_put_io_channel(ioch); + poll_threads(); +} + +static void +test_sequence_append_error(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + struct accel_io_channel *accel_ch; + struct iovec src_iovs, dst_iovs; + char buf[4096]; + TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); + TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); + int rc; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + accel_ch = spdk_io_channel_get_ctx(ioch); + + /* Check that append fails and no sequence object is allocated when there are no more free + * tasks */ + TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); + + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = 2048; + src_iovs.iov_base = &buf[2048]; + src_iovs.iov_len = 2048; + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + /* Check that the same happens when the sequence queue is empty */ + TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); + TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); + + rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = 2048; + src_iovs.iov_base = &buf[2048]; + src_iovs.iov_len = 2048; + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); + CU_ASSERT_EQUAL(rc, -ENOMEM); + CU_ASSERT_PTR_NULL(seq); + + TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); + + spdk_put_io_channel(ioch); + poll_threads(); +} + +struct ut_sequence_operation { + int complete_status; + int submit_status; +}; + +static struct ut_sequence_operation g_seq_operations[ACCEL_OPC_LAST]; + +static int +ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) +{ + struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; + + if (op->submit_status != 0) { + return op->submit_status; + } + + spdk_accel_task_complete(task, op->complete_status); + + return 0; +} + +static void +test_sequence_completion_error(void) +{ + struct spdk_accel_sequence *seq = NULL; + struct spdk_io_channel *ioch; + struct ut_sequence ut_seq; + struct iovec src_iovs, dst_iovs; + char buf[4096], tmp[4096]; + struct spdk_accel_module_if *modules[ACCEL_OPC_LAST]; + int i, rc, completed; + + ioch = spdk_accel_get_io_channel(); + SPDK_CU_ASSERT_FATAL(ioch != NULL); + + /* Override the submit_tasks function */ + g_module.submit_tasks = ut_sequnce_submit_tasks; + for (i = 0; i < ACCEL_OPC_LAST; ++i) { + modules[i] = g_modules_opc[i]; + g_modules_opc[i] = &g_module; + } + + memset(buf, 0, sizeof(buf)); + memset(tmp, 0, sizeof(tmp)); + + /* Check that if the first operation completes with an error, the whole sequence is + * completed with that error and that all operations' completion callbacks are executed + */ + g_seq_operations[ACCEL_OPC_FILL].complete_status = -E2BIG; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); + + /* Check the same with a second operation in the sequence */ + g_seq_operations[ACCEL_OPC_COPY].complete_status = -EACCES; + g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -EACCES); + + g_seq_operations[ACCEL_OPC_COPY].complete_status = 0; + g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; + + /* Check submission failure of the first operation */ + g_seq_operations[ACCEL_OPC_FILL].submit_status = -EADDRINUSE; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); + + /* Check the same with a second operation */ + g_seq_operations[ACCEL_OPC_COPY].submit_status = -EADDRNOTAVAIL; + g_seq_operations[ACCEL_OPC_FILL].submit_status = 0; + completed = 0; + seq = NULL; + rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, + ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + dst_iovs.iov_base = buf; + dst_iovs.iov_len = sizeof(buf); + src_iovs.iov_base = tmp; + src_iovs.iov_len = sizeof(tmp); + + rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, + &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, &completed); + CU_ASSERT_EQUAL(rc, 0); + + ut_seq.complete = false; + rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); + CU_ASSERT_EQUAL(rc, 0); + + poll_threads(); + CU_ASSERT_EQUAL(completed, 2); + CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); + + /* Cleanup module pointers to make subsequent tests work correctly */ + for (i = 0; i < ACCEL_OPC_LAST; ++i) { + g_modules_opc[i] = modules[i]; + } + + spdk_put_io_channel(ioch); + poll_threads(); +} + +static int +test_sequence_setup(void) +{ + int rc; + + allocate_cores(1); + allocate_threads(1); + set_thread(0); + + rc = spdk_accel_initialize(); + if (rc != 0) { + CU_ASSERT(false); + return -1; + } + + return 0; +} + +static void +accel_finish_cb(void *cb_arg) +{ + bool *done = cb_arg; + + *done = true; +} + +static int +test_sequence_cleanup(void) +{ + bool done = false; + + spdk_accel_finish(accel_finish_cb, &done); + + while (!done) { + poll_threads(); + } + + free_threads(); + free_cores(); + + return 0; +} + int main(int argc, char **argv) { - CU_pSuite suite = NULL; + CU_pSuite suite = NULL, seq_suite; unsigned int num_failures; CU_set_error_action(CUEA_ABORT); CU_initialize_registry(); - suite = CU_add_suite("accel", test_setup, test_cleanup); + /* Sequence tests require accel to be initialized normally, so run them before the other + * tests which register accel modules which aren't fully implemented, causing accel + * initialization to fail. + */ + seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); + CU_ADD_TEST(seq_suite, test_sequence_fill_copy); + CU_ADD_TEST(seq_suite, test_sequence_abort); + CU_ADD_TEST(seq_suite, test_sequence_append_error); + CU_ADD_TEST(seq_suite, test_sequence_completion_error); + suite = CU_add_suite("accel", test_setup, test_cleanup); CU_ADD_TEST(suite, test_spdk_accel_task_complete); CU_ADD_TEST(suite, test_get_task); CU_ADD_TEST(suite, test_spdk_accel_submit_copy);