diff --git a/include/spdk_internal/accel_engine.h b/include/spdk_internal/accel_engine.h index f0273fabe..94a21b883 100644 --- a/include/spdk_internal/accel_engine.h +++ b/include/spdk_internal/accel_engine.h @@ -44,9 +44,7 @@ struct spdk_accel_task; void spdk_accel_task_complete(struct spdk_accel_task *task, int status); struct accel_io_channel { - struct spdk_accel_engine *engine; - struct spdk_io_channel *engine_ch; - struct spdk_io_channel *sw_engine_ch; + struct spdk_io_channel *engine_ch[ACCEL_OPC_LAST]; void *task_pool_base; TAILQ_HEAD(, spdk_accel_task) task_pool; }; @@ -85,9 +83,11 @@ struct spdk_accel_task { }; struct spdk_accel_engine { + const char *name; bool (*supports_opcode)(enum accel_opcode); struct spdk_io_channel *(*get_io_channel)(void); int (*submit_tasks)(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task); + TAILQ_ENTRY(spdk_accel_engine) tailq; }; struct spdk_accel_module_if { @@ -118,7 +118,7 @@ struct spdk_accel_module_if { TAILQ_ENTRY(spdk_accel_module_if) tailq; }; -void spdk_accel_hw_engine_register(struct spdk_accel_engine *accel_engine); +void spdk_accel_engine_register(struct spdk_accel_engine *accel_engine); void spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module); #define SPDK_ACCEL_MODULE_REGISTER(init_fn, fini_fn, config_json, ctx_size_fn) \ diff --git a/lib/accel/accel_engine.c b/lib/accel/accel_engine.c index 2056423e8..601656549 100644 --- a/lib/accel/accel_engine.c +++ b/lib/accel/accel_engine.c @@ -60,8 +60,6 @@ /* Largest context size for all accel modules */ static size_t g_max_accel_module_size = 0; -static struct spdk_accel_engine *g_hw_accel_engine = NULL; -static struct spdk_accel_engine *g_sw_accel_engine = NULL; static struct spdk_accel_module_if *g_accel_engine_module = NULL; static spdk_accel_fini_cb g_fini_cb_fn = NULL; static void *g_fini_cb_arg = NULL; @@ -70,46 +68,48 @@ static void *g_fini_cb_arg = NULL; static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list = TAILQ_HEAD_INITIALIZER(spdk_accel_module_list); -static void _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags); -static void _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags); -static void _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags); -static int _sw_accel_compare(void *src1, void *src2, size_t nbytes); -static void _sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags); -static void _sw_accel_crc32c(uint32_t *dst, void *src, uint32_t seed, size_t nbytes); -static void _sw_accel_crc32cv(uint32_t *dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed); +/* Global list of registered engines */ +static TAILQ_HEAD(, spdk_accel_engine) g_engine_list = + TAILQ_HEAD_INITIALIZER(g_engine_list); -/* Registration of hw modules (currently supports only 1 at a time) */ -void -spdk_accel_hw_engine_register(struct spdk_accel_engine *accel_engine) +/* Global array mapping capabilities to engines */ +static struct spdk_accel_engine *g_engines_opc[ACCEL_OPC_LAST] = {}; + +static int sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task); + +static struct spdk_accel_engine * +_engine_find_by_name(const char *name) { - if (g_hw_accel_engine == NULL) { - g_hw_accel_engine = accel_engine; - } else { - SPDK_NOTICELOG("Hardware offload engine already enabled\n"); + struct spdk_accel_engine *accel_engine = NULL; + + TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) { + if (strcmp(name, accel_engine->name) == 0) { + break; + } } + + return accel_engine; } -/* Registration of sw modules (currently supports only 1) */ -static void -accel_sw_register(struct spdk_accel_engine *accel_engine) +/* Registration of all engines */ +void +spdk_accel_engine_register(struct spdk_accel_engine *engine) { - assert(g_sw_accel_engine == NULL); - g_sw_accel_engine = accel_engine; -} + if (_engine_find_by_name(engine->name)) { + SPDK_NOTICELOG("Accel engine %s already registered\n", engine->name); + assert(false); + return; + } -static void -accel_sw_unregister(void) -{ - g_sw_accel_engine = NULL; -} - -/* Used to determine whether a command is sent to an engine/module or done here - * via SW implementation. - */ -inline static bool -_is_supported(struct spdk_accel_engine *engine, enum accel_opcode operation) -{ - return (engine->supports_opcode(operation)); + /* Make sure that the software engine is at the head of the list, this + * will assure that all opcodes are later assigned to software first and + * then udpated to HW engines as they are registered. + */ + if (strcmp(engine->name, "software") == 0) { + TAILQ_INSERT_HEAD(&g_engine_list, engine, tailq); + } else { + TAILQ_INSERT_TAIL(&g_engine_list, engine, tailq); + } } void @@ -152,10 +152,8 @@ _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, voi /* Post SW completions to a list and complete in a poller as we don't want to * complete them on the caller's stack as they'll likely submit another. */ inline static void -_add_to_comp_list(struct accel_io_channel *accel_ch, struct spdk_accel_task *accel_task, int status) +_add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status) { - struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(accel_ch->sw_engine_ch); - accel_task->status = status; TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link); } @@ -176,12 +174,13 @@ _check_flags(int flags) /* Accel framework public API for copy function */ int -spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes, - int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, + uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) { struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; - int rc; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY]; accel_task = _get_task(accel_ch, cb_fn, cb_arg); if (accel_task == NULL) { @@ -194,27 +193,19 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_ accel_task->nbytes = nbytes; accel_task->flags = flags; - if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - rc = _check_flags(flags); - if (rc) { - return rc; - } - _sw_accel_copy(dst, src, (size_t)nbytes, flags); - _add_to_comp_list(accel_ch, accel_task, 0); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Accel framework public API for dual cast copy function */ int -spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src, - uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, + void *dst2, void *src, uint64_t nbytes, int flags, + spdk_accel_completion_cb cb_fn, void *cb_arg) { struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; - int rc; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_DUALCAST]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_DUALCAST]; if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); @@ -233,27 +224,19 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, v accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_DUALCAST; - if (_is_supported(accel_ch->engine, ACCEL_OPC_DUALCAST)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - rc = _check_flags(flags); - if (rc) { - return rc; - } - _sw_accel_dualcast(dst1, dst2, src, (size_t)nbytes, flags); - _add_to_comp_list(accel_ch, accel_task, 0); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Accel framework public API for compare function */ int -spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes, - spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, + void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, + void *cb_arg) { struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; - int rc; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COMPARE]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COMPARE]; accel_task = _get_task(accel_ch, cb_fn, cb_arg); if (accel_task == NULL) { @@ -265,23 +248,19 @@ spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, ui accel_task->nbytes = nbytes; accel_task->op_code = ACCEL_OPC_COMPARE; - if (_is_supported(accel_ch->engine, ACCEL_OPC_COMPARE)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - rc = _sw_accel_compare(src1, src2, (size_t)nbytes); - _add_to_comp_list(accel_ch, accel_task, rc); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Accel framework public API for fill function */ int -spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes, - int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, + uint8_t fill, uint64_t nbytes, int flags, + spdk_accel_completion_cb cb_fn, void *cb_arg) { struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; - int rc; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_FILL]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_FILL]; accel_task = _get_task(accel_ch, cb_fn, cb_arg); if (accel_task == NULL) { @@ -294,26 +273,19 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_FILL; - if (_is_supported(accel_ch->engine, ACCEL_OPC_FILL)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - rc = _check_flags(flags); - if (rc) { - return rc; - } - _sw_accel_fill(dst, fill, (size_t)nbytes, flags); - _add_to_comp_list(accel_ch, accel_task, 0); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Accel framework public API for CRC-32C function */ int -spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *src, uint32_t seed, - uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, + void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, + void *cb_arg) { struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_CRC32C]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_CRC32C]; accel_task = _get_task(accel_ch, cb_fn, cb_arg); if (accel_task == NULL) { @@ -327,22 +299,19 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *sr accel_task->nbytes = nbytes; accel_task->op_code = ACCEL_OPC_CRC32C; - if (_is_supported(accel_ch->engine, ACCEL_OPC_CRC32C)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - _sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes); - _add_to_comp_list(accel_ch, accel_task, 0); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Accel framework public API for chained CRC-32C function */ int -spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct iovec *iov, - uint32_t iov_cnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, + struct iovec *iov, uint32_t iov_cnt, uint32_t seed, + spdk_accel_completion_cb cb_fn, void *cb_arg) { - struct accel_io_channel *accel_ch; + struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_CRC32C]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_CRC32C]; if (iov == NULL) { SPDK_ERRLOG("iov should not be NULL"); @@ -354,7 +323,6 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct return -EINVAL; } - accel_ch = spdk_io_channel_get_ctx(ch); accel_task = _get_task(accel_ch, cb_fn, cb_arg); if (accel_task == NULL) { SPDK_ERRLOG("no memory\n"); @@ -368,24 +336,19 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct accel_task->seed = seed; accel_task->op_code = ACCEL_OPC_CRC32C; - if (_is_supported(accel_ch->engine, ACCEL_OPC_CRC32C)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - _sw_accel_crc32cv(crc_dst, iov, iov_cnt, seed); - _add_to_comp_list(accel_ch, accel_task, 0); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Accel framework public API for copy with CRC-32C function */ int -spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src, - uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, int flags, - spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, + void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, + int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) { struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; - int rc; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY_CRC32C]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY_CRC32C]; accel_task = _get_task(accel_ch, cb_fn, cb_arg); if (accel_task == NULL) { @@ -401,29 +364,19 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src, accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COPY_CRC32C; - if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY_CRC32C)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - rc = _check_flags(flags); - if (rc) { - return rc; - } - _sw_accel_copy(dst, src, (size_t)nbytes, flags); - _sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes); - _add_to_comp_list(accel_ch, accel_task, 0); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Accel framework public API for chained copy + CRC-32C function */ int -spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iovec *src_iovs, - uint32_t iov_cnt, uint32_t *crc_dst, uint32_t seed, int flags, - spdk_accel_completion_cb cb_fn, void *cb_arg) +spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, + struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst, + uint32_t seed, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) { - struct accel_io_channel *accel_ch; + struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct spdk_accel_task *accel_task; - int rc; + struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY_CRC32C]; + struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY_CRC32C]; if (src_iovs == NULL) { SPDK_ERRLOG("iov should not be NULL"); @@ -435,7 +388,6 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov return -EINVAL; } - accel_ch = spdk_io_channel_get_ctx(ch); accel_task = _get_task(accel_ch, cb_fn, cb_arg); if (accel_task == NULL) { SPDK_ERRLOG("no memory\n"); @@ -451,18 +403,7 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov accel_task->flags = flags; accel_task->op_code = ACCEL_OPC_COPY_CRC32C; - if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY_CRC32C)) { - return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task); - } else { - rc = _check_flags(flags); - if (rc) { - return rc; - } - _sw_accel_copyv(dst, src_iovs, iov_cnt, flags); - _sw_accel_crc32cv(crc_dst, src_iovs, iov_cnt, seed); - _add_to_comp_list(accel_ch, accel_task, 0); - return 0; - } + return engine->submit_tasks(engine_ch, accel_task); } /* Helper function when when accel modules register with the framework. */ @@ -481,7 +422,7 @@ accel_engine_create_cb(void *io_device, void *ctx_buf) struct accel_io_channel *accel_ch = ctx_buf; struct spdk_accel_task *accel_task; uint8_t *task_mem; - int i; + int i, j; accel_ch->task_pool_base = calloc(MAX_TASKS_PER_CHANNEL, g_max_accel_module_size); if (accel_ch->task_pool_base == NULL) { @@ -496,21 +437,21 @@ accel_engine_create_cb(void *io_device, void *ctx_buf) task_mem += g_max_accel_module_size; } - /* Set sw engine channel for operations where hw engine does not support. */ - accel_ch->sw_engine_ch = g_sw_accel_engine->get_io_channel(); - assert(accel_ch->sw_engine_ch != NULL); - - if (g_hw_accel_engine != NULL) { - accel_ch->engine_ch = g_hw_accel_engine->get_io_channel(); - accel_ch->engine = g_hw_accel_engine; - } else { - /* No hw engine enabled, use sw. */ - accel_ch->engine_ch = accel_ch->sw_engine_ch; - accel_ch->engine = g_sw_accel_engine; + /* Assign engines and get IO channels for each */ + for (i = 0; i < ACCEL_OPC_LAST; i++) { + accel_ch->engine_ch[i] = g_engines_opc[i]->get_io_channel(); + /* This can happen if idxd runs out of channels. */ + if (accel_ch->engine_ch[i] == NULL) { + goto err; + } } - assert(accel_ch->engine_ch != NULL); return 0; +err: + for (j = 0; j < i; j++) { + spdk_put_io_channel(accel_ch->engine_ch[j]); + } + return -EINVAL; } /* Framework level channel destroy callback. */ @@ -518,11 +459,15 @@ static void accel_engine_destroy_cb(void *io_device, void *ctx_buf) { struct accel_io_channel *accel_ch = ctx_buf; + int i; - if (accel_ch->sw_engine_ch != accel_ch->engine_ch) { - spdk_put_io_channel(accel_ch->sw_engine_ch); + for (i = 0; i < ACCEL_OPC_LAST; i++) { + if (accel_ch->engine_ch[i]) { + spdk_put_io_channel(accel_ch->engine_ch[i]); + accel_ch->engine_ch[i] = NULL; + } } - spdk_put_io_channel(accel_ch->engine_ch); + free(accel_ch->task_pool_base); } @@ -545,12 +490,33 @@ accel_engine_module_initialize(void) int spdk_accel_engine_initialize(void) { - SPDK_NOTICELOG("Accel engine initialized to use software engine.\n"); + enum accel_opcode op; + struct spdk_accel_engine *accel_engine = NULL; + accel_engine_module_initialize(); + /* Create our priority global map of opcodes to engines, we populate starting + * with the software engine (guaranteed to be first on the list) and then + * updating opcodes with HW engines that have been initilaized. + * NOTE: all opcodes must be suported by software in the event that no HW + * engines are initilaized to support the operation. + */ + TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) { + for (op = 0; op < ACCEL_OPC_LAST; op++) { + if (accel_engine->supports_opcode(op)) { + g_engines_opc[op] = accel_engine; + SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_engine->name); + } + } + } +#ifdef DEBUG + for (op = 0; op < ACCEL_OPC_LAST; op++) { + assert(g_engines_opc[op] != NULL); + } +#endif /* * We need a unique identifier for the accel engine framework, so use the - * spdk_accel_module_list address for this purpose. + * spdk_accel_module_list address for this purpose. */ spdk_io_device_register(&spdk_accel_module_list, accel_engine_create_cb, accel_engine_destroy_cb, sizeof(struct accel_io_channel), "accel_module"); @@ -625,7 +591,17 @@ spdk_accel_engine_finish(spdk_accel_fini_cb cb_fn, void *cb_arg) static bool sw_accel_supports_opcode(enum accel_opcode opc) { - return false; + switch (opc) { + case ACCEL_OPC_COPY: + case ACCEL_OPC_FILL: + case ACCEL_OPC_DUALCAST: + case ACCEL_OPC_COMPARE: + case ACCEL_OPC_CRC32C: + case ACCEL_OPC_COPY_CRC32C: + return true; + default: + return false; + } } static inline void @@ -725,12 +701,87 @@ _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_ *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed); } +static int +sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) +{ + struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); + struct spdk_accel_task *tmp; + int rc = 0; + + do { + switch (accel_task->op_code) { + case ACCEL_OPC_COPY: + rc = _check_flags(accel_task->flags); + if (rc == 0) { + _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); + } + _add_to_comp_list(sw_ch, accel_task, rc); + break; + case ACCEL_OPC_FILL: + rc = _check_flags(accel_task->flags); + if (rc == 0) { + _sw_accel_fill(accel_task->dst, accel_task->fill_pattern, accel_task->nbytes, accel_task->flags); + } + _add_to_comp_list(sw_ch, accel_task, rc); + break; + case ACCEL_OPC_DUALCAST: + rc = _check_flags(accel_task->flags); + if (rc == 0) { + _sw_accel_dualcast(accel_task->dst, accel_task->dst2, accel_task->src, accel_task->nbytes, + accel_task->flags); + } + _add_to_comp_list(sw_ch, accel_task, rc); + break; + case ACCEL_OPC_COMPARE: + rc = _sw_accel_compare(accel_task->src, accel_task->src2, accel_task->nbytes); + _add_to_comp_list(sw_ch, accel_task, rc); + break; + case ACCEL_OPC_CRC32C: + if (accel_task->v.iovcnt == 0) { + _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); + } else { + _sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed); + } + _add_to_comp_list(sw_ch, accel_task, 0); + break; + case ACCEL_OPC_COPY_CRC32C: + rc = _check_flags(accel_task->flags); + if (rc == 0) { + if (accel_task->v.iovcnt == 0) { + _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); + _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); + } else { + _sw_accel_copyv(accel_task->dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->flags); + _sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed); + } + } + _add_to_comp_list(sw_ch, accel_task, rc); + break; + default: + assert(false); + break; + } + + tmp = TAILQ_NEXT(accel_task, link); + + /* Report any build errors via the callback now. */ + if (rc) { + spdk_accel_task_complete(accel_task, rc); + } + + accel_task = tmp; + } while (accel_task); + + return 0; +} + static struct spdk_io_channel *sw_accel_get_io_channel(void); - static struct spdk_accel_engine sw_accel_engine = { + .name = "software", .supports_opcode = sw_accel_supports_opcode, .get_io_channel = sw_accel_get_io_channel, + .submit_tasks = sw_accel_submit_tasks, }; static int @@ -788,7 +839,8 @@ sw_accel_engine_get_ctx_size(void) static int sw_accel_engine_init(void) { - accel_sw_register(&sw_accel_engine); + SPDK_NOTICELOG("Accel framework software engine initialized.\n"); + spdk_accel_engine_register(&sw_accel_engine); spdk_io_device_register(&sw_accel_engine, sw_accel_create_cb, sw_accel_destroy_cb, sizeof(struct sw_accel_io_channel), "sw_accel_engine"); @@ -798,8 +850,17 @@ sw_accel_engine_init(void) static void sw_accel_engine_fini(void *ctxt) { + struct spdk_accel_engine *accel_engine; + spdk_io_device_unregister(&sw_accel_engine, NULL); - accel_sw_unregister(); + + /* unregister the software engine */ + TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) { + if (strcmp(accel_engine->name, "software") == 0) { + TAILQ_REMOVE(&g_engine_list, accel_engine, tailq); + break; + } + } spdk_accel_engine_module_finish(); } diff --git a/lib/accel/spdk_accel.map b/lib/accel/spdk_accel.map index f695392d9..ce95de420 100644 --- a/lib/accel/spdk_accel.map +++ b/lib/accel/spdk_accel.map @@ -17,7 +17,7 @@ spdk_accel_write_config_json; # functions needed by modules - spdk_accel_hw_engine_register; + spdk_accel_engine_register; spdk_accel_module_list_add; spdk_accel_task_complete; diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index 9713483a3..7e540771c 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -130,7 +130,9 @@ static void idxd_done(void *cb_arg, int status) { struct spdk_accel_task *accel_task = cb_arg; - struct idxd_io_channel *chan = spdk_io_channel_get_ctx(accel_task->accel_ch->engine_ch); + struct idxd_io_channel *chan; + + chan = spdk_io_channel_get_ctx(accel_task->accel_ch->engine_ch[accel_task->op_code]); assert(chan->num_outstanding > 0); spdk_trace_record(TRACE_IDXD_OP_COMPLETE, 0, 0, 0, chan->num_outstanding - 1); @@ -302,7 +304,7 @@ idxd_poll(void *arg) TAILQ_INIT(&chan->queued_tasks); - idxd_submit_tasks(task->accel_ch->engine_ch, task); + idxd_submit_tasks(task->accel_ch->engine_ch[task->op_code], task); } } @@ -332,6 +334,7 @@ idxd_supports_opcode(enum accel_opcode opc) } static struct spdk_accel_engine idxd_accel_engine = { + .name = "idxd", .supports_opcode = idxd_supports_opcode, .get_io_channel = idxd_get_io_channel, .submit_tasks = idxd_submit_tasks, @@ -419,8 +422,8 @@ accel_engine_idxd_init(void) } g_idxd_initialized = true; - SPDK_NOTICELOG("Accel engine updated to use IDXD DSA engine.\n"); - spdk_accel_hw_engine_register(&idxd_accel_engine); + SPDK_NOTICELOG("Accel framework IDXD engine initialized.\n"); + spdk_accel_engine_register(&idxd_accel_engine); spdk_io_device_register(&idxd_accel_engine, idxd_create_cb, idxd_destroy_cb, sizeof(struct idxd_io_channel), "idxd_accel_engine"); return 0; diff --git a/module/accel/ioat/accel_engine_ioat.c b/module/accel/ioat/accel_engine_ioat.c index a2490a531..18565e4a6 100644 --- a/module/accel/ioat/accel_engine_ioat.c +++ b/module/accel/ioat/accel_engine_ioat.c @@ -182,6 +182,7 @@ ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task } static struct spdk_accel_engine ioat_accel_engine = { + .name = "ioat", .supports_opcode = ioat_supports_opcode, .get_io_channel = ioat_get_io_channel, .submit_tasks = ioat_submit_tasks, @@ -289,8 +290,8 @@ accel_engine_ioat_init(void) } g_ioat_initialized = true; - SPDK_NOTICELOG("Accel engine updated to use IOAT engine.\n"); - spdk_accel_hw_engine_register(&ioat_accel_engine); + SPDK_NOTICELOG("Accel framework IOAT engine initialized.\n"); + spdk_accel_engine_register(&ioat_accel_engine); spdk_io_device_register(&ioat_accel_engine, ioat_create_cb, ioat_destroy_cb, sizeof(struct ioat_io_channel), "ioat_accel_engine"); return 0; diff --git a/module/accel/ioat/accel_engine_ioat_rpc.c b/module/accel/ioat/accel_engine_ioat_rpc.c index 0b3d832a4..7c9450b4d 100644 --- a/module/accel/ioat/accel_engine_ioat_rpc.c +++ b/module/accel/ioat/accel_engine_ioat_rpc.c @@ -47,6 +47,7 @@ rpc_ioat_scan_accel_engine(struct spdk_jsonrpc_request *request, return; } + SPDK_NOTICELOG("Enabling IOAT\n"); accel_engine_ioat_enable_probe(); spdk_jsonrpc_send_bool_response(request, true); diff --git a/test/unit/lib/Makefile b/test/unit/lib/Makefile index 7275063f9..6c9f71a21 100644 --- a/test/unit/lib/Makefile +++ b/test/unit/lib/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -DIRS-y = accel bdev blob blobfs dma event ioat iscsi json jsonrpc log lvol +DIRS-y = accel bdev blob blobfs dma event ioat iscsi json jsonrpc log lvol DIRS-y += notify nvme nvmf scsi sock thread util env_dpdk init rpc DIRS-$(CONFIG_IDXD) += idxd DIRS-$(CONFIG_REDUCE) += reduce diff --git a/test/unit/lib/accel/accel.c/accel_engine_ut.c b/test/unit/lib/accel/accel.c/accel_engine_ut.c index a367f707d..00580ce1b 100644 --- a/test/unit/lib/accel/accel.c/accel_engine_ut.c +++ b/test/unit/lib/accel/accel.c/accel_engine_ut.c @@ -73,6 +73,8 @@ _supports_opcode(enum accel_opcode opc) static int test_setup(void) { + int i; + g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); if (g_ch == NULL) { /* for some reason the assert fatal macro doesn't work in the setup function. */ @@ -85,9 +87,13 @@ test_setup(void) CU_ASSERT(false); return -1; } - g_accel_ch->engine_ch = g_engine_ch; - g_accel_ch->sw_engine_ch = g_engine_ch; - g_sw_ch = (struct sw_accel_io_channel *)((char *)g_accel_ch->sw_engine_ch + sizeof( + + g_accel_engine.submit_tasks = sw_accel_submit_tasks; + for (i = 0; i < ACCEL_OPC_LAST; i++) { + g_accel_ch->engine_ch[i] = g_engine_ch; + g_engines_opc[i] = &g_accel_engine; + } + g_sw_ch = (struct sw_accel_io_channel *)((char *)g_engine_ch + sizeof( struct spdk_io_channel)); TAILQ_INIT(&g_sw_ch->tasks_to_complete); g_accel_engine.supports_opcode = _supports_opcode; @@ -103,52 +109,6 @@ test_cleanup(void) return 0; } -static void -test_spdk_accel_hw_engine_register(void) -{ - /* Run once with no engine assigned, assign it. */ - g_hw_accel_engine = NULL; - spdk_accel_hw_engine_register(&g_accel_engine); - CU_ASSERT(g_hw_accel_engine == &g_accel_engine); - - /* Run with one assigned, should not change. */ - spdk_accel_hw_engine_register(&g_accel_engine); - CU_ASSERT(g_hw_accel_engine == &g_accel_engine); -} - -static int -test_accel_sw_register(void) -{ - /* Run once with no engine assigned, assign it. */ - g_sw_accel_engine = NULL; - accel_sw_register(&g_accel_engine); - CU_ASSERT(g_sw_accel_engine == &g_accel_engine); - - return 0; -} - -static void -test_accel_sw_unregister(void) -{ - /* Run once engine assigned, make sure it gets unassigned. */ - g_sw_accel_engine = &g_accel_engine; - accel_sw_unregister(); - CU_ASSERT(g_sw_accel_engine == NULL); -} - -static void -test_is_supported(void) -{ - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY) | _accel_op_to_bit( - ACCEL_OPC_DUALCAST) | - _accel_op_to_bit(ACCEL_OPC_CRC32C); - CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_COPY) == true); - CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_FILL) == false); - CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_DUALCAST) == true); - CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_COMPARE) == false); - CU_ASSERT(_is_supported(&g_accel_engine, ACCEL_OPC_CRC32C) == true); -} - #define DUMMY_ARG 0xDEADBEEF static bool g_dummy_cb_called = false; static void @@ -206,22 +166,6 @@ test_get_task(void) CU_ASSERT(_task.accel_ch == g_accel_ch); } -static bool g_dummy_submit_called = false; -static int -dummy_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task) -{ - g_dummy_submit_called = true; - return 0; -} - -static bool g_dummy_submit_cb_called = false; -static void -dummy_submit_cb_fn(void *cb_arg, int status) -{ - g_dummy_submit_cb_called = true; - CU_ASSERT(status == 0); -} - #define TEST_SUBMIT_SIZE 64 static void test_spdk_accel_submit_copy(void) @@ -238,48 +182,21 @@ test_spdk_accel_submit_copy(void) TAILQ_INIT(&g_accel_ch->task_pool); /* Fail with no tasks on _get_task() */ - rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, dummy_submit_cb_fn, cb_arg); + rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == -ENOMEM); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; task.accel_ch = g_accel_ch; task.flags = 1; TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_accel_ch->engine = &g_accel_engine; - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* HW accel submission OK. */ - rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, dummy_submit_cb_fn, cb_arg); + /* submission OK. */ + rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == 0); CU_ASSERT(task.dst == dst); CU_ASSERT(task.src == src); CU_ASSERT(task.op_code == ACCEL_OPC_COPY); CU_ASSERT(task.nbytes == nbytes); CU_ASSERT(task.flags == 0); - CU_ASSERT(g_dummy_submit_called == true); - - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - /* reset values before next case */ - g_dummy_submit_called = false; - g_opc_mask = 0; - task.dst = 0; - task.src = 0; - task.op_code = 0xff; - task.nbytes = 0; - task.flags = 1; - - /* SW engine does copy. */ - rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, dummy_submit_cb_fn, cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(task.dst == dst); - CU_ASSERT(task.src == src); - CU_ASSERT(task.op_code == ACCEL_OPC_COPY); - CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(task.flags == 0); - CU_ASSERT(g_dummy_submit_cb_called == false); CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); @@ -300,6 +217,8 @@ test_spdk_accel_submit_dualcast(void) struct spdk_accel_task *expected_accel_task = NULL; int flags = 0; + TAILQ_INIT(&g_accel_ch->task_pool); + /* Dualcast requires 4K alignment on dst addresses, * hence using the hard coded address to test the buffer alignment */ @@ -309,69 +228,33 @@ test_spdk_accel_submit_dualcast(void) SPDK_CU_ASSERT_FATAL(src != NULL); memset(src, 0x5A, TEST_SUBMIT_SIZE); - TAILQ_INIT(&g_accel_ch->task_pool); - /* This should fail since dst2 is not 4k aligned */ - rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn, - cb_arg); + rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == -EINVAL); dst1 = (void *)0x7010; dst2 = (void *)0x6000; /* This should fail since dst1 is not 4k aligned */ - rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn, - cb_arg); + rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == -EINVAL); /* Dualcast requires 4K alignment on dst addresses */ dst1 = (void *)0x7000; dst2 = (void *)0x6000; /* Fail with no tasks on _get_task() */ - rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn, - cb_arg); + rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == -ENOMEM); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; - task.accel_ch = g_accel_ch; TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_accel_ch->engine = &g_accel_engine; - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_DUALCAST); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* HW accel submission OK. */ - rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn, - cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(task.dst == dst1); - CU_ASSERT(task.dst2 == dst2); - CU_ASSERT(task.src == src); - CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); - CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(task.flags == 0); - CU_ASSERT(g_dummy_submit_called == true); - - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - /* Reset values before next case */ - g_dummy_submit_called = false; - g_opc_mask = 0; - task.dst = 0; - task.dst2 = 0; - task.src = 0; - task.op_code = 0xff; - task.nbytes = 0; - task.flags = 1; - /* Since we test the SW path next, need to use valid memory addresses - * cannot hardcode them anymore - */ + /* accel submission OK., since we test the SW path , need to use valid memory addresses + * cannot hardcode them anymore */ dst1 = spdk_dma_zmalloc(nbytes, align, NULL); SPDK_CU_ASSERT_FATAL(dst1 != NULL); dst2 = spdk_dma_zmalloc(nbytes, align, NULL); SPDK_CU_ASSERT_FATAL(dst2 != NULL); /* SW engine does the dualcast. */ - rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, dummy_submit_cb_fn, - cb_arg); + rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == 0); CU_ASSERT(task.dst == dst1); CU_ASSERT(task.dst2 == dst2); @@ -379,7 +262,6 @@ test_spdk_accel_submit_dualcast(void) CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); CU_ASSERT(task.nbytes == nbytes); CU_ASSERT(task.flags == 0); - CU_ASSERT(g_dummy_submit_cb_called == false); CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); @@ -402,54 +284,26 @@ test_spdk_accel_submit_compare(void) struct spdk_accel_task task; struct spdk_accel_task *expected_accel_task = NULL; + TAILQ_INIT(&g_accel_ch->task_pool); + src1 = calloc(1, TEST_SUBMIT_SIZE); SPDK_CU_ASSERT_FATAL(src1 != NULL); src2 = calloc(1, TEST_SUBMIT_SIZE); SPDK_CU_ASSERT_FATAL(src2 != NULL); /* Fail with no tasks on _get_task() */ - rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, dummy_submit_cb_fn, cb_arg); + rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); CU_ASSERT(rc == -ENOMEM); - TAILQ_INIT(&g_accel_ch->task_pool); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; - task.accel_ch = g_accel_ch; TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_accel_ch->engine = &g_accel_engine; - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COMPARE); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* HW accel submission OK. */ - rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, dummy_submit_cb_fn, cb_arg); + /* accel submission OK. */ + rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); CU_ASSERT(rc == 0); CU_ASSERT(task.src == src1); CU_ASSERT(task.src2 == src2); CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(g_dummy_submit_called == true); - - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - /* Reset values before next case */ - g_dummy_submit_called = false; - g_opc_mask = 0; - task.src = 0; - task.src2 = 0; - task.op_code = 0xff; - task.nbytes = 0; - - memset(src1, 0x5A, TEST_SUBMIT_SIZE); - memset(src2, 0x5A, TEST_SUBMIT_SIZE); - - /* SW engine does compare. */ - rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, dummy_submit_cb_fn, cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(task.src == src1); - CU_ASSERT(task.src2 == src2); - CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); - CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(g_dummy_submit_cb_called == false); CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); @@ -473,6 +327,8 @@ test_spdk_accel_submit_fill(void) struct spdk_accel_task *expected_accel_task = NULL; int flags = 0; + TAILQ_INIT(&g_accel_ch->task_pool); + dst = calloc(1, TEST_SUBMIT_SIZE); SPDK_CU_ASSERT_FATAL(dst != NULL); src = calloc(1, TEST_SUBMIT_SIZE); @@ -481,48 +337,20 @@ test_spdk_accel_submit_fill(void) memset(&fill64, fill, sizeof(uint64_t)); /* Fail with no tasks on _get_task() */ - rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, dummy_submit_cb_fn, cb_arg); + rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == -ENOMEM); - TAILQ_INIT(&g_accel_ch->task_pool); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; - task.accel_ch = g_accel_ch; TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_accel_ch->engine = &g_accel_engine; - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_FILL); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* HW accel submission OK. */ - rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, dummy_submit_cb_fn, cb_arg); + /* accel submission OK. */ + rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); CU_ASSERT(rc == 0); CU_ASSERT(task.dst == dst); CU_ASSERT(task.fill_pattern == fill64); CU_ASSERT(task.op_code == ACCEL_OPC_FILL); CU_ASSERT(task.nbytes == nbytes); CU_ASSERT(task.flags == 0); - CU_ASSERT(g_dummy_submit_called == true); - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - /* Reset values before next case */ - g_dummy_submit_called = false; - g_opc_mask = 0; - task.dst = 0; - task.fill_pattern = 0; - task.op_code = 0xff; - task.nbytes = 0; - task.flags = 1; - - /* SW engine does the fill. */ - rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, dummy_submit_cb_fn, cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(task.dst == dst); - CU_ASSERT(task.fill_pattern == fill64); - CU_ASSERT(task.op_code == ACCEL_OPC_FILL); - CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(task.flags == 0); - CU_ASSERT(g_dummy_submit_cb_called == false); CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); @@ -544,85 +372,16 @@ test_spdk_accel_submit_crc32c(void) struct spdk_accel_task task; struct spdk_accel_task *expected_accel_task = NULL; - /* Fail with no tasks on _get_task() */ - rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg); - CU_ASSERT(rc == -ENOMEM); - TAILQ_INIT(&g_accel_ch->task_pool); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; - task.accel_ch = g_accel_ch; - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - - g_accel_ch->engine = &g_accel_engine; - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_CRC32C); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* HW accel submission OK. */ - rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(task.crc_dst == &crc_dst); - CU_ASSERT(task.src == src); - CU_ASSERT(task.v.iovcnt == 0); - CU_ASSERT(task.seed == seed); - CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); - CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(g_dummy_submit_called == true); - - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - /* Reset values before next case */ - g_dummy_submit_called = false; - g_opc_mask = 0; - task.crc_dst = 0; - task.src = 0; - task.seed = 0; - task.op_code = 0xff; - task.nbytes = 0; - - /* SW engine does crc. */ - rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(task.crc_dst == &crc_dst); - CU_ASSERT(task.src == src); - CU_ASSERT(task.v.iovcnt == 0); - CU_ASSERT(task.seed == seed); - CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); - CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(g_dummy_submit_cb_called == false); - expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); - TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); - CU_ASSERT(expected_accel_task == &task); -} - -static void -test_spdk_accel_submit_crc32c_hw_engine_unsupported(void) -{ - const uint64_t nbytes = TEST_SUBMIT_SIZE; - uint32_t crc_dst; - uint8_t src[TEST_SUBMIT_SIZE]; - uint32_t seed = 1; - void *cb_arg = NULL; - int rc; - struct spdk_accel_task task; - struct spdk_accel_task *expected_accel_task = NULL; /* Fail with no tasks on _get_task() */ - rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg); + rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); CU_ASSERT(rc == -ENOMEM); - TAILQ_INIT(&g_accel_ch->task_pool); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; - task.accel_ch = g_accel_ch; TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_accel_ch->engine = &g_accel_engine; - /* HW engine only supports COPY and does not support CRC */ - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* Summit to HW engine while eventually handled by SW engine. */ - rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, dummy_submit_cb_fn, cb_arg); + /* accel submission OK. */ + rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); CU_ASSERT(rc == 0); CU_ASSERT(task.crc_dst == &crc_dst); CU_ASSERT(task.src == src); @@ -630,10 +389,6 @@ test_spdk_accel_submit_crc32c_hw_engine_unsupported(void) CU_ASSERT(task.seed == seed); CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); CU_ASSERT(task.nbytes == nbytes); - /* Not set in HW engine callback while handled by SW engine instead. */ - CU_ASSERT(g_dummy_submit_called == false); - - /* SW engine does crc. */ expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); CU_ASSERT(expected_accel_task == &task); @@ -652,55 +407,27 @@ test_spdk_accel_submit_crc32cv(void) struct iovec iov[32]; struct spdk_accel_task *expected_accel_task = NULL; + TAILQ_INIT(&g_accel_ch->task_pool); + for (i = 0; i < iov_cnt; i++) { iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); iov[i].iov_len = TEST_SUBMIT_SIZE; } - TAILQ_INIT(&g_accel_ch->task_pool); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; - task.accel_ch = g_accel_ch; task.nbytes = TEST_SUBMIT_SIZE; TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_accel_ch->engine = &g_accel_engine; - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_CRC32C); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* HW accel submission OK. */ - rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, dummy_submit_cb_fn, cb_arg); + /* accel submission OK. */ + rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); CU_ASSERT(rc == 0); CU_ASSERT(task.v.iovs == iov); CU_ASSERT(task.v.iovcnt == iov_cnt); CU_ASSERT(task.crc_dst == &crc_dst); CU_ASSERT(task.seed == seed); CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); - CU_ASSERT(g_dummy_submit_called == true); - CU_ASSERT(task.cb_fn == dummy_submit_cb_fn); CU_ASSERT(task.cb_arg == cb_arg); CU_ASSERT(task.nbytes == iov[0].iov_len); - - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_dummy_submit_called = false; - g_opc_mask = 0; - task.v.iovs = 0; - task.v.iovcnt = 0; - task.crc_dst = 0; - task.seed = 0; - task.op_code = 0xff; - - /* SW engine submit crc. */ - rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, dummy_submit_cb_fn, cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(task.v.iovs == iov); - CU_ASSERT(task.v.iovcnt == iov_cnt); - CU_ASSERT(task.crc_dst == &crc_dst); - CU_ASSERT(task.seed == seed); - CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); - CU_ASSERT(g_dummy_submit_cb_called == false); - expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); CU_ASSERT(expected_accel_task == &task); @@ -724,26 +451,18 @@ test_spdk_accel_submit_copy_crc32c(void) struct spdk_accel_task *expected_accel_task = NULL; int flags = 0; + TAILQ_INIT(&g_accel_ch->task_pool); + /* Fail with no tasks on _get_task() */ rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, - dummy_submit_cb_fn, - cb_arg); + NULL, cb_arg); CU_ASSERT(rc == -ENOMEM); - TAILQ_INIT(&g_accel_ch->task_pool); - task.cb_fn = dummy_submit_cb_fn; - task.cb_arg = cb_arg; - task.accel_ch = g_accel_ch; TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_accel_ch->engine = &g_accel_engine; - g_opc_mask = _accel_op_to_bit(ACCEL_OPC_COPY_CRC32C); - g_accel_ch->engine->submit_tasks = dummy_submit_tasks; - - /* HW accel submission OK. */ + /* accel submission OK. */ rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, - dummy_submit_cb_fn, - cb_arg); + NULL, cb_arg); CU_ASSERT(rc == 0); CU_ASSERT(task.dst == dst); CU_ASSERT(task.src == src); @@ -753,41 +472,75 @@ test_spdk_accel_submit_copy_crc32c(void) CU_ASSERT(task.nbytes == nbytes); CU_ASSERT(task.flags == 0); CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); - CU_ASSERT(g_dummy_submit_called == true); - - TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); - g_dummy_submit_called = false; - g_opc_mask = 0; - task.dst = 0; - task.src = 0; - task.crc_dst = 0; - task.v.iovcnt = 0; - task.seed = 0; - task.nbytes = 0; - task.flags = 1; - task.op_code = 0xff; - memset(src, 0x5A, TEST_SUBMIT_SIZE); - - /* SW engine does copy crc. */ - rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, - dummy_submit_cb_fn, - cb_arg); - CU_ASSERT(rc == 0); - CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); - CU_ASSERT(task.dst == dst); - CU_ASSERT(task.src == src); - CU_ASSERT(task.crc_dst == &crc_dst); - CU_ASSERT(task.v.iovcnt == 0); - CU_ASSERT(task.seed == seed); - CU_ASSERT(task.nbytes == nbytes); - CU_ASSERT(task.flags == 0); - CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); - CU_ASSERT(g_dummy_submit_cb_called == false); expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); CU_ASSERT(expected_accel_task == &task); } +static void +test_engine_find_by_name(void) +{ + struct spdk_accel_engine eng1, eng2, eng3; + struct spdk_accel_engine *accel_engine = NULL; + + eng1.name = "ioat"; + eng2.name = "idxd"; + eng3.name = "software"; + + TAILQ_INIT(&g_engine_list); + TAILQ_INSERT_TAIL(&g_engine_list, &eng1, tailq); + TAILQ_INSERT_TAIL(&g_engine_list, &eng2, tailq); + TAILQ_INSERT_TAIL(&g_engine_list, &eng3, tailq); + + /* Now let's find a valid engine */ + accel_engine = _engine_find_by_name("ioat"); + CU_ASSERT(accel_engine != NULL); + + /* Try to find one that doesn't exist */ + accel_engine = _engine_find_by_name("XXX"); + CU_ASSERT(accel_engine == NULL); +} + +static void +test_spdk_accel_engine_register(void) +{ + struct spdk_accel_engine eng1, eng2, eng3, eng4; + struct spdk_accel_engine *accel_engine = NULL; + int i = 0; + + eng1.name = "ioat"; + eng2.name = "idxd"; + eng3.name = "software"; + eng4.name = "nothing"; + + spdk_accel_engine_register(&eng1); + spdk_accel_engine_register(&eng2); + spdk_accel_engine_register(&eng3); + spdk_accel_engine_register(&eng4); + + /* Now confirm they're in the right order. */ + TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) { + switch (i++) { + case 0: + CU_ASSERT(strcmp(accel_engine->name, "software") == 0); + break; + case 1: + CU_ASSERT(strcmp(accel_engine->name, "ioat") == 0); + break; + case 2: + CU_ASSERT(strcmp(accel_engine->name, "idxd") == 0); + break; + case 3: + CU_ASSERT(strcmp(accel_engine->name, "nothing") == 0); + break; + default: + CU_ASSERT(false); + break; + } + } + CU_ASSERT(i == 4); +} + int main(int argc, char **argv) { CU_pSuite suite = NULL; @@ -798,10 +551,7 @@ int main(int argc, char **argv) suite = CU_add_suite("accel", test_setup, test_cleanup); - CU_ADD_TEST(suite, test_spdk_accel_hw_engine_register); - CU_ADD_TEST(suite, test_accel_sw_register); - CU_ADD_TEST(suite, test_accel_sw_unregister); - CU_ADD_TEST(suite, test_is_supported); + CU_ADD_TEST(suite, test_spdk_accel_engine_register); CU_ADD_TEST(suite, test_spdk_accel_task_complete); CU_ADD_TEST(suite, test_get_task); CU_ADD_TEST(suite, test_spdk_accel_submit_copy); @@ -809,9 +559,9 @@ int main(int argc, char **argv) CU_ADD_TEST(suite, test_spdk_accel_submit_compare); CU_ADD_TEST(suite, test_spdk_accel_submit_fill); CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); - CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c_hw_engine_unsupported); CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); + CU_ADD_TEST(suite, test_engine_find_by_name); CU_basic_set_mode(CU_BRM_VERBOSE); CU_basic_run_tests();