ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

accel: store in-use modules in an extra structure

Browse Source 
It allows accel to store private data per each opcode/module without
having to change externally visible structures or allocate anything when
a module is registered. Since a single module can service multiple
opcodes at the same time, so some of these values might be duplicated.
However, there are only a handful of opcodes, so it shouldn't be a
problem.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I609a6ccc2d241cb9b8273cc2c6d1933d2bc25e0e
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16026
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Konrad Sztyber 2022-12-14 16:48:34 +01:00 committed by Jim Harris
parent 81fe7ef0af
commit a6fef9b194
2 changed files with 46 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
lib/accel/accel.c
View File

@ -35,6 +35,10 @@
#define ACCEL_BUFFER_BASE ((void *)(1ull << 63))
#define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1)
struct accel_module {
struct spdk_accel_module_if *module;
};
/* Largest context size for all accel modules */
static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task);
@ -53,7 +57,7 @@ static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_
static struct spdk_spinlock g_keyring_spin;
/* Global array mapping capabilities to modules */
static struct spdk_accel_module_if *g_modules_opc[ACCEL_OPC_LAST] = {};
static struct accel_module g_modules_opc[ACCEL_OPC_LAST] = {};
static char *g_modules_opc_override[ACCEL_OPC_LAST] = {};
static const char *g_opcode_strings[ACCEL_OPC_LAST] = {
@ -160,8 +164,8 @@ spdk_accel_get_opc_module_name(enum accel_opcode opcode, const char **module_nam
return -EINVAL;
}
if (g_modules_opc[opcode]) {
*module_name = g_modules_opc[opcode]->name;
if (g_modules_opc[opcode].module) {
*module_name = g_modules_opc[opcode].module->name;
} else {
return -ENOENT;
}
@ -270,7 +274,7 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COPY];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
@ -303,7 +307,7 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DUALCAST];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DUALCAST].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_DUALCAST];
if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
@ -345,7 +349,7 @@ spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COMPARE];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COMPARE].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COMPARE];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
@ -377,7 +381,7 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_FILL];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_FILL].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_FILL];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
@ -407,7 +411,7 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_CRC32C];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_CRC32C].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_CRC32C];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
@ -437,7 +441,7 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_CRC32C];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_CRC32C].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_CRC32C];
if (iov == NULL) {
@ -477,7 +481,7 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY_CRC32C];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY_CRC32C].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COPY_CRC32C];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
@ -512,7 +516,7 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY_CRC32C];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY_CRC32C].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COPY_CRC32C];
uint64_t nbytes;
uint32_t i;
@ -563,7 +567,7 @@ spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbyte
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COMPRESS];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COMPRESS].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COMPRESS];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
@ -596,7 +600,7 @@ spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs,
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DECOMPRESS];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DECOMPRESS].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_DECOMPRESS];
accel_task = _get_task(accel_ch, cb_fn, cb_arg);
@ -628,7 +632,7 @@ spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_k
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_ENCRYPT];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_ENCRYPT].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_ENCRYPT];
if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
@ -662,7 +666,7 @@ spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_k
{
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_task;
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DECRYPT];
struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DECRYPT].module;
struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_DECRYPT];
if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) {
@ -1341,7 +1345,7 @@ accel_process_sequence(struct spdk_accel_sequence *seq)
assert(task->src_domain == NULL);
assert(task->dst_domain == NULL);
module = g_modules_opc[task->op_code];
module = g_modules_opc[task->op_code].module;
module_ch = accel_ch->module_ch[task->op_code];
accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK);
@ -1640,11 +1644,11 @@ spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *pa
return -EINVAL;
}
if (g_modules_opc[ACCEL_OPC_ENCRYPT] != g_modules_opc[ACCEL_OPC_DECRYPT]) {
if (g_modules_opc[ACCEL_OPC_ENCRYPT].module != g_modules_opc[ACCEL_OPC_DECRYPT].module) {
/* hardly ever possible, but let's check and warn the user */
SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n");
}
module = g_modules_opc[ACCEL_OPC_ENCRYPT];
module = g_modules_opc[ACCEL_OPC_ENCRYPT].module;
if (!module) {
SPDK_ERRLOG("No accel module found assigned for crypto operation\n");
@ -1843,7 +1847,7 @@ accel_create_channel(void *io_device, void *ctx_buf)
/* Assign modules and get IO channels for each */
for (i = 0; i < ACCEL_OPC_LAST; i++) {
accel_ch->module_ch[i] = g_modules_opc[i]->get_io_channel();
accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel();
/* This can happen if idxd runs out of channels. */
if (accel_ch->module_ch[i] == NULL) {
goto err;
@ -1932,7 +1936,7 @@ spdk_accel_initialize(void)
TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) {
for (op = 0; op < ACCEL_OPC_LAST; op++) {
if (accel_module->supports_opcode(op)) {
g_modules_opc[op] = accel_module;
g_modules_opc[op].module = accel_module;
SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name);
}
}
@ -1952,18 +1956,18 @@ spdk_accel_initialize(void)
rc = -EINVAL;
goto error;
}
g_modules_opc[op] = accel_module;
g_modules_opc[op].module = accel_module;
}
}
if (g_modules_opc[ACCEL_OPC_ENCRYPT] != g_modules_opc[ACCEL_OPC_DECRYPT]) {
if (g_modules_opc[ACCEL_OPC_ENCRYPT].module != g_modules_opc[ACCEL_OPC_DECRYPT].module) {
SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations");
return -EINVAL;
}
#ifdef DEBUG
for (op = 0; op < ACCEL_OPC_LAST; op++) {
assert(g_modules_opc[op] != NULL);
assert(g_modules_opc[op].module != NULL);
}
#endif
rc = spdk_iobuf_register_module("accel");
@ -2135,7 +2139,7 @@ spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
free(g_modules_opc_override[op]);
g_modules_opc_override[op] = NULL;
}
g_modules_opc[op] = NULL;
g_modules_opc[op].module = NULL;
}
spdk_io_device_unregister(&spdk_accel_module_list, NULL);

35
test/unit/lib/accel/accel.c/accel_ut.c
View File

@ -106,7 +106,8 @@ spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct i
}
/* global vars and setup/cleanup functions used for all test functions */
struct spdk_accel_module_if g_module = {};
struct spdk_accel_module_if g_module_if = {};
struct accel_module g_module = { .module = &g_module_if };
struct spdk_io_channel *g_ch = NULL;
struct accel_io_channel *g_accel_ch = NULL;
struct sw_accel_io_channel *g_sw_ch = NULL;
@ -147,16 +148,16 @@ test_setup(void)
return -1;
}
g_module.submit_tasks = sw_accel_submit_tasks;
g_module.name = "software";
g_module_if.submit_tasks = sw_accel_submit_tasks;
g_module_if.name = "software";
for (i = 0; i < ACCEL_OPC_LAST; i++) {
g_accel_ch->module_ch[i] = g_module_ch;
g_modules_opc[i] = &g_module;
g_modules_opc[i] = g_module;
}
g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof(
struct spdk_io_channel));
TAILQ_INIT(&g_sw_ch->tasks_to_complete);
g_module.supports_opcode = _supports_opcode;
g_module_if.supports_opcode = _supports_opcode;
return 0;
}
@ -1004,17 +1005,17 @@ test_sequence_completion_error(void)
struct ut_sequence ut_seq;
struct iovec src_iovs, dst_iovs;
char buf[4096], tmp[4096];
struct spdk_accel_module_if *modules[ACCEL_OPC_LAST];
struct accel_module modules[ACCEL_OPC_LAST];
int i, rc, completed;
ioch = spdk_accel_get_io_channel();
SPDK_CU_ASSERT_FATAL(ioch != NULL);
/* Override the submit_tasks function */
g_module.submit_tasks = ut_sequnce_submit_tasks;
g_module_if.submit_tasks = ut_sequnce_submit_tasks;
for (i = 0; i < ACCEL_OPC_LAST; ++i) {
modules[i] = g_modules_opc[i];
g_modules_opc[i] = &g_module;
g_modules_opc[i] = g_module;
}
memset(buf, 0, sizeof(buf));
@ -1471,21 +1472,21 @@ test_sequence_copy_elision(void)
struct ut_sequence ut_seq;
struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2];
char buf[4096], tmp[4][4096];
struct spdk_accel_module_if *modules[ACCEL_OPC_LAST];
struct accel_module modules[ACCEL_OPC_LAST];
int i, rc, completed;
ioch = spdk_accel_get_io_channel();
SPDK_CU_ASSERT_FATAL(ioch != NULL);
/* Override the submit_tasks function */
g_module.submit_tasks = ut_sequnce_submit_tasks;
g_module_if.submit_tasks = ut_sequnce_submit_tasks;
for (i = 0; i < ACCEL_OPC_LAST; ++i) {
g_seq_operations[i].complete_status = 0;
g_seq_operations[i].submit_status = 0;
g_seq_operations[i].count = 0;
modules[i] = g_modules_opc[i];
g_modules_opc[i] = &g_module;
g_modules_opc[i] = g_module;
}
/* Check that a copy operation at the beginning is removed */
@ -1790,7 +1791,7 @@ test_sequence_accel_buffers(void)
struct ut_sequence ut_seq;
struct iovec src_iovs[3], dst_iovs[3];
char srcbuf[4096], dstbuf[4096], expected[4096];
struct spdk_accel_module_if *modules[ACCEL_OPC_LAST];
struct accel_module modules[ACCEL_OPC_LAST];
void *buf[2], *domain_ctx[2], *iobuf_buf;
struct spdk_memory_domain *domain[2];
struct spdk_iobuf_buffer *cache_entry;
@ -1802,10 +1803,10 @@ test_sequence_accel_buffers(void)
SPDK_CU_ASSERT_FATAL(ioch != NULL);
/* Override the submit_tasks function */
g_module.submit_tasks = ut_sequnce_submit_tasks;
g_module_if.submit_tasks = ut_sequnce_submit_tasks;
for (i = 0; i < ACCEL_OPC_LAST; ++i) {
modules[i] = g_modules_opc[i];
g_modules_opc[i] = &g_module;
g_modules_opc[i] = g_module;
}
/* Intercept decompress to make it simply copy the data, so that we can chain multiple
* decompress operations together in one sequence.
@ -2265,7 +2266,7 @@ test_sequence_memory_domain(void)
struct ut_sequence ut_seq;
struct ut_domain_ctx domctx[4];
struct spdk_iobuf_buffer *cache_entry;
struct spdk_accel_module_if *modules[ACCEL_OPC_LAST];
struct accel_module modules[ACCEL_OPC_LAST];
struct spdk_memory_domain *accel_domain;
spdk_iobuf_buffer_stailq_t small_cache;
char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096];
@ -2278,10 +2279,10 @@ test_sequence_memory_domain(void)
SPDK_CU_ASSERT_FATAL(ioch != NULL);
/* Override the submit_tasks function */
g_module.submit_tasks = ut_sequnce_submit_tasks;
g_module_if.submit_tasks = ut_sequnce_submit_tasks;
for (i = 0; i < ACCEL_OPC_LAST; ++i) {
modules[i] = g_modules_opc[i];
g_modules_opc[i] = &g_module;
g_modules_opc[i] = g_module;
}
/* Intercept decompress to make it simply copy the data, so that we can chain multiple
* decompress operations together in one sequence.