ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/accel: remove the need for the app to allocate an accel_task

Browse Source 
This was sort of a clunky interface requiring a couple of inline
functions in every app that wants to use the accel_fw moving
forward. By having the accel_fw public API accept a callback arg
instead of an accel_task combined with adding a pool of accel_tasks
in the accel_fw engine we can eliminate this.

After changing the parm to a cb_arg, changes were made to all accel_fw
interfaces to put cb_fn and cb_arg as the last parms in public and
private function calls.

Related bdev_malloc changes need to be in this patch in order to pass CI.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I2b75764e534562d91484a094c3352266156d8425
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3209
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
paul luse 2020-07-03 10:08:47 -04:00 committed by Jim Harris
parent 6e1528503a
commit ee7e31f9ae
8 changed files with 403 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -4,6 +4,10 @@
### accel_fw ### accel_fw
The accel_fw was updated to no longer require the app to allocate an
accel_task on its behalf. All public APIs now take a callback arg as
the parameter that used to be the accel_task.
The accel_fw was updated to support compare, dualcast, crc32c. The accel_fw was updated to support compare, dualcast, crc32c.
The accel_fw introduced batching support for all commands in all plug-ins. The accel_fw introduced batching support for all commands in all plug-ins.

60
examples/accel/perf/accel_perf.c
View File

@ -210,20 +210,18 @@ _submit_single(void *arg1, void *arg2)
task->worker->current_queue_depth++; task->worker->current_queue_depth++;
switch (g_workload_selection) { switch (g_workload_selection) {
case ACCEL_COPY: case ACCEL_COPY:
rc = spdk_accel_submit_copy(__accel_task_from_ap_task(task), rc = spdk_accel_submit_copy(worker->ch, task->dst,
worker->ch, task->dst, task->src, g_xfer_size_bytes, accel_done,
task->src, g_xfer_size_bytes, accel_done); __accel_task_from_ap_task(task));
break; break;
case ACCEL_FILL: case ACCEL_FILL:
/* For fill use the first byte of the task->dst buffer */ /* For fill use the first byte of the task->dst buffer */
rc = spdk_accel_submit_fill(__accel_task_from_ap_task(task), rc = spdk_accel_submit_fill(worker->ch, task->dst, *(uint8_t *)task->src,
worker->ch, task->dst, *(uint8_t *)task->src, g_xfer_size_bytes, accel_done, __accel_task_from_ap_task(task));
g_xfer_size_bytes, accel_done);
break; break;
case ACCEL_CRC32C: case ACCEL_CRC32C:
rc = spdk_accel_submit_crc32c(__accel_task_from_ap_task(task), rc = spdk_accel_submit_crc32c(worker->ch, (uint32_t *)task->dst, task->src, g_crc32c_seed,
worker->ch, (uint32_t *)task->dst, task->src, g_crc32c_seed, g_xfer_size_bytes, accel_done, __accel_task_from_ap_task(task));
g_xfer_size_bytes, accel_done);
break; break;
case ACCEL_COMPARE: case ACCEL_COMPARE:
random_num = rand() % 100; random_num = rand() % 100;
@ -234,14 +232,13 @@ _submit_single(void *arg1, void *arg2)
task->expected_status = 0; task->expected_status = 0;
*(uint8_t *)task->dst = DATA_PATTERN; *(uint8_t *)task->dst = DATA_PATTERN;
} }
rc = spdk_accel_submit_compare(__accel_task_from_ap_task(task), rc = spdk_accel_submit_compare(worker->ch, task->dst, task->src,
worker->ch, task->dst, task->src, g_xfer_size_bytes, accel_done, __accel_task_from_ap_task(task));
g_xfer_size_bytes, accel_done);
break; break;
case ACCEL_DUALCAST: case ACCEL_DUALCAST:
rc = spdk_accel_submit_dualcast(__accel_task_from_ap_task(task), rc = spdk_accel_submit_dualcast(worker->ch, task->dst, task->dst2,
worker->ch, task->dst, task->dst2, task->src, g_xfer_size_bytes, accel_done,
task->src, g_xfer_size_bytes, accel_done); __accel_task_from_ap_task(task));
break; break;
default: default:
assert(false); assert(false);
@ -470,29 +467,29 @@ _batch_prep_cmd(struct worker_thread *worker, struct ap_task *task, struct spdk_
switch (g_workload_selection) { switch (g_workload_selection) {
case ACCEL_COPY: case ACCEL_COPY:
rc = spdk_accel_batch_prep_copy(__accel_task_from_ap_task(task), rc = spdk_accel_batch_prep_copy(worker->ch, batch, task->dst,
worker->ch, batch, task->dst, task->src, g_xfer_size_bytes, accel_done,
task->src, g_xfer_size_bytes, accel_done); __accel_task_from_ap_task(task));
break; break;
case ACCEL_DUALCAST: case ACCEL_DUALCAST:
rc = spdk_accel_batch_prep_dualcast(__accel_task_from_ap_task(task), rc = spdk_accel_batch_prep_dualcast(worker->ch, batch, task->dst, task->dst2,
worker->ch, batch, task->dst, task->dst2, task->src, g_xfer_size_bytes, accel_done,
task->src, g_xfer_size_bytes, accel_done); __accel_task_from_ap_task(task));
break; break;
case ACCEL_COMPARE: case ACCEL_COMPARE:
rc = spdk_accel_batch_prep_compare(__accel_task_from_ap_task(task), rc = spdk_accel_batch_prep_compare(worker->ch, batch, task->dst, task->src,
worker->ch, batch, task->dst, task->src, g_xfer_size_bytes, accel_done,
g_xfer_size_bytes, accel_done); __accel_task_from_ap_task(task));
break; break;
case ACCEL_FILL: case ACCEL_FILL:
rc = spdk_accel_batch_prep_fill(__accel_task_from_ap_task(task), rc = spdk_accel_batch_prep_fill(worker->ch, batch, task->dst, *(uint8_t *)task->src,
worker->ch, batch, task->dst, *(uint8_t *)task->src, g_xfer_size_bytes, accel_done,
g_xfer_size_bytes, accel_done); __accel_task_from_ap_task(task));
break; break;
case ACCEL_CRC32C: case ACCEL_CRC32C:
rc = spdk_accel_batch_prep_crc32c(__accel_task_from_ap_task(task), rc = spdk_accel_batch_prep_crc32c(worker->ch, batch, (uint32_t *)task->dst, task->src,
worker->ch, batch, (uint32_t *)task->dst, task->src, g_crc32c_seed, g_xfer_size_bytes, accel_done,
g_crc32c_seed, g_xfer_size_bytes, accel_done); __accel_task_from_ap_task(task));
break; break;
default: default:
assert(false); assert(false);
@ -594,8 +591,7 @@ _init_thread(void *arg1)
task->worker = worker; task->worker = worker;
task->worker->current_queue_depth++; task->worker->current_queue_depth++;
rc = spdk_accel_batch_submit(__accel_task_from_ap_task(task), rc = spdk_accel_batch_submit(worker->ch, batch, batch_done, __accel_task_from_ap_task(task));
worker->ch, batch, batch_done);
if (rc) { if (rc) {
fprintf(stderr, "error ending batch %d\n", rc); fprintf(stderr, "error ending batch %d\n", rc);
goto error; goto error;

118
include/spdk/accel_engine.h
View File

@ -123,20 +123,20 @@ uint64_t spdk_accel_get_capabilities(struct spdk_io_channel *ch);
/** /**
* Submit a copy request. * Submit a copy request.
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param dst Destination to copy to. * \param dst Destination to copy to.
* \param src Source to copy from. * \param src Source to copy from.
* \param nbytes Length in bytes to copy. * \param nbytes Length in bytes to copy.
* \param cb Called when this copy operation completes. * \param cb_fn Called when this copy operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_submit_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, void *dst, int spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
void *src, uint64_t nbytes, spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
/** /**
* Synchronous call to get batch size. This is the maximum number of * Synchronous call to get batch size. This is the maximum number of
* descriptors that a batch can contain. Once this limit is reached the batch * descriptors that a batch can contain. Once this limit is reached the batch
* should be processed with spdk_accel_batch_submit(). * should be processed with spdk_accel_batch_submit().
* *
@ -158,15 +158,15 @@ struct spdk_accel_batch *spdk_accel_batch_create(struct spdk_io_channel *ch);
/** /**
* Asynchronous call to submit a batch sequence. * Asynchronous call to submit a batch sequence.
* *
* \param accel_req Accel request task.
* \param ch I/O channel associated with this call. * \param ch I/O channel associated with this call.
* \param batch Handle provided when the batch was started with spdk_accel_batch_create(). * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
* \param cb Called when this operation completes. * \param cb_fn Called when this operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_batch_submit(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
/** /**
* Synchronous call to prepare a copy request into a previously initialized batch * Synchronous call to prepare a copy request into a previously initialized batch
@ -174,19 +174,19 @@ int spdk_accel_batch_submit(struct spdk_accel_task *accel_req, struct spdk_io_ch
* completes after the batch has been submitted by an asynchronous call to * completes after the batch has been submitted by an asynchronous call to
* spdk_accel_batch_submit(). * spdk_accel_batch_submit().
* *
* \param accel_req Accel request task.
* \param ch I/O channel associated with this call. * \param ch I/O channel associated with this call.
* \param batch Handle provided when the batch was started with spdk_accel_batch_create(). * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
* \param dst Destination to copy to. * \param dst Destination to copy to.
* \param src Source to copy from. * \param src Source to copy from.
* \param nbytes Length in bytes to copy. * \param nbytes Length in bytes to copy.
* \param cb Called when this operation completes. * \param cb_fn Called when this operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_batch_prep_copy(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, void *dst, void *src, void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
uint64_t nbytes, spdk_accel_completion_cb cb); void *cb_arg);
/** /**
* Synchronous call to prepare a dualcast request into a previously initialized batch * Synchronous call to prepare a dualcast request into a previously initialized batch
@ -194,37 +194,36 @@ int spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io
* completes after the batch has been submitted by an asynchronous call to * completes after the batch has been submitted by an asynchronous call to
* spdk_accel_batch_submit(). * spdk_accel_batch_submit().
* *
* \param accel_req Accel request task.
* \param ch I/O channel associated with this call. * \param ch I/O channel associated with this call.
* \param batch Handle provided when the batch was started with spdk_accel_batch_create(). * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
* \param dst1 First destination to copy to (must be 4K aligned). * \param dst1 First destination to copy to (must be 4K aligned).
* \param dst2 Second destination to copy to (must be 4K aligned). * \param dst2 Second destination to copy to (must be 4K aligned).
* \param src Source to copy from. * \param src Source to copy from.
* \param nbytes Length in bytes to copy. * \param nbytes Length in bytes to copy.
* \param cb Called when this operation completes. * \param cb_fn Called when this operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_batch_prep_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_batch_prep_dualcast(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, void *dst1, void *dst2, void *src, void *dst1, void *dst2, void *src, uint64_t nbytes,
uint64_t nbytes, spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
/** /**
* Submit a dual cast copy request. * Submit a dual cast copy request.
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param dst1 First destination to copy to (must be 4K aligned). * \param dst1 First destination to copy to (must be 4K aligned).
* \param dst2 Second destination to copy to (must be 4K aligned). * \param dst2 Second destination to copy to (must be 4K aligned).
* \param src Source to copy from. * \param src Source to copy from.
* \param nbytes Length in bytes to copy. * \param nbytes Length in bytes to copy.
* \param cb Called when this copy operation completes. * \param cb_fn Called when this copy operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_submit_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
void *dst1, void *dst2, void *src, uint64_t nbytes, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
spdk_accel_completion_cb cb);
/** /**
* Synchronous call to prepare a compare request into a previously initialized batch * Synchronous call to prepare a compare request into a previously initialized batch
@ -232,35 +231,34 @@ int spdk_accel_submit_dualcast(struct spdk_accel_task *accel_req, struct spdk_io
* completes after the batch has been submitted by an asynchronous call to * completes after the batch has been submitted by an asynchronous call to
* spdk_accel_batch_submit(). * spdk_accel_batch_submit().
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param batch Handle provided when the batch was started with spdk_accel_batch_create(). * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
* \param src1 First location to perform compare on. * \param src1 First location to perform compare on.
* \param src2 Second location to perform compare on. * \param src2 Second location to perform compare on.
* \param nbytes Length in bytes to compare. * \param nbytes Length in bytes to compare.
* \param cb Called when this operation completes. * \param cb_fn Called when this operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_batch_prep_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_batch_prep_compare(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, void *src1, void *src2, void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
uint64_t nbytes, spdk_accel_completion_cb cb); void *cb_arg);
/** /**
* Submit a compare request. * Submit a compare request.
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param src1 First location to perform compare on. * \param src1 First location to perform compare on.
* \param src2 Second location to perform compare on. * \param src2 Second location to perform compare on.
* \param nbytes Length in bytes to compare. * \param nbytes Length in bytes to compare.
* \param cb Called when this compare operation completes. * \param cb_fn Called when this compare operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, any other value means there was a miscompare. * \return 0 on success, any other value means there was a miscompare.
*/ */
int spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes,
void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
spdk_accel_completion_cb cb);
/** /**
* Synchronous call to prepare a fill request into a previously initialized batch * Synchronous call to prepare a fill request into a previously initialized batch
@ -268,36 +266,36 @@ int spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_
* completes after the batch has been submitted by an asynchronous call to * completes after the batch has been submitted by an asynchronous call to
* spdk_accel_batch_submit(). * spdk_accel_batch_submit().
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param batch Handle provided when the batch was started with spdk_accel_batch_create(). * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
* \param dst Destination to fill. * \param dst Destination to fill.
* \param fill Constant byte to fill to the destination. * \param fill Constant byte to fill to the destination.
* \param nbytes Length in bytes to fill. * \param nbytes Length in bytes to fill.
* \param cb Called when this operation completes. * \param cb_fn Called when this operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_batch_prep_fill(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_batch_prep_fill(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, void *dst, uint8_t fill, void *dst, uint8_t fill, uint64_t nbytes,
uint64_t nbytes, spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
/** /**
* Submit a fill request. * Submit a fill request.
* *
* This operation will fill the destination buffer with the specified value. * This operation will fill the destination buffer with the specified value.
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param dst Destination to fill. * \param dst Destination to fill.
* \param fill Constant byte to fill to the destination. * \param fill Constant byte to fill to the destination.
* \param nbytes Length in bytes to fill. * \param nbytes Length in bytes to fill.
* \param cb Called when this fill operation completes. * \param cb_fn Called when this fill operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_submit_fill(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes,
void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
/** /**
* Synchronous call to prepare a crc32c request into a previously initialized batch * Synchronous call to prepare a crc32c request into a previously initialized batch
@ -305,38 +303,38 @@ int spdk_accel_submit_fill(struct spdk_accel_task *accel_req, struct spdk_io_cha
* completes after the batch has been submitted by an asynchronous call to * completes after the batch has been submitted by an asynchronous call to
* spdk_accel_batch_submit(). * spdk_accel_batch_submit().
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param batch Handle provided when the batch was started with spdk_accel_batch_create(). * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
* \param dst Destination to write the CRC-32C to. * \param dst Destination to write the CRC-32C to.
* \param src The source address for the data. * \param src The source address for the data.
* \param seed Four byte seed value. * \param seed Four byte seed value.
* \param nbytes Length in bytes. * \param nbytes Length in bytes.
* \param cb Called when this operation completes. * \param cb_fn Called when this operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_batch_prep_crc32c(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, uint32_t *dst, void *src, uint32_t seed, uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
uint64_t nbytes, spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
/** /**
* Submit a CRC-32C calculation request. * Submit a CRC-32C calculation request.
* *
* This operation will calculate the 4 byte CRC32-C for the given data. * This operation will calculate the 4 byte CRC32-C for the given data.
* *
* \param accel_req Accel request task. * \param ch I/O channel associated with this call.
* \param ch I/O channel to submit request to the accel engine.
* \param dst Destination to write the CRC-32C to. * \param dst Destination to write the CRC-32C to.
* \param src The source address for the data. * \param src The source address for the data.
* \param seed Four byte seed value. * \param seed Four byte seed value.
* \param nbytes Length in bytes. * \param nbytes Length in bytes.
* \param cb Called when this CRC-32C operation completes. * \param cb_fn Called when this CRC-32C operation completes.
* \param cb_arg Callback argument.
* *
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_accel_submit_crc32c(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, int spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed,
uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
/** /**
* Get the size of an acceleration task. * Get the size of an acceleration task.

46
include/spdk_internal/accel_engine.h
View File

@ -41,36 +41,38 @@
struct spdk_accel_task { struct spdk_accel_task {
spdk_accel_completion_cb cb; spdk_accel_completion_cb cb;
void *cb_arg;
uint8_t offload_ctx[0]; uint8_t offload_ctx[0];
}; };
struct spdk_accel_engine { struct spdk_accel_engine {
uint64_t (*get_capabilities)(void); uint64_t (*get_capabilities)(void);
int (*copy)(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, int (*copy)(struct spdk_io_channel *ch, void *dst, void *src,
uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
int (*dualcast)(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src, int (*dualcast)(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
uint32_t (*batch_get_max)(void); uint32_t (*batch_get_max)(void);
struct spdk_accel_batch *(*batch_create)(struct spdk_io_channel *ch); struct spdk_accel_batch *(*batch_create)(struct spdk_io_channel *ch);
int (*batch_prep_copy)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, int (*batch_prep_copy)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb); void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
int (*batch_prep_dualcast)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, int (*batch_prep_dualcast)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
void *dst1, void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb); void *dst1, void *dst2, void *src, uint64_t nbytes,
int (*batch_prep_compare)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, spdk_accel_completion_cb cb_fn, void *cb_arg);
void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb); int (*batch_prep_compare)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
int (*batch_prep_fill)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb); int (*batch_prep_fill)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
int (*batch_prep_crc32c)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
int (*batch_prep_crc32c)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes, uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
int (*batch_submit)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, int (*batch_submit)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
spdk_accel_completion_cb cb); spdk_accel_completion_cb cb_fn, void *cb_arg);
int (*compare)(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2, int (*compare)(struct spdk_io_channel *ch, void *src1, void *src2,
uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
int (*fill)(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, int (*fill)(struct spdk_io_channel *ch, void *dst, uint8_t fill,
uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
int (*crc32c)(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void *src, int (*crc32c)(struct spdk_io_channel *ch, uint32_t *dst, void *src,
uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb); uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
struct spdk_io_channel *(*get_io_channel)(void); struct spdk_io_channel *(*get_io_channel)(void);
}; };

341
lib/accel/accel_engine.c
View File

@ -48,7 +48,10 @@
* later in this file. * later in this file.
*/ */
#define ALIGN_4K 0x1000 #define ALIGN_4K 0x1000
#define SPDK_ACCEL_NUM_TASKS 0x4000
static struct spdk_mempool *g_accel_task_pool;
/* Largest context size for all accel modules */ /* Largest context size for all accel modules */
static size_t g_max_accel_module_size = 0; static size_t g_max_accel_module_size = 0;
@ -71,16 +74,17 @@ struct accel_io_channel {
/* Forward declarations of software implementations used when an /* Forward declarations of software implementations used when an
* engine has not implemented the capability. * engine has not implemented the capability.
*/ */
static int sw_accel_submit_dualcast(void *cb_arg, struct spdk_io_channel *ch, void *dst1, static int sw_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
static int sw_accel_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, static int sw_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
static int sw_accel_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2, static int sw_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2,
uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
static int sw_accel_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, static int sw_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill,
uint64_t nbytes, spdk_accel_completion_cb cb); uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
static int sw_accel_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, static int sw_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src,
void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb); uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
void *cb_arg);
/* Registration of hw modules (currently supports only 1 at a time) */ /* Registration of hw modules (currently supports only 1 at a time) */
void void
@ -113,7 +117,8 @@ _accel_engine_done(void *ref, int status)
{ {
struct spdk_accel_task *req = (struct spdk_accel_task *)ref; struct spdk_accel_task *req = (struct spdk_accel_task *)ref;
req->cb(req, status); req->cb(req->cb_arg, status);
spdk_mempool_put(g_accel_task_pool, req);
} }
uint64_t uint64_t
@ -127,45 +132,58 @@ spdk_accel_get_capabilities(struct spdk_io_channel *ch)
/* Accel framework public API for copy function */ /* Accel framework public API for copy function */
int int
spdk_accel_submit_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
SPDK_ERRLOG("Unable to get an accel task.\n");
return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
/* If the engine does not support it, fallback to the sw implementation. */ /* If the engine does not support it, fallback to the sw implementation. */
if (accel_ch->engine->copy) { if (accel_ch->engine->copy) {
return accel_ch->engine->copy(accel_req->offload_ctx, accel_ch->ch, dst, src, nbytes, return accel_ch->engine->copy(accel_ch->ch, dst, src, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} else { } else {
return sw_accel_submit_copy(accel_req->offload_ctx, accel_ch->ch, dst, src, nbytes, return sw_accel_submit_copy(accel_ch->ch, dst, src, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} }
} }
/* Accel framework public API for dual cast copy function */ /* Accel framework public API for dual cast copy function */
int int
spdk_accel_submit_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
void *dst1, void *dst2, void *src, uint64_t nbytes, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_completion_cb cb)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
if (accel_req == NULL) {
SPDK_ERRLOG("Unable to get an accel task.\n");
return -ENOMEM;
}
if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
return -EINVAL; return -EINVAL;
} }
accel_req->cb = cb; accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
/* If the engine does not support it, fallback to the sw implementation. */ /* If the engine does not support it, fallback to the sw implementation. */
if (accel_ch->engine->dualcast) { if (accel_ch->engine->dualcast) {
return accel_ch->engine->dualcast(accel_req->offload_ctx, accel_ch->ch, dst1, dst2, src, nbytes, return accel_ch->engine->dualcast(accel_ch->ch, dst1, dst2, src, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} else { } else {
return sw_accel_submit_dualcast(accel_req->offload_ctx, accel_ch->ch, dst1, dst2, src, nbytes, return sw_accel_submit_dualcast(accel_ch->ch, dst1, dst2, src, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} }
} }
@ -184,14 +202,22 @@ spdk_accel_batch_create(struct spdk_io_channel *ch)
* required to implement this API. * required to implement this API.
*/ */
int int
spdk_accel_batch_submit(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
return accel_ch->engine->batch_submit(accel_req->offload_ctx, accel_ch->ch, batch, SPDK_ERRLOG("Unable to get an accel task.\n");
_accel_engine_done); return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
return accel_ch->engine->batch_submit(accel_ch->ch, batch, _accel_engine_done,
accel_req->offload_ctx);
} }
/* Accel framework public API for getting max batch. All engines are /* Accel framework public API for getting max batch. All engines are
@ -209,136 +235,195 @@ spdk_accel_batch_get_max(struct spdk_io_channel *ch)
* required to implement this API. * required to implement this API.
*/ */
int int
spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_batch_prep_copy(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *dst,
struct spdk_accel_batch *batch, void *dst, void *src, uint64_t nbytes, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_completion_cb cb)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
return accel_ch->engine->batch_prep_copy(accel_req->offload_ctx, accel_ch->ch, batch, dst, src, SPDK_ERRLOG("Unable to get an accel task.\n");
nbytes, _accel_engine_done); return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
return accel_ch->engine->batch_prep_copy(accel_ch->ch, batch, dst, src, nbytes,
_accel_engine_done, accel_req->offload_ctx);
} }
/* Accel framework public API for batch prep_dualcast function. All engines are /* Accel framework public API for batch prep_dualcast function. All engines are
* required to implement this API. * required to implement this API.
*/ */
int int
spdk_accel_batch_prep_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_batch_prep_dualcast(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, void *dst1, void *dst2, void *src, uint64_t nbytes, void *dst1, void *dst2, void *src, uint64_t nbytes,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
if (accel_req == NULL) {
SPDK_ERRLOG("Unable to get an accel task.\n");
return -ENOMEM;
}
if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
return -EINVAL; return -EINVAL;
} }
accel_req->cb = cb; accel_req->cb = cb_fn;
return accel_ch->engine->batch_prep_dualcast(accel_req->offload_ctx, accel_ch->ch, accel_req->cb_arg = cb_arg;
batch, dst1, dst2, src, nbytes, _accel_engine_done);
return accel_ch->engine->batch_prep_dualcast(accel_ch->ch, batch, dst1, dst2, src,
nbytes, _accel_engine_done, accel_req->offload_ctx);
} }
/* Accel framework public API for batch prep_compare function. All engines are /* Accel framework public API for batch prep_compare function. All engines are
* required to implement this API. * required to implement this API.
*/ */
int int
spdk_accel_batch_prep_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_batch_prep_compare(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, void *src1, void *src2, uint64_t nbytes, void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
spdk_accel_completion_cb cb) void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
return accel_ch->engine->batch_prep_compare(accel_req->offload_ctx, accel_ch->ch, SPDK_ERRLOG("Unable to get an accel task.\n");
batch, src1, src2, nbytes, _accel_engine_done); return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
return accel_ch->engine->batch_prep_compare(accel_ch->ch, batch, src1, src2, nbytes,
_accel_engine_done, accel_req->offload_ctx);
} }
/* Accel framework public API for batch prep_fill function. All engines are /* Accel framework public API for batch prep_fill function. All engines are
* required to implement this API. * required to implement this API.
*/ */
int int
spdk_accel_batch_prep_fill(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_batch_prep_fill(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *dst,
struct spdk_accel_batch *batch, void *dst, uint8_t fill, uint64_t nbytes, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
spdk_accel_completion_cb cb)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
return accel_ch->engine->batch_prep_fill(accel_req->offload_ctx, accel_ch->ch, SPDK_ERRLOG("Unable to get an accel task.\n");
batch, dst, fill, nbytes, _accel_engine_done); return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
return accel_ch->engine->batch_prep_fill(accel_ch->ch, batch, dst, fill, nbytes,
_accel_engine_done, accel_req->offload_ctx);
} }
/* Accel framework public API for batch prep_crc32c function. All engines are /* Accel framework public API for batch prep_crc32c function. All engines are
* required to implement this API. * required to implement this API.
*/ */
int int
spdk_accel_batch_prep_crc32c(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, uint32_t *dst, void *src, uint32_t seed, uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
uint64_t nbytes, spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
return accel_ch->engine->batch_prep_crc32c(accel_req->offload_ctx, accel_ch->ch, SPDK_ERRLOG("Unable to get an accel task.\n");
batch, dst, src, seed, nbytes, _accel_engine_done); return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
return accel_ch->engine->batch_prep_crc32c(accel_ch->ch, batch, dst, src, seed, nbytes,
_accel_engine_done, accel_req->offload_ctx);
} }
/* Accel framework public API for compare function */ /* Accel framework public API for compare function */
int int
spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes,
void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
SPDK_ERRLOG("Unable to get an accel task.\n");
return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
/* If the engine does not support it, fallback to the sw implementation. */ /* If the engine does not support it, fallback to the sw implementation. */
if (accel_ch->engine->compare) { if (accel_ch->engine->compare) {
return accel_ch->engine->compare(accel_req->offload_ctx, accel_ch->ch, src1, src2, nbytes, return accel_ch->engine->compare(accel_ch->ch, src1, src2, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} else { } else {
return sw_accel_submit_compare(accel_req->offload_ctx, accel_ch->ch, src1, src2, nbytes, return sw_accel_submit_compare(accel_ch->ch, src1, src2, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} }
} }
/* Accel framework public API for fill function */ /* Accel framework public API for fill function */
int int
spdk_accel_submit_fill(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes,
void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
SPDK_ERRLOG("Unable to get an accel task.\n");
return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
/* If the engine does not support it, fallback to the sw implementation. */ /* If the engine does not support it, fallback to the sw implementation. */
if (accel_ch->engine->fill) { if (accel_ch->engine->fill) {
return accel_ch->engine->fill(accel_req->offload_ctx, accel_ch->ch, dst, fill, nbytes, return accel_ch->engine->fill(accel_ch->ch, dst, fill, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} else { } else {
return sw_accel_submit_fill(accel_req->offload_ctx, accel_ch->ch, dst, fill, nbytes, return sw_accel_submit_fill(accel_ch->ch, dst, fill, nbytes,
_accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} }
} }
/* Accel framework public API for CRC-32C function */ /* Accel framework public API for CRC-32C function */
int int
spdk_accel_submit_crc32c(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed,
uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);
struct spdk_accel_task *accel_req = spdk_mempool_get(g_accel_task_pool);
accel_req->cb = cb; if (accel_req == NULL) {
SPDK_ERRLOG("Unable to get an accel task.\n");
return -ENOMEM;
}
accel_req->cb = cb_fn;
accel_req->cb_arg = cb_arg;
/* If the engine does not support it, fallback to the sw implementation. */ /* If the engine does not support it, fallback to the sw implementation. */
if (accel_ch->engine->crc32c) { if (accel_ch->engine->crc32c) {
return accel_ch->engine->crc32c(accel_req->offload_ctx, accel_ch->ch, dst, src, return accel_ch->engine->crc32c(accel_ch->ch, dst, src, seed, nbytes,
seed, nbytes, _accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} else { } else {
return sw_accel_submit_crc32c(accel_req->offload_ctx, accel_ch->ch, dst, src, return sw_accel_submit_crc32c(accel_ch->ch, dst, src, seed, nbytes,
seed, nbytes, _accel_engine_done); _accel_engine_done, accel_req->offload_ctx);
} }
} }
@ -398,10 +483,20 @@ static void
accel_engine_module_initialize(void) accel_engine_module_initialize(void)
{ {
struct spdk_accel_module_if *accel_engine_module; struct spdk_accel_module_if *accel_engine_module;
char task_pool_name[30];
TAILQ_FOREACH(accel_engine_module, &spdk_accel_module_list, tailq) { TAILQ_FOREACH(accel_engine_module, &spdk_accel_module_list, tailq) {
accel_engine_module->module_init(); accel_engine_module->module_init();
} }
snprintf(task_pool_name, sizeof(task_pool_name), "accel_task_pool");
g_accel_task_pool = spdk_mempool_create(task_pool_name,
SPDK_ACCEL_NUM_TASKS,
g_max_accel_module_size,
SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
SPDK_ENV_SOCKET_ID_ANY);
assert(g_accel_task_pool);
} }
int int
@ -479,6 +574,7 @@ spdk_accel_engine_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
spdk_io_device_unregister(&spdk_accel_module_list, NULL); spdk_io_device_unregister(&spdk_accel_module_list, NULL);
spdk_accel_engine_module_finish(); spdk_accel_engine_module_finish();
spdk_mempool_free(g_accel_task_pool);
} }
void void
@ -562,8 +658,8 @@ sw_accel_batch_start(struct spdk_io_channel *ch)
} }
static struct sw_accel_op * static struct sw_accel_op *
_prep_op(void *cb_arg, struct sw_accel_io_channel *sw_ch, struct spdk_accel_batch *batch, _prep_op(struct sw_accel_io_channel *sw_ch, struct spdk_accel_batch *batch,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct sw_accel_op *op; struct sw_accel_op *op;
@ -581,20 +677,20 @@ _prep_op(void *cb_arg, struct sw_accel_io_channel *sw_ch, struct spdk_accel_batc
} }
op->cb_arg = cb_arg; op->cb_arg = cb_arg;
op->cb_fn = cb; op->cb_fn = cb_fn;
op->sw_ch = sw_ch; op->sw_ch = sw_ch;
return op; return op;
} }
static int static int
sw_accel_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, sw_accel_batch_prep_copy(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb) void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct sw_accel_op *op; struct sw_accel_op *op;
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
op = _prep_op(cb_arg, sw_ch, batch, cb); op = _prep_op(sw_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -610,14 +706,14 @@ sw_accel_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_a
} }
static int static int
sw_accel_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch, sw_accel_batch_prep_dualcast(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *dst1,
struct spdk_accel_batch *batch, void *dst1, void *dst2, void *dst2,
void *src, uint64_t nbytes, spdk_accel_completion_cb cb) void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct sw_accel_op *op; struct sw_accel_op *op;
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
op = _prep_op(cb_arg, sw_ch, batch, cb); op = _prep_op(sw_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -634,14 +730,13 @@ sw_accel_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
sw_accel_batch_prep_compare(void *cb_arg, struct spdk_io_channel *ch, sw_accel_batch_prep_compare(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *src1,
struct spdk_accel_batch *batch, void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
void *src2, uint64_t nbytes, spdk_accel_completion_cb cb)
{ {
struct sw_accel_op *op; struct sw_accel_op *op;
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
op = _prep_op(cb_arg, sw_ch, batch, cb); op = _prep_op(sw_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -657,14 +752,14 @@ sw_accel_batch_prep_compare(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
sw_accel_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch, sw_accel_batch_prep_fill(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *dst,
struct spdk_accel_batch *batch, void *dst, uint8_t fill, uint8_t fill,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct sw_accel_op *op; struct sw_accel_op *op;
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
op = _prep_op(cb_arg, sw_ch, batch, cb); op = _prep_op(sw_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -680,14 +775,14 @@ sw_accel_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
sw_accel_batch_prep_crc32c(void *cb_arg, struct spdk_io_channel *ch, sw_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
struct spdk_accel_batch *batch, uint32_t *dst, void *src, uint32_t *dst,
uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb) void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct sw_accel_op *op; struct sw_accel_op *op;
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
op = _prep_op(cb_arg, sw_ch, batch, cb); op = _prep_op(sw_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -704,8 +799,8 @@ sw_accel_batch_prep_crc32c(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
sw_accel_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, sw_accel_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct sw_accel_op *op; struct sw_accel_op *op;
struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);
@ -753,14 +848,14 @@ sw_accel_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_acce
/* Now complete the batch request itself. */ /* Now complete the batch request itself. */
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
offsetof(struct spdk_accel_task, offload_ctx)); offsetof(struct spdk_accel_task, offload_ctx));
cb(accel_req, batch_status); cb_fn(accel_req, batch_status);
return 0; return 0;
} }
static int static int
sw_accel_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, sw_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct spdk_accel_task *accel_req; struct spdk_accel_task *accel_req;
@ -768,13 +863,13 @@ sw_accel_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
offsetof(struct spdk_accel_task, offload_ctx)); offsetof(struct spdk_accel_task, offload_ctx));
cb(accel_req, 0); cb_fn(accel_req, 0);
return 0; return 0;
} }
static int static int
sw_accel_submit_dualcast(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, sw_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2,
void *src, uint64_t nbytes, spdk_accel_completion_cb cb) void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct spdk_accel_task *accel_req; struct spdk_accel_task *accel_req;
@ -783,13 +878,13 @@ sw_accel_submit_dualcast(void *cb_arg, struct spdk_io_channel *ch, void *dst1, v
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
offsetof(struct spdk_accel_task, offload_ctx)); offsetof(struct spdk_accel_task, offload_ctx));
cb(accel_req, 0); cb_fn(accel_req, 0);
return 0; return 0;
} }
static int static int
sw_accel_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2, sw_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct spdk_accel_task *accel_req; struct spdk_accel_task *accel_req;
int result; int result;
@ -798,35 +893,35 @@ sw_accel_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, vo
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
offsetof(struct spdk_accel_task, offload_ctx)); offsetof(struct spdk_accel_task, offload_ctx));
cb(accel_req, result); cb_fn(accel_req, result);
return 0; return 0;
} }
static int static int
sw_accel_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, sw_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct spdk_accel_task *accel_req; struct spdk_accel_task *accel_req;
memset(dst, fill, nbytes); memset(dst, fill, nbytes);
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
offsetof(struct spdk_accel_task, offload_ctx)); offsetof(struct spdk_accel_task, offload_ctx));
cb(accel_req, 0); cb_fn(accel_req, 0);
return 0; return 0;
} }
static int static int
sw_accel_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void *src, sw_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src,
uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb) uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct spdk_accel_task *accel_req; struct spdk_accel_task *accel_req;
*dst = spdk_crc32c_update(src, nbytes, ~seed); *dst = spdk_crc32c_update(src, nbytes, ~seed);
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
offsetof(struct spdk_accel_task, offload_ctx)); offsetof(struct spdk_accel_task, offload_ctx));
cb(accel_req, 0); cb_fn(accel_req, 0);
return 0; return 0;
} }

67
module/accel/idxd/accel_engine_idxd.c
View File

@ -219,14 +219,14 @@ _prep_queue_command(struct idxd_io_channel *chan, spdk_accel_completion_cb cb_fn
} }
static int static int
idxd_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes, idxd_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
int rc = 0; int rc = 0;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
if (chan->state == IDXD_CHANNEL_ACTIVE) { if (chan->state == IDXD_CHANNEL_ACTIVE) {
rc = spdk_idxd_submit_copy(chan->chan, dst, src, nbytes, idxd_done, idxd_task); rc = spdk_idxd_submit_copy(chan->chan, dst, src, nbytes, idxd_done, idxd_task);
@ -259,14 +259,14 @@ idxd_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src,
} }
static int static int
idxd_submit_dualcast(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src, idxd_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
int rc = 0; int rc = 0;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
if (chan->state == IDXD_CHANNEL_ACTIVE) { if (chan->state == IDXD_CHANNEL_ACTIVE) {
rc = spdk_idxd_submit_dualcast(chan->chan, dst1, dst2, src, nbytes, idxd_done, idxd_task); rc = spdk_idxd_submit_dualcast(chan->chan, dst1, dst2, src, nbytes, idxd_done, idxd_task);
@ -300,14 +300,14 @@ idxd_submit_dualcast(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void
} }
static int static int
idxd_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2, idxd_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
int rc = 0; int rc = 0;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
if (chan->state == IDXD_CHANNEL_ACTIVE) { if (chan->state == IDXD_CHANNEL_ACTIVE) {
rc = spdk_idxd_submit_compare(chan->chan, src1, src2, nbytes, idxd_done, idxd_task); rc = spdk_idxd_submit_compare(chan->chan, src1, src2, nbytes, idxd_done, idxd_task);
@ -340,15 +340,15 @@ idxd_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *
} }
static int static int
idxd_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, idxd_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
int rc = 0; int rc = 0;
uint64_t fill_pattern; uint64_t fill_pattern;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
memset(&fill_pattern, fill, sizeof(uint64_t)); memset(&fill_pattern, fill, sizeof(uint64_t));
if (chan->state == IDXD_CHANNEL_ACTIVE) { if (chan->state == IDXD_CHANNEL_ACTIVE) {
@ -382,14 +382,14 @@ idxd_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fi
} }
static int static int
idxd_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void *src, idxd_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src,
uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb) uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
int rc = 0; int rc = 0;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
if (chan->state == IDXD_CHANNEL_ACTIVE) { if (chan->state == IDXD_CHANNEL_ACTIVE) {
rc = spdk_idxd_submit_crc32c(chan->chan, dst, src, seed, nbytes, idxd_done, idxd_task); rc = spdk_idxd_submit_crc32c(chan->chan, dst, src, seed, nbytes, idxd_done, idxd_task);
@ -444,15 +444,15 @@ idxd_batch_start(struct spdk_io_channel *ch)
} }
static int static int
idxd_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, idxd_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
struct idxd_batch *batch = (struct idxd_batch *)_batch; struct idxd_batch *batch = (struct idxd_batch *)_batch;
int rc = 0; int rc = 0;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
if (chan->state == IDXD_CHANNEL_ACTIVE) { if (chan->state == IDXD_CHANNEL_ACTIVE) {
rc = spdk_idxd_batch_submit(chan->chan, batch, idxd_done, idxd_task); rc = spdk_idxd_batch_submit(chan->chan, batch, idxd_done, idxd_task);
@ -483,29 +483,29 @@ idxd_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_ba
} }
static int static int
idxd_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, idxd_batch_prep_copy(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch,
void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb) void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
struct idxd_batch *batch = (struct idxd_batch *)_batch; struct idxd_batch *batch = (struct idxd_batch *)_batch;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
return spdk_idxd_batch_prep_copy(chan->chan, batch, dst, src, nbytes, return spdk_idxd_batch_prep_copy(chan->chan, batch, dst, src, nbytes,
idxd_done, idxd_task); idxd_done, idxd_task);
} }
static int static int
idxd_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, idxd_batch_prep_fill(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch,
void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb) void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
uint64_t fill_pattern; uint64_t fill_pattern;
struct idxd_batch *batch = (struct idxd_batch *)_batch; struct idxd_batch *batch = (struct idxd_batch *)_batch;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
memset(&fill_pattern, fill, sizeof(uint64_t)); memset(&fill_pattern, fill, sizeof(uint64_t));
return spdk_idxd_batch_prep_fill(chan->chan, batch, dst, fill_pattern, nbytes, idxd_done, return spdk_idxd_batch_prep_fill(chan->chan, batch, dst, fill_pattern, nbytes, idxd_done,
@ -513,43 +513,44 @@ idxd_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel
} }
static int static int
idxd_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, idxd_batch_prep_dualcast(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch,
void *dst1, void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb) void *dst1, void *dst2, void *src, uint64_t nbytes,
spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
struct idxd_batch *batch = (struct idxd_batch *)_batch; struct idxd_batch *batch = (struct idxd_batch *)_batch;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
return spdk_idxd_batch_prep_dualcast(chan->chan, batch, dst1, dst2, src, nbytes, idxd_done, return spdk_idxd_batch_prep_dualcast(chan->chan, batch, dst1, dst2, src, nbytes, idxd_done,
idxd_task); idxd_task);
} }
static int static int
idxd_batch_prep_crc32c(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, idxd_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch,
uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes, uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
struct idxd_batch *batch = (struct idxd_batch *)_batch; struct idxd_batch *batch = (struct idxd_batch *)_batch;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
return spdk_idxd_batch_prep_crc32c(chan->chan, batch, dst, src, seed, nbytes, idxd_done, return spdk_idxd_batch_prep_crc32c(chan->chan, batch, dst, src, seed, nbytes, idxd_done,
idxd_task); idxd_task);
} }
static int static int
idxd_batch_prep_compare(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, idxd_batch_prep_compare(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch,
void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb) void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
struct idxd_batch *batch = (struct idxd_batch *)_batch; struct idxd_batch *batch = (struct idxd_batch *)_batch;
idxd_task->cb = cb; idxd_task->cb = cb_fn;
return spdk_idxd_batch_prep_compare(chan->chan, batch, src1, src2, nbytes, idxd_done, return spdk_idxd_batch_prep_compare(chan->chan, batch, src1, src2, nbytes, idxd_done,
idxd_task); idxd_task);

55
module/accel/ioat/accel_engine_ioat.c
View File

@ -182,22 +182,22 @@ ioat_done(void *cb_arg)
} }
static int static int
ioat_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes, ioat_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
assert(ioat_ch->ioat_ch != NULL); assert(ioat_ch->ioat_ch != NULL);
ioat_task->cb = cb; ioat_task->cb = cb_fn;
return spdk_ioat_submit_copy(ioat_ch->ioat_ch, ioat_task, ioat_done, dst, src, nbytes); return spdk_ioat_submit_copy(ioat_ch->ioat_ch, ioat_task, ioat_done, dst, src, nbytes);
} }
static int static int
ioat_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, ioat_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill,
uint64_t nbytes, spdk_accel_completion_cb cb) uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
@ -205,7 +205,7 @@ ioat_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fi
assert(ioat_ch->ioat_ch != NULL); assert(ioat_ch->ioat_ch != NULL);
ioat_task->cb = cb; ioat_task->cb = cb_fn;
return spdk_ioat_submit_fill(ioat_ch->ioat_ch, ioat_task, ioat_done, dst, fill64, nbytes); return spdk_ioat_submit_fill(ioat_ch->ioat_ch, ioat_task, ioat_done, dst, fill64, nbytes);
} }
@ -259,8 +259,8 @@ ioat_batch_create(struct spdk_io_channel *ch)
} }
static struct ioat_accel_op * static struct ioat_accel_op *
_prep_op(void *cb_arg, struct ioat_io_channel *ioat_ch, struct spdk_accel_batch *batch, _prep_op(struct ioat_io_channel *ioat_ch, struct spdk_accel_batch *batch,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_accel_op *op; struct ioat_accel_op *op;
@ -278,20 +278,20 @@ _prep_op(void *cb_arg, struct ioat_io_channel *ioat_ch, struct spdk_accel_batch
} }
op->cb_arg = cb_arg; op->cb_arg = cb_arg;
op->cb_fn = cb; op->cb_fn = cb_fn;
op->ioat_ch = ioat_ch; op->ioat_ch = ioat_ch;
return op; return op;
} }
static int static int
ioat_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, ioat_batch_prep_copy(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb) void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
ioat_task->cb = cb; ioat_task->cb = cb_fn;
ioat_ch->hw_batch = true; ioat_ch->hw_batch = true;
/* Call the IOAT library prep function. */ /* Call the IOAT library prep function. */
@ -299,14 +299,13 @@ ioat_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel
} }
static int static int
ioat_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch, ioat_batch_prep_fill(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, void *dst,
struct spdk_accel_batch *batch, void *dst, uint8_t fill, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
uint64_t nbytes, spdk_accel_completion_cb cb)
{ {
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
ioat_task->cb = cb; ioat_task->cb = cb_fn;
ioat_ch->hw_batch = true; ioat_ch->hw_batch = true;
/* Call the IOAT library prep function. */ /* Call the IOAT library prep function. */
@ -314,9 +313,9 @@ ioat_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
ioat_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch, ioat_batch_prep_dualcast(struct spdk_io_channel *ch,
struct spdk_accel_batch *batch, void *dst1, void *dst2, struct spdk_accel_batch *batch, void *dst1, void *dst2,
void *src, uint64_t nbytes, spdk_accel_completion_cb cb) void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_accel_op *op; struct ioat_accel_op *op;
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
@ -326,7 +325,7 @@ ioat_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch,
return -EINVAL; return -EINVAL;
} }
op = _prep_op(cb_arg, ioat_ch, batch, cb); op = _prep_op(ioat_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -343,14 +342,14 @@ ioat_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
ioat_batch_prep_compare(void *cb_arg, struct spdk_io_channel *ch, ioat_batch_prep_compare(struct spdk_io_channel *ch,
struct spdk_accel_batch *batch, void *src1, struct spdk_accel_batch *batch, void *src1,
void *src2, uint64_t nbytes, spdk_accel_completion_cb cb) void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_accel_op *op; struct ioat_accel_op *op;
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
op = _prep_op(cb_arg, ioat_ch, batch, cb); op = _prep_op(ioat_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -366,14 +365,14 @@ ioat_batch_prep_compare(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
ioat_batch_prep_crc32c(void *cb_arg, struct spdk_io_channel *ch, ioat_batch_prep_crc32c(struct spdk_io_channel *ch,
struct spdk_accel_batch *batch, uint32_t *dst, void *src, struct spdk_accel_batch *batch, uint32_t *dst, void *src,
uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb) uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_accel_op *op; struct ioat_accel_op *op;
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
op = _prep_op(cb_arg, ioat_ch, batch, cb); op = _prep_op(ioat_ch, batch, cb_fn, cb_arg);
if (op == NULL) { if (op == NULL) {
return -EINVAL; return -EINVAL;
} }
@ -390,8 +389,8 @@ ioat_batch_prep_crc32c(void *cb_arg, struct spdk_io_channel *ch,
} }
static int static int
ioat_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, ioat_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
spdk_accel_completion_cb cb) spdk_accel_completion_cb cb_fn, void *cb_arg)
{ {
struct ioat_accel_op *op; struct ioat_accel_op *op;
struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch);
@ -437,7 +436,7 @@ ioat_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_ba
/* Now complete the batch request itself. */ /* Now complete the batch request itself. */
accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg -
offsetof(struct spdk_accel_task, offload_ctx)); offsetof(struct spdk_accel_task, offload_ctx));
cb(accel_req, batch_status); cb_fn(accel_req, batch_status);
return 0; return 0;
} }

34
module/bdev/malloc/bdev_malloc.c
View File

@ -58,22 +58,10 @@ struct malloc_task {
enum spdk_bdev_io_status status; enum spdk_bdev_io_status status;
}; };
static struct malloc_task *
__malloc_task_from_accel_task(struct spdk_accel_task *ct)
{
return (struct malloc_task *)((uintptr_t)ct - sizeof(struct malloc_task));
}
static struct spdk_accel_task *
__accel_task_from_malloc_task(struct malloc_task *mt)
{
return (struct spdk_accel_task *)((uintptr_t)mt + sizeof(struct malloc_task));
}
static void static void
malloc_done(void *ref, int status) malloc_done(void *ref, int status)
{ {
struct malloc_task *task = __malloc_task_from_accel_task(ref); struct malloc_task *task = (struct malloc_task *)ref;
if (status != 0) { if (status != 0) {
if (status == -ENOMEM) { if (status == -ENOMEM) {
@ -98,7 +86,7 @@ static void bdev_malloc_get_spdk_running_config(FILE *fp);
static int static int
bdev_malloc_get_ctx_size(void) bdev_malloc_get_ctx_size(void)
{ {
return sizeof(struct malloc_task) + spdk_accel_task_size(); return sizeof(struct malloc_task);
} }
static struct spdk_bdev_module malloc_if = { static struct spdk_bdev_module malloc_if = {
@ -171,12 +159,11 @@ bdev_malloc_readv(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
task->num_outstanding = iovcnt; task->num_outstanding = iovcnt;
for (i = 0; i < iovcnt; i++) { for (i = 0; i < iovcnt; i++) {
res = spdk_accel_submit_copy(__accel_task_from_malloc_task(task), res = spdk_accel_submit_copy(ch, iov[i].iov_base,
ch, iov[i].iov_base, src, iov[i].iov_len, malloc_done, task);
src, iov[i].iov_len, malloc_done);
if (res != 0) { if (res != 0) {
malloc_done(__accel_task_from_malloc_task(task), res); malloc_done(task, res);
} }
src += iov[i].iov_len; src += iov[i].iov_len;
@ -206,12 +193,11 @@ bdev_malloc_writev(struct malloc_disk *mdisk, struct spdk_io_channel *ch,
task->num_outstanding = iovcnt; task->num_outstanding = iovcnt;
for (i = 0; i < iovcnt; i++) { for (i = 0; i < iovcnt; i++) {
res = spdk_accel_submit_copy(__accel_task_from_malloc_task(task), res = spdk_accel_submit_copy(ch, dst, iov[i].iov_base,
ch, dst, iov[i].iov_base, iov[i].iov_len, malloc_done, task);
iov[i].iov_len, malloc_done);
if (res != 0) { if (res != 0) {
malloc_done(__accel_task_from_malloc_task(task), res); malloc_done(task, res);
} }
dst += iov[i].iov_len; dst += iov[i].iov_len;
@ -228,8 +214,8 @@ bdev_malloc_unmap(struct malloc_disk *mdisk,
task->status = SPDK_BDEV_IO_STATUS_SUCCESS; task->status = SPDK_BDEV_IO_STATUS_SUCCESS;
task->num_outstanding = 1; task->num_outstanding = 1;
return spdk_accel_submit_fill(__accel_task_from_malloc_task(task), ch, return spdk_accel_submit_fill(ch, mdisk->malloc_buf + offset, 0,
mdisk->malloc_buf + offset, 0, byte_count, malloc_done); byte_count, malloc_done, task);
} }
static int64_t static int64_t