ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

idxd: Add flags parameter to all low level API for prep/submission

Browse Source 
Previously required flags were hardcoded in the low level library.
By having the user pass them in there is more flexbility and control.
This was driven by the need to add a new flag for pmem durability,
coming in a future patch in this series.

There is no change in functionality with this patch, just movement
of where flags are set and by whom and the plumbing of 'flags'..

Also note that some flags in scenarios that we know are required are
still set by the library.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I194278f9e3cec0886628585cf84bcc2eae635e0a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9449
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: GangCao <gang.cao@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
paul luse 2021-09-08 11:32:07 -04:00 committed by Jim Harris
parent 5990c4ecef
commit 7d63e716eb
5 changed files with 71 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -6,6 +6,11 @@
Removed deprecated spdk_bdev_module_finish_done(). Use spdk_bdev_module_fini_done() instead. Removed deprecated spdk_bdev_module_finish_done(). Use spdk_bdev_module_fini_done() instead.
### idxd
A new parameter `flags` was added to all low level submission and preparation
APIs to enable the caller to pass operation flags per the DSA specification.
### bdev_nvme ### bdev_nvme
Added `bdev_nvme_add_error_injection` and `bdev_nvme_remove_error_injection` RPCs to add and Added `bdev_nvme_add_error_injection` and `bdev_nvme_remove_error_injection` RPCs to add and

11
examples/idxd/perf/perf.c
View File

@ -520,6 +520,7 @@ _submit_single(struct idxd_chan_entry *t, struct idxd_task *task)
int rc = 0; int rc = 0;
struct iovec siov = {}; struct iovec siov = {};
struct iovec diov = {}; struct iovec diov = {};
int flags = 0;
assert(t); assert(t);
@ -536,7 +537,7 @@ _submit_single(struct idxd_chan_entry *t, struct idxd_task *task)
siov.iov_len = g_xfer_size_bytes; siov.iov_len = g_xfer_size_bytes;
diov.iov_base = task->dst; diov.iov_base = task->dst;
diov.iov_len = g_xfer_size_bytes; diov.iov_len = g_xfer_size_bytes;
rc = spdk_idxd_submit_copy(t->ch, &diov, 1, &siov, 1, rc = spdk_idxd_submit_copy(t->ch, &diov, 1, &siov, 1, flags,
idxd_done, task); idxd_done, task);
break; break;
case IDXD_FILL: case IDXD_FILL:
@ -544,14 +545,14 @@ _submit_single(struct idxd_chan_entry *t, struct idxd_task *task)
diov.iov_base = task->dst; diov.iov_base = task->dst;
diov.iov_len = g_xfer_size_bytes; diov.iov_len = g_xfer_size_bytes;
rc = spdk_idxd_submit_fill(t->ch, &diov, 1, *(uint8_t *)task->src, rc = spdk_idxd_submit_fill(t->ch, &diov, 1, *(uint8_t *)task->src,
idxd_done, task); flags, idxd_done, task);
break; break;
case IDXD_CRC32C: case IDXD_CRC32C:
assert(task->iovs != NULL); assert(task->iovs != NULL);
assert(task->iov_cnt > 0); assert(task->iov_cnt > 0);
rc = spdk_idxd_submit_crc32c(t->ch, task->iovs, task->iov_cnt, rc = spdk_idxd_submit_crc32c(t->ch, task->iovs, task->iov_cnt,
g_crc32c_seed, &task->crc_dst, g_crc32c_seed, &task->crc_dst,
idxd_done, task); flags, idxd_done, task);
break; break;
case IDXD_COMPARE: case IDXD_COMPARE:
random_num = rand() % 100; random_num = rand() % 100;
@ -567,11 +568,11 @@ _submit_single(struct idxd_chan_entry *t, struct idxd_task *task)
siov.iov_len = g_xfer_size_bytes; siov.iov_len = g_xfer_size_bytes;
diov.iov_base = task->dst; diov.iov_base = task->dst;
diov.iov_len = g_xfer_size_bytes; diov.iov_len = g_xfer_size_bytes;
rc = spdk_idxd_submit_compare(t->ch, &siov, 1, &diov, 1, idxd_done, task); rc = spdk_idxd_submit_compare(t->ch, &siov, 1, &diov, 1, flags, idxd_done, task);
break; break;
case IDXD_DUALCAST: case IDXD_DUALCAST:
rc = spdk_idxd_submit_dualcast(t->ch, task->dst, task->dst2, rc = spdk_idxd_submit_dualcast(t->ch, task->dst, task->dst2,
task->src, g_xfer_size_bytes, idxd_done, task); task->src, g_xfer_size_bytes, flags, idxd_done, task);
break; break;
default: default:
assert(false); assert(false);

18
include/spdk/idxd.h
View File

@ -127,6 +127,7 @@ void spdk_idxd_set_config(uint32_t config_number, bool kernel_mode);
* \param diovcnt Number of elements in diov * \param diovcnt Number of elements in diov
* \param siov Source iovec * \param siov Source iovec
* \param siovcnt Number of elements in siov * \param siovcnt Number of elements in siov
* \param flags Flags, optional flags that can vary per operation.
* \param cb_fn Callback function which will be called when the request is complete. * \param cb_fn Callback function which will be called when the request is complete.
* \param cb_arg Opaque value which will be passed back as the arg parameter in * \param cb_arg Opaque value which will be passed back as the arg parameter in
* the completion callback. * the completion callback.
@ -136,7 +137,7 @@ void spdk_idxd_set_config(uint32_t config_number, bool kernel_mode);
int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
struct iovec *diov, uint32_t diovcnt, struct iovec *diov, uint32_t diovcnt,
struct iovec *siov, uint32_t siovcnt, struct iovec *siov, uint32_t siovcnt,
spdk_idxd_req_cb cb_fn, void *cb_arg); int flags, spdk_idxd_req_cb cb_fn, void *cb_arg);
/** /**
* Build and submit an idxd dualcast request. * Build and submit an idxd dualcast request.
@ -149,6 +150,7 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
* \param dst2 Second destination virtual address (must be 4K aligned). * \param dst2 Second destination virtual address (must be 4K aligned).
* \param src Source virtual address. * \param src Source virtual address.
* \param nbytes Number of bytes to copy. * \param nbytes Number of bytes to copy.
* \param flags Flags, optional flags that can vary per operation.
* \param cb_fn Callback function which will be called when the request is complete. * \param cb_fn Callback function which will be called when the request is complete.
* \param cb_arg Opaque value which will be passed back as the arg parameter in * \param cb_arg Opaque value which will be passed back as the arg parameter in
* the completion callback. * the completion callback.
@ -156,7 +158,7 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
* \return 0 on success, negative errno on failure. * \return 0 on success, negative errno on failure.
*/ */
int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan,
void *dst1, void *dst2, const void *src, uint64_t nbytes, void *dst1, void *dst2, const void *src, uint64_t nbytes, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg); spdk_idxd_req_cb cb_fn, void *cb_arg);
/** /**
@ -170,6 +172,7 @@ int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan,
* \param siov1cnt Number of elements in siov1 * \param siov1cnt Number of elements in siov1
* \param siov2 Second source iovec * \param siov2 Second source iovec
* \param siov2cnt Number of elements in siov2 * \param siov2cnt Number of elements in siov2
* \param flags Flags, optional flags that can vary per operation.
* \param cb_fn Callback function which will be called when the request is complete. * \param cb_fn Callback function which will be called when the request is complete.
* \param cb_arg Opaque value which will be passed back as the arg parameter in * \param cb_arg Opaque value which will be passed back as the arg parameter in
* the completion callback. * the completion callback.
@ -179,7 +182,7 @@ int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan,
int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan,
struct iovec *siov1, size_t siov1cnt, struct iovec *siov1, size_t siov1cnt,
struct iovec *siov2, size_t siov2cnt, struct iovec *siov2, size_t siov2cnt,
spdk_idxd_req_cb cb_fn, void *cb_arg); int flags, spdk_idxd_req_cb cb_fn, void *cb_arg);
/** /**
* Build and submit a idxd memory fill request. * Build and submit a idxd memory fill request.
@ -191,6 +194,7 @@ int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan,
* \param diov Destination iovec * \param diov Destination iovec
* \param diovcnt Number of elements in diov * \param diovcnt Number of elements in diov
* \param fill_pattern Repeating eight-byte pattern to use for memory fill. * \param fill_pattern Repeating eight-byte pattern to use for memory fill.
* \param flags Flags, optional flags that can vary per operation.
* \param cb_fn Callback function which will be called when the request is complete. * \param cb_fn Callback function which will be called when the request is complete.
* \param cb_arg Opaque value which will be passed back as the cb_arg parameter * \param cb_arg Opaque value which will be passed back as the cb_arg parameter
* in the completion callback. * in the completion callback.
@ -199,7 +203,7 @@ int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan,
*/ */
int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
struct iovec *diov, size_t diovcnt, struct iovec *diov, size_t diovcnt,
uint64_t fill_pattern, spdk_idxd_req_cb cb_fn, void *cb_arg); uint64_t fill_pattern, int flags, spdk_idxd_req_cb cb_fn, void *cb_arg);
/** /**
* Build and submit a memory CRC32-C request. * Build and submit a memory CRC32-C request.
@ -212,6 +216,7 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
* \param siovcnt Number of elements in siov * \param siovcnt Number of elements in siov
* \param seed Four byte CRC-32C seed value. * \param seed Four byte CRC-32C seed value.
* \param crc_dst Resulting calculation. * \param crc_dst Resulting calculation.
* \param flags Flags, optional flags that can vary per operation.
* \param cb_fn Callback function which will be called when the request is complete. * \param cb_fn Callback function which will be called when the request is complete.
* \param cb_arg Opaque value which will be passed back as the cb_arg parameter * \param cb_arg Opaque value which will be passed back as the cb_arg parameter
* in the completion callback. * in the completion callback.
@ -220,7 +225,7 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
*/ */
int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan,
struct iovec *siov, size_t siovcnt, struct iovec *siov, size_t siovcnt,
uint32_t seed, uint32_t *crc_dst, uint32_t seed, uint32_t *crc_dst, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg); spdk_idxd_req_cb cb_fn, void *cb_arg);
/** /**
@ -236,6 +241,7 @@ int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan,
* \param siovcnt Number of elements in siov * \param siovcnt Number of elements in siov
* \param seed Four byte CRC-32C seed value. * \param seed Four byte CRC-32C seed value.
* \param crc_dst Resulting calculation. * \param crc_dst Resulting calculation.
* \param flags Flags, optional flags that can vary per operation.
* \param cb_fn Callback function which will be called when the request is complete. * \param cb_fn Callback function which will be called when the request is complete.
* \param cb_arg Opaque value which will be passed back as the cb_arg parameter * \param cb_arg Opaque value which will be passed back as the cb_arg parameter
* in the completion callback. * in the completion callback.
@ -245,7 +251,7 @@ int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan,
int spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, int spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan,
struct iovec *diov, size_t diovcnt, struct iovec *diov, size_t diovcnt,
struct iovec *siov, size_t siovcnt, struct iovec *siov, size_t siovcnt,
uint32_t seed, uint32_t *crc_dst, uint32_t seed, uint32_t *crc_dst, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg); spdk_idxd_req_cb cb_fn, void *cb_arg);
/** /**

76
lib/idxd/idxd.c
View File

@ -354,8 +354,8 @@ spdk_idxd_detach(struct spdk_idxd_device *idxd)
} }
static int static int
_idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, _idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, void *cb_arg,
void *cb_arg, struct idxd_hw_desc **_desc, struct idxd_ops **_op) int flags, struct idxd_hw_desc **_desc, struct idxd_ops **_op)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -373,7 +373,10 @@ _idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,
return -EBUSY; return -EBUSY;
} }
desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION; flags |= IDXD_FLAG_COMPLETION_ADDR_VALID;
flags |= IDXD_FLAG_REQUEST_COMPLETION;
desc->flags = flags;
op->cb_arg = cb_arg; op->cb_arg = cb_arg;
op->cb_fn = cb_fn; op->cb_fn = cb_fn;
op->batch = NULL; op->batch = NULL;
@ -389,7 +392,7 @@ _is_batch_valid(struct idxd_batch *batch, struct spdk_idxd_io_channel *chan)
static int static int
_idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, _idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,
void *cb_arg, struct idxd_batch *batch, void *cb_arg, struct idxd_batch *batch, int flags,
struct idxd_hw_desc **_desc, struct idxd_ops **_op) struct idxd_hw_desc **_desc, struct idxd_ops **_op)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
@ -417,8 +420,9 @@ _idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,
comp_addr = desc->completion_addr; comp_addr = desc->completion_addr;
memset(desc, 0, sizeof(*desc)); memset(desc, 0, sizeof(*desc));
desc->completion_addr = comp_addr; desc->completion_addr = comp_addr;
flags |= IDXD_FLAG_COMPLETION_ADDR_VALID;
desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION; flags |= IDXD_FLAG_REQUEST_COMPLETION;
desc->flags = flags;
op->cb_arg = cb_arg; op->cb_arg = cb_arg;
op->cb_fn = cb_fn; op->cb_fn = cb_fn;
op->batch = batch; op->batch = batch;
@ -501,7 +505,7 @@ idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
int i, rc; int i, rc, flags = 0;
assert(chan != NULL); assert(chan != NULL);
assert(batch != NULL); assert(batch != NULL);
@ -516,7 +520,7 @@ idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op); rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -600,7 +604,7 @@ _idxd_flush_batch(struct spdk_idxd_io_channel *chan)
static inline int static inline int
_idxd_submit_copy_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src, _idxd_submit_copy_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src,
uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) uint64_t nbytes, int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -617,7 +621,7 @@ _idxd_submit_copy_single(struct spdk_idxd_io_channel *chan, void *dst, const voi
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, flags, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -650,7 +654,7 @@ int
spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
struct iovec *diov, uint32_t diovcnt, struct iovec *diov, uint32_t diovcnt,
struct iovec *siov, uint32_t siovcnt, struct iovec *siov, uint32_t siovcnt,
spdk_idxd_req_cb cb_fn, void *cb_arg) int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -673,7 +677,7 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
return _idxd_submit_copy_single(chan, diov[0].iov_base, return _idxd_submit_copy_single(chan, diov[0].iov_base,
siov[0].iov_base, siov[0].iov_len, siov[0].iov_base, siov[0].iov_len,
cb_fn, cb_arg); flags, cb_fn, cb_arg);
} }
if (chan->batch) { if (chan->batch) {
@ -696,7 +700,7 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst); for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst);
len > 0; len > 0;
len = spdk_ioviter_next(&iter, &src, &dst)) { len = spdk_ioviter_next(&iter, &src, &dst)) {
rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, flags, &desc, &op);
if (rc) { if (rc) {
goto err; goto err;
} }
@ -732,7 +736,8 @@ err:
/* Dual-cast copies the same source to two separate destination buffers. */ /* Dual-cast copies the same source to two separate destination buffers. */
int int
spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2, spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2,
const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) const void *src, uint64_t nbytes, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -750,7 +755,7 @@ spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *d
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &op); rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -789,7 +794,7 @@ error:
static inline int static inline int
_idxd_submit_compare_single(struct spdk_idxd_io_channel *chan, void *src1, const void *src2, _idxd_submit_compare_single(struct spdk_idxd_io_channel *chan, void *src1, const void *src2,
uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) uint64_t nbytes, int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -806,7 +811,7 @@ _idxd_submit_compare_single(struct spdk_idxd_io_channel *chan, void *src1, const
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, flags, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -838,7 +843,7 @@ int
spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan,
struct iovec *siov1, size_t siov1cnt, struct iovec *siov1, size_t siov1cnt,
struct iovec *siov2, size_t siov2cnt, struct iovec *siov2, size_t siov2cnt,
spdk_idxd_req_cb cb_fn, void *cb_arg) int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -856,7 +861,7 @@ spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan,
} }
return _idxd_submit_compare_single(chan, siov1[0].iov_base, siov2[0].iov_base, siov1[0].iov_len, return _idxd_submit_compare_single(chan, siov1[0].iov_base, siov2[0].iov_base, siov1[0].iov_len,
cb_fn, cb_arg); flags, cb_fn, cb_arg);
} }
if (chan->batch) { if (chan->batch) {
@ -879,7 +884,7 @@ spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan,
for (len = spdk_ioviter_first(&iter, siov1, siov1cnt, siov2, siov2cnt, &src1, &src2); for (len = spdk_ioviter_first(&iter, siov1, siov1cnt, siov2, siov2cnt, &src1, &src2);
len > 0; len > 0;
len = spdk_ioviter_next(&iter, &src1, &src2)) { len = spdk_ioviter_next(&iter, &src1, &src2)) {
rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, flags, &desc, &op);
if (rc) { if (rc) {
goto err; goto err;
} }
@ -914,7 +919,7 @@ err:
static inline int static inline int
_idxd_submit_fill_single(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fill_pattern, _idxd_submit_fill_single(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fill_pattern,
uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) uint64_t nbytes, int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -930,7 +935,7 @@ _idxd_submit_fill_single(struct spdk_idxd_io_channel *chan, void *dst, uint64_t
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, flags, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -957,7 +962,8 @@ error:
int int
spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
struct iovec *diov, size_t diovcnt, struct iovec *diov, size_t diovcnt,
uint64_t fill_pattern, spdk_idxd_req_cb cb_fn, void *cb_arg) uint64_t fill_pattern, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -969,7 +975,7 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
if (diovcnt == 1) { if (diovcnt == 1) {
/* Simple case - filling one buffer */ /* Simple case - filling one buffer */
return _idxd_submit_fill_single(chan, diov[0].iov_base, fill_pattern, return _idxd_submit_fill_single(chan, diov[0].iov_base, fill_pattern,
diov[0].iov_len, cb_fn, cb_arg); diov[0].iov_len, flags, cb_fn, cb_arg);
} }
if (chan->batch) { if (chan->batch) {
@ -990,7 +996,7 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
} }
for (i = 0; i < diovcnt; i++) { for (i = 0; i < diovcnt; i++) {
rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, flags, &desc, &op);
if (rc) { if (rc) {
goto err; goto err;
} }
@ -1021,7 +1027,7 @@ err:
static inline int static inline int
_idxd_submit_crc32c_single(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, void *src, _idxd_submit_crc32c_single(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, void *src,
uint32_t seed, uint64_t nbytes, uint32_t seed, uint64_t nbytes, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
@ -1039,7 +1045,7 @@ _idxd_submit_crc32c_single(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst,
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, flags, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -1067,7 +1073,7 @@ error:
int int
spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan,
struct iovec *siov, size_t siovcnt, struct iovec *siov, size_t siovcnt,
uint32_t seed, uint32_t *crc_dst, uint32_t seed, uint32_t *crc_dst, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
@ -1081,7 +1087,7 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan,
if (siovcnt == 1) { if (siovcnt == 1) {
/* Simple case - crc on one buffer */ /* Simple case - crc on one buffer */
return _idxd_submit_crc32c_single(chan, crc_dst, siov[0].iov_base, return _idxd_submit_crc32c_single(chan, crc_dst, siov[0].iov_base,
seed, siov[0].iov_len, cb_fn, cb_arg); seed, siov[0].iov_len, flags, cb_fn, cb_arg);
} }
if (chan->batch) { if (chan->batch) {
@ -1103,7 +1109,7 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan,
prev_crc = NULL; prev_crc = NULL;
for (i = 0; i < siovcnt; i++) { for (i = 0; i < siovcnt; i++) {
rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, flags, &desc, &op);
if (rc) { if (rc) {
goto err; goto err;
} }
@ -1146,7 +1152,7 @@ err:
static inline int static inline int
_idxd_submit_copy_crc32c_single(struct spdk_idxd_io_channel *chan, void *dst, void *src, _idxd_submit_copy_crc32c_single(struct spdk_idxd_io_channel *chan, void *dst, void *src,
uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,
spdk_idxd_req_cb cb_fn, void *cb_arg) int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
struct idxd_ops *op; struct idxd_ops *op;
@ -1164,7 +1170,7 @@ _idxd_submit_copy_crc32c_single(struct spdk_idxd_io_channel *chan, void *dst, vo
} }
/* Common prep. */ /* Common prep. */
rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, chan->batch, flags, &desc, &op);
if (rc) { if (rc) {
return rc; return rc;
} }
@ -1199,7 +1205,7 @@ int
spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan,
struct iovec *diov, size_t diovcnt, struct iovec *diov, size_t diovcnt,
struct iovec *siov, size_t siovcnt, struct iovec *siov, size_t siovcnt,
uint32_t seed, uint32_t *crc_dst, uint32_t seed, uint32_t *crc_dst, int flags,
spdk_idxd_req_cb cb_fn, void *cb_arg) spdk_idxd_req_cb cb_fn, void *cb_arg)
{ {
struct idxd_hw_desc *desc; struct idxd_hw_desc *desc;
@ -1219,7 +1225,7 @@ spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan,
if (siovcnt == 1 && diovcnt == 1) { if (siovcnt == 1 && diovcnt == 1) {
/* Simple case - crc on one buffer */ /* Simple case - crc on one buffer */
return _idxd_submit_copy_crc32c_single(chan, diov[0].iov_base, siov[0].iov_base, return _idxd_submit_copy_crc32c_single(chan, diov[0].iov_base, siov[0].iov_base,
crc_dst, seed, siov[0].iov_len, cb_fn, cb_arg); crc_dst, seed, siov[0].iov_len, flags, cb_fn, cb_arg);
} }
if (chan->batch) { if (chan->batch) {
@ -1243,7 +1249,7 @@ spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan,
for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst); for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst);
len > 0; len > 0;
len = spdk_ioviter_next(&iter, &src, &dst)) { len = spdk_ioviter_next(&iter, &src, &dst)) {
rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, flags, &desc, &op);
if (rc) { if (rc) {
goto err; goto err;
} }

13
module/accel/idxd/accel_engine_idxd.c
View File

@ -150,6 +150,7 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
uint32_t iovcnt; uint32_t iovcnt;
struct iovec siov = {}; struct iovec siov = {};
struct iovec diov = {}; struct iovec diov = {};
int flags = 0;
switch (task->op_code) { switch (task->op_code) {
case ACCEL_OPCODE_MEMMOVE: case ACCEL_OPCODE_MEMMOVE:
@ -157,24 +158,24 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
siov.iov_len = task->nbytes; siov.iov_len = task->nbytes;
diov.iov_base = task->dst; diov.iov_base = task->dst;
diov.iov_len = task->nbytes; diov.iov_len = task->nbytes;
rc = spdk_idxd_submit_copy(chan->chan, &diov, 1, &siov, 1, idxd_done, task); rc = spdk_idxd_submit_copy(chan->chan, &diov, 1, &siov, 1, flags, idxd_done, task);
break; break;
case ACCEL_OPCODE_DUALCAST: case ACCEL_OPCODE_DUALCAST:
rc = spdk_idxd_submit_dualcast(chan->chan, task->dst, task->dst2, task->src, task->nbytes, rc = spdk_idxd_submit_dualcast(chan->chan, task->dst, task->dst2, task->src, task->nbytes,
idxd_done, task); flags, idxd_done, task);
break; break;
case ACCEL_OPCODE_COMPARE: case ACCEL_OPCODE_COMPARE:
siov.iov_base = task->src; siov.iov_base = task->src;
siov.iov_len = task->nbytes; siov.iov_len = task->nbytes;
diov.iov_base = task->dst; diov.iov_base = task->dst;
diov.iov_len = task->nbytes; diov.iov_len = task->nbytes;
rc = spdk_idxd_submit_compare(chan->chan, &siov, 1, &diov, 1, idxd_done, task); rc = spdk_idxd_submit_compare(chan->chan, &siov, 1, &diov, 1, flags, idxd_done, task);
break; break;
case ACCEL_OPCODE_MEMFILL: case ACCEL_OPCODE_MEMFILL:
memset(&task->fill_pattern, fill_pattern, sizeof(uint64_t)); memset(&task->fill_pattern, fill_pattern, sizeof(uint64_t));
diov.iov_base = task->dst; diov.iov_base = task->dst;
diov.iov_len = task->nbytes; diov.iov_len = task->nbytes;
rc = spdk_idxd_submit_fill(chan->chan, &diov, 1, task->fill_pattern, idxd_done, rc = spdk_idxd_submit_fill(chan->chan, &diov, 1, task->fill_pattern, flags, idxd_done,
task); task);
break; break;
case ACCEL_OPCODE_CRC32C: case ACCEL_OPCODE_CRC32C:
@ -188,7 +189,7 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
iovcnt = task->v.iovcnt; iovcnt = task->v.iovcnt;
} }
rc = spdk_idxd_submit_crc32c(chan->chan, iov, iovcnt, task->seed, task->crc_dst, rc = spdk_idxd_submit_crc32c(chan->chan, iov, iovcnt, task->seed, task->crc_dst,
idxd_done, task); flags, idxd_done, task);
break; break;
case ACCEL_OPCODE_COPY_CRC32C: case ACCEL_OPCODE_COPY_CRC32C:
if (task->v.iovcnt == 0) { if (task->v.iovcnt == 0) {
@ -203,7 +204,7 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task)
diov.iov_base = task->dst; diov.iov_base = task->dst;
diov.iov_len = task->nbytes; diov.iov_len = task->nbytes;
rc = spdk_idxd_submit_copy_crc32c(chan->chan, &diov, 1, iov, iovcnt, rc = spdk_idxd_submit_copy_crc32c(chan->chan, &diov, 1, iov, iovcnt,
task->seed, task->crc_dst, task->seed, task->crc_dst, flags,
idxd_done, task); idxd_done, task);
break; break;
default: default: