From fa6ac87778f0bf4c080fc0c105fb2db5abae2d14 Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Wed, 17 Nov 2021 10:08:05 -0700 Subject: [PATCH] idxd: Add support for vectored fill operations Signed-off-by: Ben Walker Change-Id: I0d58320a03ee82169e83be6449ba52c9d2ee3a55 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10288 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Changpeng Liu --- examples/idxd/perf/perf.c | 6 ++- include/spdk/idxd.h | 8 ++-- lib/idxd/idxd.c | 54 +++++++++++++++++++++++++-- module/accel/idxd/accel_engine_idxd.c | 4 +- 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/examples/idxd/perf/perf.c b/examples/idxd/perf/perf.c index 76563c4e1..b89522284 100644 --- a/examples/idxd/perf/perf.c +++ b/examples/idxd/perf/perf.c @@ -632,8 +632,10 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task) break; case IDXD_FILL: /* For fill use the first byte of the task->dst buffer */ - rc = spdk_idxd_submit_fill(t->ch, task->dst, *(uint8_t *)task->src, - g_xfer_size_bytes, idxd_done, task); + diov.iov_base = task->dst; + diov.iov_len = g_xfer_size_bytes; + rc = spdk_idxd_submit_fill(t->ch, &diov, 1, *(uint8_t *)task->src, + idxd_done, task); break; case IDXD_CRC32C: assert(task->iovs != NULL); diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index bf7226554..7c03c24ca 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -323,9 +323,9 @@ int spdk_idxd_batch_prep_fill(struct spdk_idxd_io_channel *chan, struct idxd_bat * by writing to the proper device portal. * * \param chan IDXD channel to submit request. - * \param dst Destination virtual address. + * \param diov Destination iovec + * \param diovcnt Number of elements in diov * \param fill_pattern Repeating eight-byte pattern to use for memory fill. - * \param nbytes Number of bytes to fill. * \param cb_fn Callback function which will be called when the request is complete. * \param cb_arg Opaque value which will be passed back as the cb_arg parameter * in the completion callback. @@ -333,8 +333,8 @@ int spdk_idxd_batch_prep_fill(struct spdk_idxd_io_channel *chan, struct idxd_bat * \return 0 on success, negative errno on failure. */ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, - void *dst, uint64_t fill_pattern, uint64_t nbytes, - spdk_idxd_req_cb cb_fn, void *cb_arg); + struct iovec *diov, size_t diovcnt, + uint64_t fill_pattern, spdk_idxd_req_cb cb_fn, void *cb_arg); /** * Synchronous call to prepare a crc32c request into a previously initialized batch diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 3bbaf83b2..fdb257856 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -730,9 +730,9 @@ err: return rc; } -int -spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fill_pattern, - uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) +static inline int +_idxd_submit_fill_single(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fill_pattern, + uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) { struct idxd_hw_desc *desc; struct idxd_ops *op; @@ -767,6 +767,54 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fil return 0; } +int +spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, + struct iovec *diov, size_t diovcnt, + uint64_t fill_pattern, spdk_idxd_req_cb cb_fn, void *cb_arg) +{ + struct idxd_hw_desc *desc; + struct idxd_ops *op; + uint64_t dst_addr; + int rc; + size_t i; + struct idxd_batch *batch; + + if (diovcnt == 1) { + /* Simple case - filling one buffer */ + return _idxd_submit_fill_single(chan, diov[0].iov_base, fill_pattern, + diov[0].iov_len, cb_fn, cb_arg); + } + + batch = spdk_idxd_batch_create(chan); + if (!batch) { + return -EBUSY; + } + + for (i = 0; i < diovcnt; i++) { + rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); + if (rc) { + goto err; + } + + rc = _vtophys(diov[i].iov_base, &dst_addr, diov[i].iov_len); + if (rc) { + goto err; + } + + desc->opcode = IDXD_OPCODE_MEMFILL; + desc->pattern = fill_pattern; + desc->dst_addr = dst_addr; + desc->xfer_size = diov[i].iov_len; + desc->flags |= IDXD_FLAG_CACHE_CONTROL; /* direct IO to CPU cache instead of mem */ + } + + return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + +err: + spdk_idxd_batch_cancel(chan, batch); + return rc; +} + int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes, diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index 29eacce92..3a8e6f92f 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -181,7 +181,9 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task) break; case ACCEL_OPCODE_MEMFILL: memset(&task->fill_pattern, fill_pattern, sizeof(uint64_t)); - rc = spdk_idxd_submit_fill(chan->chan, task->dst, task->fill_pattern, task->nbytes, idxd_done, + diov.iov_base = task->dst; + diov.iov_len = task->nbytes; + rc = spdk_idxd_submit_fill(chan->chan, &diov, 1, task->fill_pattern, idxd_done, task); break; case ACCEL_OPCODE_CRC32C: