diff --git a/CHANGELOG.md b/CHANGELOG.md index b3efb9405..b2870d05e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ The batching capability was removed. Batching is now considered an implementation detail of the low level drivers. +### idxd + +Many APIs are now vectored rather than scalar, meaning they take iovecs instead of individual pointers. + ### nvme API `spdk_nvme_trtype_is_fabrics` was added to return existing transport type diff --git a/examples/idxd/perf/perf.c b/examples/idxd/perf/perf.c index f5afce107..bacb70304 100644 --- a/examples/idxd/perf/perf.c +++ b/examples/idxd/perf/perf.c @@ -609,6 +609,8 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task) { int random_num; int rc = 0; + struct iovec siov = {}; + struct iovec diov = {}; assert(t); t->current_queue_depth++; @@ -621,8 +623,12 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task) switch (g_workload_selection) { case IDXD_COPY: - rc = spdk_idxd_submit_copy(t->ch, task->dst, task->src, - g_xfer_size_bytes, idxd_done, task); + siov.iov_base = task->src; + siov.iov_len = g_xfer_size_bytes; + diov.iov_base = task->dst; + diov.iov_len = g_xfer_size_bytes; + rc = spdk_idxd_submit_copy(t->ch, &diov, 1, &siov, 1, + idxd_done, task); break; case IDXD_FILL: /* For fill use the first byte of the task->dst buffer */ diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index ad00301cd..a2034e4e4 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -217,9 +217,10 @@ int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd * by writing to the proper device portal. * * \param chan IDXD channel to submit request. - * \param dst Destination virtual address. - * \param src Source virtual address. - * \param nbytes Number of bytes to copy. + * \param diov Destination iovec + * \param diovcnt Number of elements in diov + * \param siov Source iovec + * \param siovcnt Number of elements in siov * \param cb_fn Callback function which will be called when the request is complete. * \param cb_arg Opaque value which will be passed back as the arg parameter in * the completion callback. @@ -227,7 +228,8 @@ int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd * \return 0 on success, negative errno on failure. */ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, - void *dst, const void *src, uint64_t nbytes, + struct iovec *diov, uint32_t diovcnt, + struct iovec *siov, uint32_t siovcnt, spdk_idxd_req_cb cb_fn, void *cb_arg); /** diff --git a/lib/idxd/Makefile b/lib/idxd/Makefile index 7d599e567..38a872f9f 100644 --- a/lib/idxd/Makefile +++ b/lib/idxd/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -SO_VER := 6 +SO_VER := 7 SO_MINOR := 0 C_SRCS = idxd.c idxd_user.c diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 7f090b0b0..477ba5152 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -456,10 +456,9 @@ _idxd_batch_prep_nop(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch return 0; } - -int -spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *src, - uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) +static inline int +_idxd_submit_copy_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src, + uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) { struct idxd_hw_desc *desc; struct idxd_ops *op; @@ -502,6 +501,72 @@ error: return rc; } +int +spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, + struct iovec *diov, uint32_t diovcnt, + struct iovec *siov, uint32_t siovcnt, + spdk_idxd_req_cb cb_fn, void *cb_arg) +{ + struct idxd_hw_desc *desc; + struct idxd_ops *op; + void *src, *dst; + uint64_t src_addr, dst_addr; + int rc; + uint64_t len; + struct idxd_batch *batch; + struct spdk_ioviter iter; + + assert(chan != NULL); + assert(diov != NULL); + assert(siov != NULL); + + if (diovcnt == 1 && siovcnt == 1) { + /* Simple case - copying one buffer to another */ + if (diov[0].iov_len < siov[0].iov_len) { + return -EINVAL; + } + + return _idxd_submit_copy_single(chan, diov[0].iov_base, + siov[0].iov_base, siov[0].iov_len, + cb_fn, cb_arg); + } + + batch = spdk_idxd_batch_create(chan); + if (!batch) { + return -EBUSY; + } + + for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst); + len > 0; + len = spdk_ioviter_next(&iter, &src, &dst)) { + rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); + if (rc) { + goto err; + } + + rc = _vtophys(src, &src_addr, len); + if (rc) { + goto err; + } + + rc = _vtophys(dst, &dst_addr, len); + if (rc) { + goto err; + } + + desc->opcode = IDXD_OPCODE_MEMMOVE; + desc->src_addr = src_addr; + desc->dst_addr = dst_addr; + desc->xfer_size = len; + } + + return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + +err: + spdk_idxd_batch_cancel(chan, batch); + return rc; +} + /* Dual-cast copies the same source to two separate destination buffers. */ int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2, diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index ae6c7fe9f..ce18568c2 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -47,7 +47,7 @@ DEPDIRS-rte_vhost := DEPDIRS-env_dpdk := log util DEPDIRS-ioat := log -DEPDIRS-idxd := log +DEPDIRS-idxd := log util DEPDIRS-sock := log $(JSON_LIBS) DEPDIRS-util := log DEPDIRS-vmd := log diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index 145c55eae..18097e821 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -152,6 +152,8 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task) int rc = 0; uint8_t fill_pattern = (uint8_t)task->fill_pattern; void *src; + struct iovec siov = {}; + struct iovec diov = {}; if (chan->num_outstanding == chan->max_outstanding) { chan->state = IDXD_CHANNEL_PAUSED; @@ -160,7 +162,11 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task) switch (task->op_code) { case ACCEL_OPCODE_MEMMOVE: - rc = spdk_idxd_submit_copy(chan->chan, task->dst, task->src, task->nbytes, idxd_done, task); + siov.iov_base = task->src; + siov.iov_len = task->nbytes; + diov.iov_base = task->dst; + diov.iov_len = task->nbytes; + rc = spdk_idxd_submit_copy(chan->chan, &diov, 1, &siov, 1, idxd_done, task); break; case ACCEL_OPCODE_DUALCAST: rc = spdk_idxd_submit_dualcast(chan->chan, task->dst, task->dst2, task->src, task->nbytes,