diff --git a/examples/idxd/perf/perf.c b/examples/idxd/perf/perf.c index b89522284..a9b624327 100644 --- a/examples/idxd/perf/perf.c +++ b/examples/idxd/perf/perf.c @@ -640,8 +640,8 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task) case IDXD_CRC32C: assert(task->iovs != NULL); assert(task->iov_cnt > 0); - rc = spdk_idxd_submit_crc32c(t->ch, &task->crc_dst, - task->iovs[0].iov_base, g_crc32c_seed, task->iovs[0].iov_len, + rc = spdk_idxd_submit_crc32c(t->ch, task->iovs, task->iov_cnt, + g_crc32c_seed, &task->crc_dst, idxd_done, task); break; case IDXD_COMPARE: diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index 7c03c24ca..3f4932c31 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -365,18 +365,19 @@ int spdk_idxd_batch_prep_crc32c(struct spdk_idxd_io_channel *chan, struct idxd_b * by writing to the proper device portal. * * \param chan IDXD channel to submit request. - * \param crc_dst Resulting calculation. - * \param src Source virtual address. + * \param siov Source iovec + * \param siovcnt Number of elements in siov * \param seed Four byte CRC-32C seed value. - * \param nbytes Number of bytes to calculate on. + * \param crc_dst Resulting calculation. * \param cb_fn Callback function which will be called when the request is complete. * \param cb_arg Opaque value which will be passed back as the cb_arg parameter * in the completion callback. * * \return 0 on success, negative errno on failure. */ -int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, void *src, - uint32_t seed, uint64_t nbytes, +int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, + struct iovec *siov, size_t siovcnt, + uint32_t seed, uint32_t *crc_dst, spdk_idxd_req_cb cb_fn, void *cb_arg); /** diff --git a/include/spdk/idxd_spec.h b/include/spdk/idxd_spec.h index 0437db2ed..3ae97a722 100644 --- a/include/spdk/idxd_spec.h +++ b/include/spdk/idxd_spec.h @@ -84,6 +84,8 @@ extern "C" { #define IDXD_FLAG_REQUEST_COMPLETION (1 << 3) #define IDXD_FLAG_CACHE_CONTROL (1 << 8) +#define IDXD_FLAG_CRC_READ_CRC_SEED (1 << 16) + /* * IDXD is a family of devices, DSA is the only currently * supported one. diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index fdb257856..812c059df 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -815,10 +815,10 @@ err: return rc; } -int -spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, void *src, - uint32_t seed, uint64_t nbytes, - spdk_idxd_req_cb cb_fn, void *cb_arg) +static inline int +_idxd_submit_crc32c_single(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, void *src, + uint32_t seed, uint64_t nbytes, + spdk_idxd_req_cb cb_fn, void *cb_arg) { struct idxd_hw_desc *desc; struct idxd_ops *op; @@ -856,6 +856,64 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *crc_dst, vo return 0; } +int +spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, + struct iovec *siov, size_t siovcnt, + uint32_t seed, uint32_t *crc_dst, + spdk_idxd_req_cb cb_fn, void *cb_arg) +{ + struct idxd_hw_desc *desc; + struct idxd_ops *op; + uint64_t src_addr; + int rc; + size_t i; + struct idxd_batch *batch; + void *prev_crc; + + if (siovcnt == 1) { + /* Simple case - crc on one buffer */ + return _idxd_submit_crc32c_single(chan, crc_dst, siov[0].iov_base, + seed, siov[0].iov_len, cb_fn, cb_arg); + } + + batch = spdk_idxd_batch_create(chan); + if (!batch) { + return -EBUSY; + } + + for (i = 0; i < siovcnt; i++) { + rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op); + if (rc) { + goto err; + } + + rc = _vtophys(siov[i].iov_base, &src_addr, siov[i].iov_len); + if (rc) { + goto err; + } + + desc->opcode = IDXD_OPCODE_CRC32C_GEN; + desc->dst_addr = 0; /* Per spec, needs to be clear. */ + desc->src_addr = src_addr; + if (i == 0) { + desc->crc32c.seed = seed; + } else { + desc->flags |= IDXD_FLAG_FENCE | IDXD_FLAG_CRC_READ_CRC_SEED; + desc->crc32c.addr = (uint64_t)prev_crc; + } + + desc->xfer_size = siov[i].iov_len; + prev_crc = &op->hw.crc32c_val; + op->crc_dst = crc_dst; + } + + return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg); + +err: + spdk_idxd_batch_cancel(chan, batch); + return rc; +} + int spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, void *dst, void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index 3a8e6f92f..48ff30fd7 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -152,6 +152,8 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task) int rc = 0; uint8_t fill_pattern = (uint8_t)task->fill_pattern; void *src; + struct iovec *iov; + uint32_t iovcnt; struct iovec siov = {}; struct iovec diov = {}; @@ -187,9 +189,17 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task) task); break; case ACCEL_OPCODE_CRC32C: - src = (task->v.iovcnt == 0) ? task->src : task->v.iovs[0].iov_base; - rc = spdk_idxd_submit_crc32c(chan->chan, task->crc_dst, src, task->seed, task->nbytes, idxd_done, - task); + if (task->v.iovcnt == 0) { + siov.iov_base = task->src; + siov.iov_len = task->nbytes; + iov = &siov; + iovcnt = 1; + } else { + iov = task->v.iovs; + iovcnt = task->v.iovcnt; + } + rc = spdk_idxd_submit_crc32c(chan->chan, iov, iovcnt, task->seed, task->crc_dst, + idxd_done, task); break; case ACCEL_OPCODE_COPY_CRC32C: src = (task->v.iovcnt == 0) ? task->src : task->v.iovs[0].iov_base;