diff --git a/CHANGELOG.md b/CHANGELOG.md index ed00b7cf3..2c03ec32e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ Using mlx5_dv requires libmlx5 installed on the system. ## v20.04: +IDXD engine support for compare has been added. + ### configuration Legacy INI style configuration for SPDK applications has been deprecated and will be diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index 0be6dfe12..6bb3305af 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -157,6 +157,26 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); +/** + * Build and submit a memory compare request. + * + * This function will build the compare descriptor and then immediately submit + * by writing to the proper device portal. + * + * \param chan IDXD channel to submit request. + * \param src1 First source to compare. + * \param src2 Second source to compare. + * \param nbytes Number of bytes to compare. + * \param cb_fn Callback function which will be called when the request is complete. + * \param cb_arg Opaque value which will be passed back as the arg parameter in + * the completion callback. + * + * \return 0 on success, negative errno on failure. + */ +int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, + void *src1, const void *src2, uint64_t nbytes, + spdk_idxd_req_cb cb_fn, void *cb_arg); + /** * Build and submit a DMA engine memory fill request. * diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index e0c6cfca0..5971bc2f5 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -688,6 +688,31 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void * return 0; } +int +spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, void *src1, const void *src2, + uint64_t nbytes, + spdk_idxd_req_cb cb_fn, void *cb_arg) +{ + struct idxd_hw_desc *desc; + + /* Common prep. */ + desc = _idxd_prep_command(chan, cb_fn, cb_arg); + if (desc == NULL) { + return -EBUSY; + } + + /* Command specific. */ + desc->opcode = IDXD_OPCODE_COMPARE; + desc->src_addr = (uintptr_t)src1; + desc->src2_addr = (uintptr_t)src2; + desc->xfer_size = nbytes; + + /* Submit operation. */ + movdir64b((uint64_t *)chan->ring_ctrl.portal, desc); + + return 0; +} + int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fill_pattern, uint64_t nbytes, @@ -784,9 +809,16 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) } desc = &chan->ring_ctrl.data_desc[index]; - if (desc->opcode == IDXD_OPCODE_CRC32C_GEN) { + switch (desc->opcode) { + case IDXD_OPCODE_CRC32C_GEN: *(uint32_t *)desc->dst_addr = comp->hw.crc32c_val; *(uint32_t *)desc->dst_addr ^= ~0; + break; + case IDXD_OPCODE_COMPARE: + if (status == 0) { + status = comp->hw.result; + } + break; } comp->cb_fn((void *)comp->cb_arg, status); diff --git a/lib/idxd/spdk_idxd.map b/lib/idxd/spdk_idxd.map index 6ffe442cc..916ef3c4e 100644 --- a/lib/idxd/spdk_idxd.map +++ b/lib/idxd/spdk_idxd.map @@ -7,6 +7,7 @@ spdk_idxd_probe; spdk_idxd_detach; spdk_idxd_set_config; + spdk_idxd_submit_compare; spdk_idxd_submit_crc32c; spdk_idxd_submit_copy; spdk_idxd_submit_fill; diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index 727c6bf5b..d2770b59c 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -85,7 +85,10 @@ struct idxd_op { void *cb_arg; spdk_idxd_req_cb cb_fn; void *src; - void *dst; + union { + void *dst; + void *src2; + }; uint32_t seed; uint64_t fill_pattern; uint32_t op_code; @@ -147,6 +150,10 @@ idxd_poll(void *arg) rc = spdk_idxd_submit_copy(op->chan, op->dst, op->src, op->nbytes, op->cb_fn, op->cb_arg); break; + case IDXD_OPCODE_COMPARE: + rc = spdk_idxd_submit_compare(op->chan, op->src, op->src2, op->nbytes, + op->cb_fn, op->cb_arg); + break; case IDXD_OPCODE_MEMFILL: rc = spdk_idxd_submit_fill(op->chan, op->dst, op->fill_pattern, op->nbytes, op->cb_fn, op->cb_arg); @@ -248,6 +255,47 @@ idxd_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, return rc; } +static int +idxd_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2, + uint64_t nbytes, + spdk_accel_completion_cb cb) +{ + struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; + struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); + int rc = 0; + + idxd_task->cb = cb; + + if (chan->state == IDXD_CHANNEL_ACTIVE) { + rc = spdk_idxd_submit_compare(chan->chan, src1, src2, nbytes, idxd_done, idxd_task); + } + + if (chan->state == IDXD_CHANNEL_PAUSED || rc == -EBUSY) { + struct idxd_op *op_to_queue; + + /* Commpom prep. */ + op_to_queue = _prep_queue_command(chan, idxd_done, idxd_task); + if (op_to_queue == NULL) { + return -ENOMEM; + } + + /* Command specific. */ + op_to_queue->src = src1; + op_to_queue->src2 = src2; + op_to_queue->nbytes = nbytes; + op_to_queue->op_code = IDXD_OPCODE_COMPARE; + + /* Queue the operation. */ + TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link); + return 0; + + } else if (chan->state == IDXD_CHANNEL_ERROR) { + return -EINVAL; + } + + return rc; +} + static int idxd_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb) @@ -334,12 +382,13 @@ idxd_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void static uint64_t idxd_get_capabilities(void) { - return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C; + return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C | ACCEL_COMPARE; } static struct spdk_accel_engine idxd_accel_engine = { .get_capabilities = idxd_get_capabilities, .copy = idxd_submit_copy, + .compare = idxd_submit_compare, .fill = idxd_submit_fill, .crc32c = idxd_submit_crc32c, .get_io_channel = idxd_get_io_channel,