diff --git a/doc/idxd.md b/doc/idxd.md index 088efb001..0f9f69329 100644 --- a/doc/idxd.md +++ b/doc/idxd.md @@ -10,6 +10,9 @@ Function | Description --------------------------------------- | ----------- spdk_idxd_probe() | @copybrief spdk_idxd_probe() spdk_idxd_submit_copy() | @copybrief spdk_idxd_submit_copy() +spdk_idxd_submit_compare() | @copybrief spdk_idxd_submit_compare() +spdk_idxd_submit_crc32c() | @copybrief spdk_idxd_submit_crc32c() +spdk_idxd_submit_dualcast | @copybrief spdk_idxd_submit_dualcast() spdk_idxd_submit_fill() | @copybrief spdk_idxd_submit_fill() # Pre-defined configurations {#idxd_configs} diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index 9b2039e9e..266d92c19 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -157,6 +157,27 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); +/** + * Build and submit an accel engine dual cast copy request. + * + * This function will build the dual cast descriptor and then immediately submit + * by writing to the proper device portal. + * + * \param chan IDXD channel to submit request. + * \param dst1 First destination virtual address (must be 4K aligned). + * \param dst2 Second destination virtual address (must be 4K aligned). + * \param src Source virtual address. + * \param nbytes Number of bytes to copy. + * \param cb_fn Callback function which will be called when the request is complete. + * \param cb_arg Opaque value which will be passed back as the arg parameter in + * the completion callback. + * + * \return 0 on success, negative errno on failure. + */ +int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, + void *dst1, void *dst2, const void *src, uint64_t nbytes, + spdk_idxd_req_cb cb_fn, void *cb_arg); + /** * Build and submit a memory compare request. * diff --git a/include/spdk_internal/accel_engine.h b/include/spdk_internal/accel_engine.h index f7a24f7df..63c82ea3b 100644 --- a/include/spdk_internal/accel_engine.h +++ b/include/spdk_internal/accel_engine.h @@ -48,6 +48,8 @@ struct spdk_accel_engine { uint64_t (*get_capabilities)(void); int (*copy)(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb); + int (*dualcast)(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src, + uint64_t nbytes, spdk_accel_completion_cb cb); int (*compare)(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb); int (*fill)(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 5971bc2f5..df3430412 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -43,6 +43,8 @@ #include "idxd.h" +#define ALIGN_4K 0x1000 + pthread_mutex_t g_driver_lock = PTHREAD_MUTEX_INITIALIZER; /* @@ -688,6 +690,37 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void * return 0; } +/* Dual-cast copies the same source to two separate destination buffers. */ +int +spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2, + const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) +{ + struct idxd_hw_desc *desc; + + if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { + SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); + return -EINVAL; + } + + /* Common prep. */ + desc = _idxd_prep_command(chan, cb_fn, cb_arg); + if (desc == NULL) { + return -EBUSY; + } + + /* Command specific. */ + desc->opcode = IDXD_OPCODE_DUALCAST; + desc->src_addr = (uintptr_t)src; + desc->dst_addr = (uintptr_t)dst1; + desc->dest2 = (uintptr_t)dst2; + desc->xfer_size = nbytes; + + /* Submit operation. */ + movdir64b((uint64_t *)chan->ring_ctrl.portal, desc); + + return 0; +} + int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, void *src1, const void *src2, uint64_t nbytes, diff --git a/lib/idxd/spdk_idxd.map b/lib/idxd/spdk_idxd.map index 916ef3c4e..4abce91ec 100644 --- a/lib/idxd/spdk_idxd.map +++ b/lib/idxd/spdk_idxd.map @@ -10,6 +10,7 @@ spdk_idxd_submit_compare; spdk_idxd_submit_crc32c; spdk_idxd_submit_copy; + spdk_idxd_submit_dualcast; spdk_idxd_submit_fill; spdk_idxd_process_events; spdk_idxd_get_channel; diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index c7345cf79..6bfee2c05 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -89,6 +89,7 @@ struct idxd_op { void *dst; void *src2; }; + void *dst2; uint32_t seed; uint64_t fill_pattern; uint32_t op_code; @@ -150,6 +151,10 @@ idxd_poll(void *arg) rc = spdk_idxd_submit_copy(op->chan, op->dst, op->src, op->nbytes, op->cb_fn, op->cb_arg); break; + case IDXD_OPCODE_DUALCAST: + rc = spdk_idxd_submit_dualcast(op->chan, op->dst, op->dst2, op->src, op->nbytes, + op->cb_fn, op->cb_arg); + break; case IDXD_OPCODE_COMPARE: rc = spdk_idxd_submit_compare(op->chan, op->src, op->src2, op->nbytes, op->cb_fn, op->cb_arg); @@ -255,6 +260,48 @@ idxd_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, return rc; } +static int +idxd_submit_dualcast(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src, + uint64_t nbytes, + spdk_accel_completion_cb cb) +{ + struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; + struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); + int rc = 0; + + idxd_task->cb = cb; + + if (chan->state == IDXD_CHANNEL_ACTIVE) { + rc = spdk_idxd_submit_dualcast(chan->chan, dst1, dst2, src, nbytes, idxd_done, idxd_task); + } + + if (chan->state == IDXD_CHANNEL_PAUSED || rc == -EBUSY) { + struct idxd_op *op_to_queue; + + /* Commpom prep. */ + op_to_queue = _prep_queue_command(chan, idxd_done, idxd_task); + if (op_to_queue == NULL) { + return -ENOMEM; + } + + /* Command specific. */ + op_to_queue->dst = dst1; + op_to_queue->dst2 = dst2; + op_to_queue->src = src; + op_to_queue->nbytes = nbytes; + op_to_queue->op_code = IDXD_OPCODE_DUALCAST; + + /* Queue the operation. */ + TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link); + return 0; + + } else if (chan->state == IDXD_CHANNEL_ERROR) { + return -EINVAL; + } + + return rc; +} + static int idxd_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes, @@ -382,12 +429,14 @@ idxd_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void static uint64_t idxd_get_capabilities(void) { - return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C | ACCEL_COMPARE; + return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C | ACCEL_COMPARE | + ACCEL_DUALCAST; } static struct spdk_accel_engine idxd_accel_engine = { .get_capabilities = idxd_get_capabilities, .copy = idxd_submit_copy, + .dualcast = idxd_submit_dualcast, .compare = idxd_submit_compare, .fill = idxd_submit_fill, .crc32c = idxd_submit_crc32c,