From 22058d70fc2d884579f024e56a7ac9719a2170dc Mon Sep 17 00:00:00 2001 From: paul luse Date: Tue, 16 Jun 2020 13:24:35 -0400 Subject: [PATCH] lib/idxd: add support for batching dualcast commands Signed-off-by: paul luse Change-Id: Ia5b5a346ae5c8470d9792766dba2bcf68c6fe2fd Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2915 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- include/spdk/idxd.h | 27 ++++++++++++++-- lib/idxd/idxd.c | 45 +++++++++++++++++++++++++++ lib/idxd/spdk_idxd.map | 1 + module/accel/idxd/accel_engine_idxd.c | 17 ++++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index 2a54fc883..89850954d 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -32,7 +32,7 @@ */ /** \file - * IDXD accel engine driver public interface + * IDXD driver public interface */ #ifndef SPDK_IDXD_H @@ -193,7 +193,28 @@ int spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_bat void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); /** - * Build and submit a idxd memory copy request. + * Synchronous call to prepare a dualcast request into a previously initialized batch + * created with spdk_idxd_batch_create(). The callback will be called when the dualcast + * completes after the batch has been submitted by an asynchronous call to + * spdk_idxd_batch_submit(). + * + * \param chan IDXD channel to submit request. + * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). + * \param dst1 First destination virtual address (must be 4K aligned). + * \param dst2 Second destination virtual address (must be 4K aligned). + * \param src Source virtual address. + * \param nbytes Number of bytes to copy. + * \param cb_fn Callback function which will be called when the request is complete. + * \param cb_arg Opaque value which will be passed back as the arg parameter in + * the completion callback. + * + * \return 0 on success, negative errno on failure. + */ +int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, + void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); + +/** + * Build and submit an idxd memory copy request. * * This function will build the copy descriptor and then immediately submit * by writing to the proper device portal. @@ -213,7 +234,7 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, void *cb_arg); /** - * Build and submit an idxd dual cast copy request. + * Build and submit an idxd dualcast request. * * This function will build the dual cast descriptor and then immediately submit * by writing to the proper device portal. diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 11081b7c7..a9cab6fa6 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -1005,6 +1005,50 @@ spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_batch * return 0; } +int +spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, + void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg) +{ + struct idxd_hw_desc *desc; + struct idxd_comp *comp; + + if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { + SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); + return -EINVAL; + } + + if (_does_batch_exist(batch, chan) == false) { + SPDK_ERRLOG("Attempt to add to a batch that doesn't exist\n."); + return -EINVAL; + } + + if ((batch->cur_index - batch->start_index) == DESC_PER_BATCH) { + SPDK_ERRLOG("Attempt to add to a batch that is already full\n."); + return -ENOMEM; + } + + desc = &chan->ring_ctrl.user_desc[batch->cur_index]; + comp = &chan->ring_ctrl.user_completions[batch->cur_index]; + SPDK_DEBUGLOG(SPDK_LOG_IDXD, "Prep batch %p index %u\n", batch, batch->cur_index); + + batch->cur_index++; + assert(batch->cur_index > batch->start_index); + + desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION; + desc->opcode = IDXD_OPCODE_DUALCAST; + desc->src_addr = (uint64_t)src; + desc->dst_addr = (uint64_t)dst1; + desc->dest2 = (uint64_t)dst2; + desc->xfer_size = nbytes; + + desc->completion_addr = (uint64_t)&comp->hw; + comp->cb_arg = cb_arg; + comp->cb_fn = cb_fn; + comp->batch = batch; + + return 0; +} + static void _dump_error_reg(struct spdk_idxd_io_channel *chan) { @@ -1070,6 +1114,7 @@ _spdk_idxd_process_batch_events(struct spdk_idxd_io_channel *chan) status = comp->hw.result; } break; + case IDXD_OPCODE_DUALCAST: case IDXD_OPCODE_MEMMOVE: break; default: diff --git a/lib/idxd/spdk_idxd.map b/lib/idxd/spdk_idxd.map index 0177bdd57..ea7dc619d 100644 --- a/lib/idxd/spdk_idxd.map +++ b/lib/idxd/spdk_idxd.map @@ -7,6 +7,7 @@ spdk_idxd_probe; spdk_idxd_detach; spdk_idxd_batch_prep_copy; + spdk_idxd_batch_prep_dualcast; spdk_idxd_batch_submit; spdk_idxd_batch_create; spdk_idxd_batch_get_max; diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index f83d1bead..e4f8aa634 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -47,6 +47,8 @@ #include "spdk/util.h" #include "spdk/json.h" +#define ALIGN_4K 0x1000 + static bool g_idxd_enable = false; uint32_t g_config_number; @@ -494,12 +496,27 @@ idxd_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel idxd_done, idxd_task); } +static int +idxd_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, + void *dst1, void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb) +{ + struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; + struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); + struct idxd_batch *batch = (struct idxd_batch *)_batch; + + idxd_task->cb = cb; + + return spdk_idxd_batch_prep_dualcast(chan->chan, batch, dst1, dst2, src, nbytes, idxd_done, + idxd_task); +} + static struct spdk_accel_engine idxd_accel_engine = { .get_capabilities = idxd_get_capabilities, .copy = idxd_submit_copy, .batch_get_max = idxd_batch_get_max, .batch_create = idxd_batch_start, .batch_prep_copy = idxd_batch_prep_copy, + .batch_prep_dualcast = idxd_batch_prep_dualcast, .batch_submit = idxd_batch_submit, .dualcast = idxd_submit_dualcast, .compare = idxd_submit_compare,