From 8d059e7a18c46bbff29c457564a77358e79f891c Mon Sep 17 00:00:00 2001 From: paul luse Date: Thu, 16 Jul 2020 18:47:59 -0400 Subject: [PATCH] accel: add API to cancel a batch sequence Added to the framework as well as all 3 engines. Needed by apps in the event that they have to fail following the creation of a batch, allows them to tell the framework to forget about the batch as they have no intent to send it. Signed-off-by: paul luse Change-Id: Id94754ab1350e5a969a5fd2306bd59c38f0a0120 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3389 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- include/spdk/accel_engine.h | 11 +++++++++ include/spdk/idxd.h | 10 ++++++++ include/spdk_internal/accel_engine.h | 1 + lib/accel/accel_engine.c | 33 +++++++++++++++++++++++++++ lib/accel/spdk_accel.map | 1 + lib/idxd/idxd.c | 20 ++++++++++++++++ lib/idxd/spdk_idxd.map | 1 + module/accel/idxd/accel_engine_idxd.c | 10 ++++++++ module/accel/ioat/accel_engine_ioat.c | 25 ++++++++++++++++++++ 9 files changed, 112 insertions(+) diff --git a/include/spdk/accel_engine.h b/include/spdk/accel_engine.h index 82e76bd4e..be48e2ce3 100644 --- a/include/spdk/accel_engine.h +++ b/include/spdk/accel_engine.h @@ -166,6 +166,17 @@ struct spdk_accel_batch *spdk_accel_batch_create(struct spdk_io_channel *ch); int spdk_accel_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, spdk_accel_completion_cb cb_fn, void *cb_arg); +/** + * Synchronous call to cancel a batch sequence. In some cases prepared commands will be + * processed if they cannot be cancelled. + * + * \param ch I/O channel associated with this call. + * \param batch Handle provided when the batch was started with spdk_accel_batch_create(). + * + * \return 0 on success, negative errno on failure. + */ +int spdk_accel_batch_cancel(struct spdk_io_channel *ch, struct spdk_accel_batch *batch); + /** * Synchronous call to prepare a copy request into a previously initialized batch * created with spdk_accel_batch_create(). The callback will be called when the copy diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index 9e22e9710..cb9ebe8b8 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -172,6 +172,16 @@ struct idxd_batch *spdk_idxd_batch_create(struct spdk_idxd_io_channel *chan); int spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, spdk_idxd_req_cb cb_fn, void *cb_arg); +/** + * Cancel a batch sequence. + * + * \param chan IDXD channel to submit request. + * \param batch Handle provided when the batch was started with spdk_idxd_batch_create(). + * + * \return 0 on success, negative errno on failure. + */ +int spdk_idxd_batch_cancel(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch); + /** * Synchronous call to prepare a copy request into a previously initialized batch * created with spdk_idxd_batch_create(). The callback will be called when the copy diff --git a/include/spdk_internal/accel_engine.h b/include/spdk_internal/accel_engine.h index 3e047cf4a..9b78bc967 100644 --- a/include/spdk_internal/accel_engine.h +++ b/include/spdk_internal/accel_engine.h @@ -67,6 +67,7 @@ struct spdk_accel_engine { spdk_accel_completion_cb cb_fn, void *cb_arg); int (*batch_submit)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, spdk_accel_completion_cb cb_fn, void *cb_arg); + int (*batch_cancel)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch); int (*compare)(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg); int (*fill)(struct spdk_io_channel *ch, void *dst, uint8_t fill, diff --git a/lib/accel/accel_engine.c b/lib/accel/accel_engine.c index 753cef4cb..03a405439 100644 --- a/lib/accel/accel_engine.c +++ b/lib/accel/accel_engine.c @@ -231,6 +231,17 @@ spdk_accel_batch_get_max(struct spdk_io_channel *ch) return accel_ch->engine->batch_get_max(); } +/* Accel framework public API for for when an app is unable to complete a batch sequence, + * it cancels with this API. + */ +int +spdk_accel_batch_cancel(struct spdk_io_channel *ch, struct spdk_accel_batch *batch) +{ + struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); + + return accel_ch->engine->batch_cancel(accel_ch->ch, batch); +} + /* Accel framework public API for batch prep_copy function. All engines are * required to implement this API. */ @@ -791,6 +802,27 @@ sw_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch * return 0; } + +static int +sw_accel_batch_cancel(struct spdk_io_channel *ch, struct spdk_accel_batch *batch) +{ + struct sw_accel_op *op; + struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); + + if ((struct spdk_accel_batch *)&sw_ch->batch != batch) { + SPDK_ERRLOG("Invalid batch\n"); + return -EINVAL; + } + + /* Cancel the batch items by moving them back to the op_pool. */ + while ((op = TAILQ_FIRST(&sw_ch->batch))) { + TAILQ_REMOVE(&sw_ch->batch, op, link); + TAILQ_INSERT_TAIL(&sw_ch->op_pool, op, link); + } + + return 0; +} + static int sw_accel_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, spdk_accel_completion_cb cb_fn, void *cb_arg) @@ -927,6 +959,7 @@ static struct spdk_accel_engine sw_accel_engine = { .dualcast = sw_accel_submit_dualcast, .batch_get_max = sw_accel_batch_get_max, .batch_create = sw_accel_batch_start, + .batch_cancel = sw_accel_batch_cancel, .batch_prep_copy = sw_accel_batch_prep_copy, .batch_prep_dualcast = sw_accel_batch_prep_dualcast, .batch_prep_compare = sw_accel_batch_prep_compare, diff --git a/lib/accel/spdk_accel.map b/lib/accel/spdk_accel.map index 8adb611a3..bfccf0a90 100644 --- a/lib/accel/spdk_accel.map +++ b/lib/accel/spdk_accel.map @@ -16,6 +16,7 @@ spdk_accel_batch_prep_fill; spdk_accel_batch_prep_crc32c; spdk_accel_batch_submit; + spdk_accel_batch_cancel; spdk_accel_submit_copy; spdk_accel_submit_dualcast; spdk_accel_submit_compare; diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index a333bd3eb..992d96211 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -926,6 +926,26 @@ _does_batch_exist(struct idxd_batch *batch, struct spdk_idxd_io_channel *chan) return found; } +int +spdk_idxd_batch_cancel(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch) +{ + if (_does_batch_exist(batch, chan) == false) { + SPDK_ERRLOG("Attempt to cancel a batch that doesn't exist\n."); + return -EINVAL; + } + + if (batch->remaining > 0) { + SPDK_ERRLOG("Cannot cancel batch, already submitted to HW\n."); + return -EINVAL; + } + + TAILQ_REMOVE(&chan->batches, batch, link); + spdk_bit_array_clear(chan->ring_ctrl.user_ring_slots, batch->batch_num); + TAILQ_INSERT_TAIL(&chan->batch_pool, batch, link); + + return 0; +} + int spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch, spdk_idxd_req_cb cb_fn, void *cb_arg) diff --git a/lib/idxd/spdk_idxd.map b/lib/idxd/spdk_idxd.map index a12ec5036..4bffdf209 100644 --- a/lib/idxd/spdk_idxd.map +++ b/lib/idxd/spdk_idxd.map @@ -13,6 +13,7 @@ spdk_idxd_batch_prep_compare; spdk_idxd_batch_submit; spdk_idxd_batch_create; + spdk_idxd_batch_cancel; spdk_idxd_batch_get_max; spdk_idxd_set_config; spdk_idxd_submit_compare; diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index e9d92eab5..e5af0181f 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -443,6 +443,15 @@ idxd_batch_start(struct spdk_io_channel *ch) return (struct spdk_accel_batch *)spdk_idxd_batch_create(chan->chan); } +static int +idxd_batch_cancel(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch) +{ + struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); + struct idxd_batch *batch = (struct idxd_batch *)_batch; + + return spdk_idxd_batch_cancel(chan->chan, batch); +} + static int idxd_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *_batch, spdk_accel_completion_cb cb_fn, void *cb_arg) @@ -561,6 +570,7 @@ static struct spdk_accel_engine idxd_accel_engine = { .copy = idxd_submit_copy, .batch_get_max = idxd_batch_get_max, .batch_create = idxd_batch_start, + .batch_cancel = idxd_batch_cancel, .batch_prep_copy = idxd_batch_prep_copy, .batch_prep_fill = idxd_batch_prep_fill, .batch_prep_dualcast = idxd_batch_prep_dualcast, diff --git a/module/accel/ioat/accel_engine_ioat.c b/module/accel/ioat/accel_engine_ioat.c index 966c547ae..0fff3a7c5 100644 --- a/module/accel/ioat/accel_engine_ioat.c +++ b/module/accel/ioat/accel_engine_ioat.c @@ -390,6 +390,30 @@ ioat_batch_prep_crc32c(struct spdk_io_channel *ch, return 0; } +static int +ioat_batch_cancel(struct spdk_io_channel *ch, struct spdk_accel_batch *batch) +{ + struct ioat_accel_op *op; + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + + if ((struct spdk_accel_batch *)&ioat_ch->hw_batch != batch) { + SPDK_ERRLOG("Invalid batch\n"); + return -EINVAL; + } + + /* Flush the batched HW items, there's no way to cancel these without resetting. */ + spdk_ioat_flush(ioat_ch->ioat_ch); + ioat_ch->hw_batch = false; + + /* Return batched software items to the pool. */ + while ((op = TAILQ_FIRST(&ioat_ch->sw_batch))) { + TAILQ_REMOVE(&ioat_ch->sw_batch, op, link); + TAILQ_INSERT_TAIL(&ioat_ch->op_pool, op, link); + } + + return 0; +} + static int ioat_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch, spdk_accel_completion_cb cb_fn, void *cb_arg) @@ -449,6 +473,7 @@ static struct spdk_accel_engine ioat_accel_engine = { .fill = ioat_submit_fill, .batch_get_max = ioat_batch_get_max, .batch_create = ioat_batch_create, + .batch_cancel = ioat_batch_cancel, .batch_prep_copy = ioat_batch_prep_copy, .batch_prep_dualcast = ioat_batch_prep_dualcast, .batch_prep_compare = ioat_batch_prep_compare,