From be34c31e16b1ae67baf2fe2008f9c887790a705c Mon Sep 17 00:00:00 2001 From: paul luse Date: Wed, 1 Jul 2020 14:50:01 -0400 Subject: [PATCH] module/accel/ioat: add batching suport for sw operations This patch has the basic infrastructure to support the accel framework batching API but only for commands not HW accelerated by IOAT, that will come in the next patch... Signed-off-by: paul luse Change-Id: I6168831ac5698a9e58a81ef35ce919d75a72d0f5 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3153 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker Community-CI: Mellanox Build Bot --- mk/spdk.lib_deps.mk | 2 +- module/accel/ioat/accel_engine_ioat.c | 276 +++++++++++++++++++++++++- 2 files changed, 270 insertions(+), 8 deletions(-) diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index d9bcaff6e..be57bb220 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -110,7 +110,7 @@ DEPDIRS-blob_bdev := log thread bdev DEPDIRS-blobfs_bdev := $(BDEV_DEPS_THREAD) blob_bdev blobfs # module/accel -DEPDIRS-accel_ioat := log ioat conf thread $(JSON_LIBS) accel +DEPDIRS-accel_ioat := log ioat conf thread $(JSON_LIBS) accel util DEPDIRS-accel_idxd := log idxd thread $(JSON_LIBS) accel # module/env_dpdk diff --git a/module/accel/ioat/accel_engine_ioat.c b/module/accel/ioat/accel_engine_ioat.c index 41f8cc94a..d4c7cbccd 100644 --- a/module/accel/ioat/accel_engine_ioat.c +++ b/module/accel/ioat/accel_engine_ioat.c @@ -43,7 +43,36 @@ #include "spdk/event.h" #include "spdk/thread.h" #include "spdk/ioat.h" +#include "spdk/crc32.h" +#define ALIGN_4K 0x1000 + +enum ioat_accel_opcode { + IOAT_ACCEL_OPCODE_MEMMOVE = 0, + IOAT_ACCEL_OPCODE_MEMFILL = 1, + IOAT_ACCEL_OPCODE_COMPARE = 2, + IOAT_ACCEL_OPCODE_CRC32C = 3, + IOAT_ACCEL_OPCODE_DUALCAST = 4, +}; + +struct ioat_accel_op { + struct ioat_io_channel *ioat_ch; + void *cb_arg; + spdk_accel_completion_cb cb_fn; + void *src; + union { + void *dst; + void *src2; + }; + void *dst2; + uint32_t seed; + uint64_t fill_pattern; + enum ioat_accel_opcode op_code; + uint64_t nbytes; + TAILQ_ENTRY(ioat_accel_op) link; +}; + +static int g_batch_size; static bool g_ioat_enable = false; static bool g_ioat_initialized = false; @@ -71,11 +100,13 @@ static pthread_mutex_t g_ioat_mutex = PTHREAD_MUTEX_INITIALIZER; static TAILQ_HEAD(, pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices); - struct ioat_io_channel { - struct spdk_ioat_chan *ioat_ch; - struct ioat_device *ioat_dev; - struct spdk_poller *poller; + struct spdk_ioat_chan *ioat_ch; + struct ioat_device *ioat_dev; + struct spdk_poller *poller; + TAILQ_HEAD(, ioat_accel_op) op_pool; + TAILQ_HEAD(, ioat_accel_op) sw_batch; /* for operations not hw accelerated */ + bool hw_batch; /* for operations that are hw accelerated */ }; static int @@ -191,19 +222,224 @@ ioat_poll(void *arg) static struct spdk_io_channel *ioat_get_io_channel(void); /* - * The IOAT engine has more capabilities than this but these are - * the only ones we expose via the accel engine. + * The IOAT engine only supports these capabilities as hardware + * accelerated. The accel fw will handle unsupported functions + * by calling the software implementations of the functions. */ static uint64_t ioat_get_capabilities(void) { - return ACCEL_COPY | ACCEL_FILL; + return ACCEL_COPY | ACCEL_FILL | ACCEL_BATCH; +} + +/* The IOAT batch functions exposed by the accel fw do not match up 1:1 + * with the functions in the IOAT library. The IOAT library directly only + * supports construction of accelerated functions via the IOAT native + * interface. The accel_fw batch capabilities are implemented here in the + * plug-in and rely on either the IOAT library for accelerated commands + * or software functions for non-accelerated. + */ +static uint32_t +ioat_batch_get_max(void) +{ + return g_batch_size; +} + +static struct spdk_accel_batch * +ioat_batch_create(struct spdk_io_channel *ch) +{ + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + + if (!TAILQ_EMPTY(&ioat_ch->sw_batch) || (ioat_ch->hw_batch == true)) { + SPDK_ERRLOG("IOAT accel engine only supports one batch at a time.\n"); + return NULL; + } + + return (struct spdk_accel_batch *)&ioat_ch->hw_batch; +} + +static struct ioat_accel_op * +_prep_op(void *cb_arg, struct ioat_io_channel *ioat_ch, struct spdk_accel_batch *batch, + spdk_accel_completion_cb cb) +{ + struct ioat_accel_op *op; + + if ((struct spdk_accel_batch *)&ioat_ch->hw_batch != batch) { + SPDK_ERRLOG("Invalid batch\n"); + return NULL; + } + + if (!TAILQ_EMPTY(&ioat_ch->op_pool)) { + op = TAILQ_FIRST(&ioat_ch->op_pool); + TAILQ_REMOVE(&ioat_ch->op_pool, op, link); + } else { + SPDK_ERRLOG("Ran out of operations for batch\n"); + return NULL; + } + + op->cb_arg = cb_arg; + op->cb_fn = cb; + op->ioat_ch = ioat_ch; + + return op; +} + +static int +ioat_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, + void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb) +{ + /* TODO - HW ACCELERATED */ + return 0;; +} + +static int +ioat_batch_prep_fill(void *cb_arg, struct spdk_io_channel *ch, + struct spdk_accel_batch *batch, void *dst, uint8_t fill, + uint64_t nbytes, spdk_accel_completion_cb cb) +{ + /* TODO - HW ACCELERATED */ + return 0; +} + +static int +ioat_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch, + struct spdk_accel_batch *batch, void *dst1, void *dst2, + void *src, uint64_t nbytes, spdk_accel_completion_cb cb) +{ + struct ioat_accel_op *op; + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + + if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { + SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); + return -EINVAL; + } + + op = _prep_op(cb_arg, ioat_ch, batch, cb); + if (op == NULL) { + return -EINVAL; + } + + /* Command specific. */ + op->src = src; + op->dst = dst1; + op->dst2 = dst2; + op->nbytes = nbytes; + op->op_code = IOAT_ACCEL_OPCODE_DUALCAST; + TAILQ_INSERT_TAIL(&ioat_ch->sw_batch, op, link); + + return 0; +} + +static int +ioat_batch_prep_compare(void *cb_arg, struct spdk_io_channel *ch, + struct spdk_accel_batch *batch, void *src1, + void *src2, uint64_t nbytes, spdk_accel_completion_cb cb) +{ + struct ioat_accel_op *op; + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + + op = _prep_op(cb_arg, ioat_ch, batch, cb); + if (op == NULL) { + return -EINVAL; + } + + /* Command specific. */ + op->src = src1; + op->src2 = src2; + op->nbytes = nbytes; + op->op_code = IOAT_ACCEL_OPCODE_COMPARE; + TAILQ_INSERT_TAIL(&ioat_ch->sw_batch, op, link); + + return 0; +} + +static int +ioat_batch_prep_crc32c(void *cb_arg, struct spdk_io_channel *ch, + struct spdk_accel_batch *batch, uint32_t *dst, void *src, + uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb) +{ + struct ioat_accel_op *op; + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + + op = _prep_op(cb_arg, ioat_ch, batch, cb); + if (op == NULL) { + return -EINVAL; + } + + /* Command specific. */ + op->dst = (void *)dst; + op->src = src; + op->seed = seed; + op->nbytes = nbytes; + op->op_code = IOAT_ACCEL_OPCODE_CRC32C; + TAILQ_INSERT_TAIL(&ioat_ch->sw_batch, op, link); + + return 0; +} + +static int +ioat_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch, + spdk_accel_completion_cb cb) +{ + struct ioat_accel_op *op; + struct ioat_io_channel *ioat_ch = spdk_io_channel_get_ctx(ch); + struct spdk_accel_task *accel_req; + int batch_status = 0, cmd_status = 0; + + if ((struct spdk_accel_batch *)&ioat_ch->hw_batch != batch) { + SPDK_ERRLOG("Invalid batch\n"); + return -EINVAL; + } + + /* TODO submit the batched HW items first. */ + + /* Complete the batched software items. */ + while ((op = TAILQ_FIRST(&ioat_ch->sw_batch))) { + TAILQ_REMOVE(&ioat_ch->sw_batch, op, link); + accel_req = (struct spdk_accel_task *)((uintptr_t)op->cb_arg - + offsetof(struct spdk_accel_task, offload_ctx)); + + switch (op->op_code) { + case IOAT_ACCEL_OPCODE_DUALCAST: + memcpy(op->dst, op->src, op->nbytes); + memcpy(op->dst2, op->src, op->nbytes); + break; + case IOAT_ACCEL_OPCODE_COMPARE: + cmd_status = memcmp(op->src, op->src2, op->nbytes); + break; + case IOAT_ACCEL_OPCODE_CRC32C: + *(uint32_t *)op->dst = spdk_crc32c_update(op->src, op->nbytes, ~op->seed); + break; + default: + assert(false); + break; + } + + batch_status |= cmd_status; + op->cb_fn(accel_req, cmd_status); + TAILQ_INSERT_TAIL(&ioat_ch->op_pool, op, link); + } + + /* Now complete the batch request itself. */ + accel_req = (struct spdk_accel_task *)((uintptr_t)cb_arg - + offsetof(struct spdk_accel_task, offload_ctx)); + cb(accel_req, batch_status); + + return 0; } static struct spdk_accel_engine ioat_accel_engine = { .get_capabilities = ioat_get_capabilities, .copy = ioat_submit_copy, .fill = ioat_submit_fill, + .batch_get_max = ioat_batch_get_max, + .batch_create = ioat_batch_create, + .batch_prep_copy = ioat_batch_prep_copy, + .batch_prep_dualcast = ioat_batch_prep_dualcast, + .batch_prep_compare = ioat_batch_prep_compare, + .batch_prep_fill = ioat_batch_prep_fill, + .batch_prep_crc32c = ioat_batch_prep_crc32c, + .batch_submit = ioat_batch_submit, .get_io_channel = ioat_get_io_channel, }; @@ -212,12 +448,32 @@ ioat_create_cb(void *io_device, void *ctx_buf) { struct ioat_io_channel *ch = ctx_buf; struct ioat_device *ioat_dev; + struct ioat_accel_op *op; + int i; ioat_dev = ioat_allocate_device(); if (ioat_dev == NULL) { return -1; } + TAILQ_INIT(&ch->sw_batch); + ch->hw_batch = false; + TAILQ_INIT(&ch->op_pool); + + g_batch_size = spdk_ioat_get_max_descriptors(ioat_dev->ioat); + for (i = 0 ; i < g_batch_size ; i++) { + op = calloc(1, sizeof(struct ioat_accel_op)); + if (op == NULL) { + SPDK_ERRLOG("Failed to allocate operation for batch.\n"); + while ((op = TAILQ_FIRST(&ch->op_pool))) { + TAILQ_REMOVE(&ch->op_pool, op, link); + free(op); + } + return -ENOMEM; + } + TAILQ_INSERT_TAIL(&ch->op_pool, op, link); + } + ch->ioat_dev = ioat_dev; ch->ioat_ch = ioat_dev->ioat; ch->poller = SPDK_POLLER_REGISTER(ioat_poll, ch->ioat_ch, 0); @@ -228,6 +484,12 @@ static void ioat_destroy_cb(void *io_device, void *ctx_buf) { struct ioat_io_channel *ch = ctx_buf; + struct ioat_accel_op *op; + + while ((op = TAILQ_FIRST(&ch->op_pool))) { + TAILQ_REMOVE(&ch->op_pool, op, link); + free(op); + } ioat_free_device(ch->ioat_dev); spdk_poller_unregister(&ch->poller);