diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e5dc85e..3d2fd066c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ A new capability, CRC-32C, was added via `spdk_accel_submit_crc32c`. The software accel engine implemenation has added support for CRC-32C. +### idxd + +IDXD engine support for CRC-32C has been added. + ## v20.04: ### configuration diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index bcfff08b2..0be6dfe12 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -177,6 +177,27 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fill_pattern, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg); +/** + * Build and submit a memory CRC32-C request. + * + * This function will build the CRC-32C descriptor and then immediately submit + * by writing to the proper device portal. + * + * \param chan IDXD channel to submit request. + * \param dst Resulting calculation. + * \param src Source virtual address. + * \param seed Four byte CRC-32C seed value. + * \param nbytes Number of bytes to calculate on. + * \param cb_fn Callback function which will be called when the request is complete. + * \param cb_arg Opaque value which will be passed back as the cb_arg parameter + * in the completion callback. + * + * \return 0 on success, negative errno on failure. + */ +int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *dst, void *src, + uint32_t seed, uint64_t nbytes, + spdk_idxd_req_cb cb_fn, void *cb_arg); + /** * Check for completed requests on an IDXD channel. * diff --git a/include/spdk_internal/idxd.h b/include/spdk_internal/idxd.h index 590c3f96d..17db2405d 100644 --- a/include/spdk_internal/idxd.h +++ b/include/spdk_internal/idxd.h @@ -58,7 +58,7 @@ enum dsa_opcode { IDXD_OPCODE_CR_DELTA = 7, IDXD_OPCODE_AP_DELTA = 8, IDXD_OPCODE_DUALCAST = 9, - IDXD_OPCODE_CRCGEN = 16, + IDXD_OPCODE_CRC32C_GEN = 16, IDXD_OPCODE_COPY_CRC = 17, IDXD_OPCODE_DIF_CHECK = 18, IDXD_OPCODE_DIF_INS = 19, diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index e1985e44a..e0c6cfca0 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -713,6 +713,33 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fil return 0; } +int +spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *dst, void *src, + uint32_t seed, uint64_t nbytes, + spdk_idxd_req_cb cb_fn, void *cb_arg) +{ + struct idxd_hw_desc *desc; + + /* Common prep. */ + desc = _idxd_prep_command(chan, cb_fn, cb_arg); + if (desc == NULL) { + return -EBUSY; + } + + /* Command specific. */ + desc->opcode = IDXD_OPCODE_CRC32C_GEN; + desc->dst_addr = (uintptr_t)dst; + desc->src_addr = (uintptr_t)src; + desc->flags &= IDXD_CLEAR_CRC_FLAGS; + desc->crc32c.seed = seed; + desc->xfer_size = nbytes; + + /* Submit operation. */ + movdir64b((uint64_t *)chan->ring_ctrl.portal, desc); + + return 0; +} + static void _dump_error_reg(struct spdk_idxd_io_channel *chan) { @@ -748,12 +775,20 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) if (spdk_bit_array_get(chan->ring_ctrl.ring_slots, index)) { comp = &chan->ring_ctrl.completions[index]; if (comp->hw.status == 1) { + struct idxd_hw_desc *desc; + sw_error_0 = _idxd_read_8(chan->idxd, IDXD_SWERR_OFFSET); if (sw_error_0 & 0x1) { _dump_error_reg(chan); status = -EINVAL; } + desc = &chan->ring_ctrl.data_desc[index]; + if (desc->opcode == IDXD_OPCODE_CRC32C_GEN) { + *(uint32_t *)desc->dst_addr = comp->hw.crc32c_val; + *(uint32_t *)desc->dst_addr ^= ~0; + } + comp->cb_fn((void *)comp->cb_arg, status); comp->hw.status = status = 0; spdk_bit_array_clear(chan->ring_ctrl.ring_slots, index); diff --git a/lib/idxd/idxd_spec.h b/lib/idxd/idxd_spec.h index 924467c47..51d52cdcc 100644 --- a/lib/idxd/idxd_spec.h +++ b/lib/idxd/idxd_spec.h @@ -74,6 +74,8 @@ extern "C" { #define IDXD_OPCAP_WORDS 0x4 +#define IDXD_CLEAR_CRC_FLAGS 0xFFFFu + #define IDXD_FLAG_FENCE (1 << 0) #define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2) #define IDXD_FLAG_REQUEST_COMPLETION (1 << 3) @@ -227,11 +229,11 @@ struct idxd_hw_desc { } delta; uint32_t delta_rec_size; uint64_t dest2; - struct crc { + struct crc32c { uint32_t seed; uint32_t rsvd; uint64_t addr; - } crc; + } crc32c; struct dif_chk { uint8_t src_flags; uint8_t rsvd1; @@ -277,8 +279,8 @@ struct idxd_hw_comp_record { uint32_t bytes_completed; uint64_t fault_addr; union { - uint16_t delta_rec_size; - uint16_t crc_val; + uint32_t delta_rec_size; + uint32_t crc32c_val; struct { uint32_t dif_chk_ref_tag; uint16_t dif_chk_app_tag_mask; diff --git a/lib/idxd/spdk_idxd.map b/lib/idxd/spdk_idxd.map index c0e3e7bdb..6ffe442cc 100644 --- a/lib/idxd/spdk_idxd.map +++ b/lib/idxd/spdk_idxd.map @@ -7,6 +7,7 @@ spdk_idxd_probe; spdk_idxd_detach; spdk_idxd_set_config; + spdk_idxd_submit_crc32c; spdk_idxd_submit_copy; spdk_idxd_submit_fill; spdk_idxd_process_events; diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index 64d647b8b..727c6bf5b 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -86,6 +86,7 @@ struct idxd_op { spdk_idxd_req_cb cb_fn; void *src; void *dst; + uint32_t seed; uint64_t fill_pattern; uint32_t op_code; uint64_t nbytes; @@ -150,6 +151,10 @@ idxd_poll(void *arg) rc = spdk_idxd_submit_fill(op->chan, op->dst, op->fill_pattern, op->nbytes, op->cb_fn, op->cb_arg); break; + case IDXD_OPCODE_CRC32C_GEN: + rc = spdk_idxd_submit_crc32c(op->chan, op->dst, op->src, op->seed, op->nbytes, + op->cb_fn, op->cb_arg); + break; default: /* Should never get here */ assert(false); @@ -234,6 +239,8 @@ idxd_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, /* Queue the operation. */ TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link); + return 0; + } else if (chan->state == IDXD_CHANNEL_ERROR) { return -EINVAL; } @@ -274,6 +281,49 @@ idxd_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fi /* Queue the operation. */ TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link); + return 0; + + } else if (chan->state == IDXD_CHANNEL_ERROR) { + return -EINVAL; + } + + return rc; +} + +static int +idxd_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void *src, + uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb) +{ + struct idxd_task *idxd_task = (struct idxd_task *)cb_arg; + struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch); + int rc = 0; + + idxd_task->cb = cb; + + if (chan->state == IDXD_CHANNEL_ACTIVE) { + rc = spdk_idxd_submit_crc32c(chan->chan, dst, src, seed, nbytes, idxd_done, idxd_task); + } + + if (chan->state == IDXD_CHANNEL_PAUSED || rc == -EBUSY) { + struct idxd_op *op_to_queue; + + /* Commpom prep. */ + op_to_queue = _prep_queue_command(chan, idxd_done, idxd_task); + if (op_to_queue == NULL) { + return -ENOMEM; + } + + /* Command specific. */ + op_to_queue->dst = dst; + op_to_queue->src = src; + op_to_queue->seed = seed; + op_to_queue->nbytes = nbytes; + op_to_queue->op_code = IDXD_OPCODE_CRC32C_GEN; + + /* Queue the operation. */ + TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link); + return 0; + } else if (chan->state == IDXD_CHANNEL_ERROR) { return -EINVAL; } @@ -284,13 +334,14 @@ idxd_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fi static uint64_t idxd_get_capabilities(void) { - return ACCEL_COPY | ACCEL_FILL; + return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C; } static struct spdk_accel_engine idxd_accel_engine = { .get_capabilities = idxd_get_capabilities, .copy = idxd_submit_copy, .fill = idxd_submit_fill, + .crc32c = idxd_submit_crc32c, .get_io_channel = idxd_get_io_channel, };