lib/idxd: implement idxd back end for CRC

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: Ib23c23f69d8f002023dd72be1b7369e50ac44fb2
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2105
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

CHANGELOG.md

@@ -11,6 +11,10 @@ A new capability, CRC-32C, was added via `spdk_accel_submit_crc32c`.
 
 The software accel engine implemenation has added support for CRC-32C.
 
+### idxd
+
+IDXD engine support for CRC-32C has been added.
+
 ## v20.04:
 
 ### configuration

include/spdk/idxd.h

@@ -177,6 +177,27 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan,
 			  void *dst, uint64_t fill_pattern, uint64_t nbytes,
 			  spdk_idxd_req_cb cb_fn, void *cb_arg);
 
+/**
+ * Build and submit a memory CRC32-C request.
+ *
+ * This function will build the CRC-32C descriptor and then immediately submit
+ * by writing to the proper device portal.
+ *
+ * \param chan IDXD channel to submit request.
+ * \param dst Resulting calculation.
+ * \param src Source virtual address.
+ * \param seed Four byte CRC-32C seed value.
+ * \param nbytes Number of bytes to calculate on.
+ * \param cb_fn Callback function which will be called when the request is complete.
+ * \param cb_arg Opaque value which will be passed back as the cb_arg parameter
+ * in the completion callback.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *dst, void *src,
+			    uint32_t seed, uint64_t nbytes,
+			    spdk_idxd_req_cb cb_fn, void *cb_arg);
+
 /**
  * Check for completed requests on an IDXD channel.
  *

include/spdk_internal/idxd.h

@@ -58,7 +58,7 @@ enum dsa_opcode {
 	IDXD_OPCODE_CR_DELTA	= 7,
 	IDXD_OPCODE_AP_DELTA	= 8,
 	IDXD_OPCODE_DUALCAST	= 9,
-	IDXD_OPCODE_CRCGEN	= 16,
+	IDXD_OPCODE_CRC32C_GEN	= 16,
 	IDXD_OPCODE_COPY_CRC	= 17,
 	IDXD_OPCODE_DIF_CHECK	= 18,
 	IDXD_OPCODE_DIF_INS	= 19,

lib/idxd/idxd.c

@@ -713,6 +713,33 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fil
 	return 0;
 }
 
+int
+spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *dst, void *src,
+			uint32_t seed, uint64_t nbytes,
+			spdk_idxd_req_cb cb_fn, void *cb_arg)
+{
+	struct idxd_hw_desc *desc;
+
+	/* Common prep. */
+	desc = _idxd_prep_command(chan, cb_fn, cb_arg);
+	if (desc == NULL) {
+		return -EBUSY;
+	}
+
+	/* Command specific. */
+	desc->opcode = IDXD_OPCODE_CRC32C_GEN;
+	desc->dst_addr = (uintptr_t)dst;
+	desc->src_addr = (uintptr_t)src;
+	desc->flags &= IDXD_CLEAR_CRC_FLAGS;
+	desc->crc32c.seed = seed;
+	desc->xfer_size = nbytes;
+
+	/* Submit operation. */
+	movdir64b((uint64_t *)chan->ring_ctrl.portal, desc);
+
+	return 0;
+}
+
 static void
 _dump_error_reg(struct spdk_idxd_io_channel *chan)
 {
@@ -748,12 +775,20 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan)
 		if (spdk_bit_array_get(chan->ring_ctrl.ring_slots, index)) {
 			comp = &chan->ring_ctrl.completions[index];
 			if (comp->hw.status == 1) {
+				struct idxd_hw_desc *desc;
+
 				sw_error_0 = _idxd_read_8(chan->idxd, IDXD_SWERR_OFFSET);
 				if (sw_error_0 & 0x1) {
 					_dump_error_reg(chan);
 					status = -EINVAL;
 				}
 
+				desc = &chan->ring_ctrl.data_desc[index];
+				if (desc->opcode == IDXD_OPCODE_CRC32C_GEN) {
+					*(uint32_t *)desc->dst_addr = comp->hw.crc32c_val;
+					*(uint32_t *)desc->dst_addr ^= ~0;
+				}
+
 				comp->cb_fn((void *)comp->cb_arg, status);
 				comp->hw.status = status = 0;
 				spdk_bit_array_clear(chan->ring_ctrl.ring_slots, index);

lib/idxd/idxd_spec.h

@@ -74,6 +74,8 @@ extern "C" {
 
 #define IDXD_OPCAP_WORDS		0x4
 
+#define IDXD_CLEAR_CRC_FLAGS		0xFFFFu
+
 #define IDXD_FLAG_FENCE                 (1 << 0)
 #define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2)
 #define IDXD_FLAG_REQUEST_COMPLETION    (1 << 3)
@@ -227,11 +229,11 @@ struct idxd_hw_desc {
 		} delta;
 		uint32_t	delta_rec_size;
 		uint64_t	dest2;
-		struct crc {
+		struct crc32c {
 			uint32_t	seed;
 			uint32_t	rsvd;
 			uint64_t	addr;
-		} crc;
+		} crc32c;
 		struct dif_chk {
 			uint8_t		src_flags;
 			uint8_t		rsvd1;
@@ -277,8 +279,8 @@ struct idxd_hw_comp_record {
 	uint32_t		bytes_completed;
 	uint64_t		fault_addr;
 	union {
-		uint16_t	delta_rec_size;
+		uint32_t	delta_rec_size;
-		uint16_t	crc_val;
+		uint32_t	crc32c_val;
 		struct {
 			uint32_t	dif_chk_ref_tag;
 			uint16_t	dif_chk_app_tag_mask;

lib/idxd/spdk_idxd.map

@@ -7,6 +7,7 @@
 	spdk_idxd_probe;
 	spdk_idxd_detach;
 	spdk_idxd_set_config;
+	spdk_idxd_submit_crc32c;
 	spdk_idxd_submit_copy;
 	spdk_idxd_submit_fill;
 	spdk_idxd_process_events;

module/accel/idxd/accel_engine_idxd.c

@@ -86,6 +86,7 @@ struct idxd_op {
 	spdk_idxd_req_cb		cb_fn;
 	void				*src;
 	void				*dst;
+	uint32_t			seed;
 	uint64_t			fill_pattern;
 	uint32_t			op_code;
 	uint64_t			nbytes;
@@ -150,6 +151,10 @@ idxd_poll(void *arg)
 			rc = spdk_idxd_submit_fill(op->chan, op->dst, op->fill_pattern, op->nbytes,
 						   op->cb_fn, op->cb_arg);
 			break;
+		case IDXD_OPCODE_CRC32C_GEN:
+			rc = spdk_idxd_submit_crc32c(op->chan, op->dst, op->src, op->seed, op->nbytes,
+						     op->cb_fn, op->cb_arg);
+			break;
 		default:
 			/* Should never get here */
 			assert(false);
@@ -234,6 +239,8 @@ idxd_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src,
 
 		/* Queue the operation. */
 		TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link);
+		return 0;
+
 	} else if (chan->state == IDXD_CHANNEL_ERROR) {
 		return -EINVAL;
 	}
@@ -274,6 +281,49 @@ idxd_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fi
 
 		/* Queue the operation. */
 		TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link);
+		return 0;
+
+	} else if (chan->state == IDXD_CHANNEL_ERROR) {
+		return -EINVAL;
+	}
+
+	return rc;
+}
+
+static int
+idxd_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void *src,
+		   uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb)
+{
+	struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
+	struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
+	int rc = 0;
+
+	idxd_task->cb = cb;
+
+	if (chan->state == IDXD_CHANNEL_ACTIVE) {
+		rc = spdk_idxd_submit_crc32c(chan->chan, dst, src, seed, nbytes, idxd_done, idxd_task);
+	}
+
+	if (chan->state == IDXD_CHANNEL_PAUSED || rc == -EBUSY) {
+		struct idxd_op *op_to_queue;
+
+		/* Commpom prep. */
+		op_to_queue = _prep_queue_command(chan, idxd_done, idxd_task);
+		if (op_to_queue == NULL) {
+			return -ENOMEM;
+		}
+
+		/* Command specific. */
+		op_to_queue->dst = dst;
+		op_to_queue->src = src;
+		op_to_queue->seed = seed;
+		op_to_queue->nbytes = nbytes;
+		op_to_queue->op_code = IDXD_OPCODE_CRC32C_GEN;
+
+		/* Queue the operation. */
+		TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link);
+		return 0;
+
 	} else if (chan->state == IDXD_CHANNEL_ERROR) {
 		return -EINVAL;
 	}
@@ -284,13 +334,14 @@ idxd_submit_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fi
 static uint64_t
 idxd_get_capabilities(void)
 {
-	return ACCEL_COPY | ACCEL_FILL;
+	return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C;
 }
 
 static struct spdk_accel_engine idxd_accel_engine = {
 	.get_capabilities	= idxd_get_capabilities,
 	.copy			= idxd_submit_copy,
 	.fill			= idxd_submit_fill,
+	.crc32c			= idxd_submit_crc32c,
 	.get_io_channel		= idxd_get_io_channel,
 };