lib/idxd: add support for dual cast

Dual-cast copies the same source to two separate destination buffers.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: Icadae34a75c35e2db672a193287b147416012a5a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2129
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Community-CI: Mellanox Build Bot

doc/idxd.md

@@ -10,6 +10,9 @@ Function                                | Description
 --------------------------------------- | -----------
 spdk_idxd_probe()                       | @copybrief spdk_idxd_probe()
 spdk_idxd_submit_copy()                 | @copybrief spdk_idxd_submit_copy()
+spdk_idxd_submit_compare()              | @copybrief spdk_idxd_submit_compare()
+spdk_idxd_submit_crc32c()               | @copybrief spdk_idxd_submit_crc32c()
+spdk_idxd_submit_dualcast               | @copybrief spdk_idxd_submit_dualcast()
 spdk_idxd_submit_fill()                 | @copybrief spdk_idxd_submit_fill()
 
 # Pre-defined configurations {#idxd_configs}

include/spdk/idxd.h

@@ -157,6 +157,27 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
 			  void *dst, const void *src, uint64_t nbytes,
 			  spdk_idxd_req_cb cb_fn, void *cb_arg);
 
+/**
+ * Build and submit an accel engine dual cast copy request.
+ *
+ * This function will build the dual cast descriptor and then immediately submit
+ * by writing to the proper device portal.
+ *
+ * \param chan IDXD channel to submit request.
+ * \param dst1 First destination virtual address (must be 4K aligned).
+ * \param dst2 Second destination virtual address (must be 4K aligned).
+ * \param src Source virtual address.
+ * \param nbytes Number of bytes to copy.
+ * \param cb_fn Callback function which will be called when the request is complete.
+ * \param cb_arg Opaque value which will be passed back as the arg parameter in
+ * the completion callback.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan,
+			      void *dst1, void *dst2, const void *src, uint64_t nbytes,
+			      spdk_idxd_req_cb cb_fn, void *cb_arg);
+
 /**
  * Build and submit a memory compare request.
  *

include/spdk_internal/accel_engine.h

@@ -48,6 +48,8 @@ struct spdk_accel_engine {
 	uint64_t (*get_capabilities)(void);
 	int (*copy)(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src,
 		    uint64_t nbytes, spdk_accel_completion_cb cb);
+	int (*dualcast)(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
+			uint64_t nbytes, spdk_accel_completion_cb cb);
 	int (*compare)(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2,
 		       uint64_t nbytes, spdk_accel_completion_cb cb);
 	int (*fill)(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill,

lib/idxd/idxd.c

@@ -43,6 +43,8 @@
 
 #include "idxd.h"
 
+#define ALIGN_4K 0x1000
+
 pthread_mutex_t	g_driver_lock = PTHREAD_MUTEX_INITIALIZER;
 
 /*
@@ -688,6 +690,37 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *
 	return 0;
 }
 
+/* Dual-cast copies the same source to two separate destination buffers. */
+int
+spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2,
+			  const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
+{
+	struct idxd_hw_desc *desc;
+
+	if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
+		SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
+		return -EINVAL;
+	}
+
+	/* Common prep. */
+	desc = _idxd_prep_command(chan, cb_fn, cb_arg);
+	if (desc == NULL) {
+		return -EBUSY;
+	}
+
+	/* Command specific. */
+	desc->opcode = IDXD_OPCODE_DUALCAST;
+	desc->src_addr = (uintptr_t)src;
+	desc->dst_addr = (uintptr_t)dst1;
+	desc->dest2 = (uintptr_t)dst2;
+	desc->xfer_size = nbytes;
+
+	/* Submit operation. */
+	movdir64b((uint64_t *)chan->ring_ctrl.portal, desc);
+
+	return 0;
+}
+
 int
 spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, void *src1, const void *src2,
 			 uint64_t nbytes,

lib/idxd/spdk_idxd.map

@@ -10,6 +10,7 @@
 	spdk_idxd_submit_compare;
 	spdk_idxd_submit_crc32c;
 	spdk_idxd_submit_copy;
+	spdk_idxd_submit_dualcast;
 	spdk_idxd_submit_fill;
 	spdk_idxd_process_events;
 	spdk_idxd_get_channel;

module/accel/idxd/accel_engine_idxd.c

@@ -89,6 +89,7 @@ struct idxd_op {
 		void			*dst;
 		void			*src2;
 	};
+	void				*dst2;
 	uint32_t			seed;
 	uint64_t			fill_pattern;
 	uint32_t			op_code;
@@ -150,6 +151,10 @@ idxd_poll(void *arg)
 			rc = spdk_idxd_submit_copy(op->chan, op->dst, op->src, op->nbytes,
 						   op->cb_fn, op->cb_arg);
 			break;
+		case IDXD_OPCODE_DUALCAST:
+			rc = spdk_idxd_submit_dualcast(op->chan, op->dst, op->dst2, op->src, op->nbytes,
+						       op->cb_fn, op->cb_arg);
+			break;
 		case IDXD_OPCODE_COMPARE:
 			rc = spdk_idxd_submit_compare(op->chan, op->src, op->src2, op->nbytes,
 						      op->cb_fn, op->cb_arg);
@@ -255,6 +260,48 @@ idxd_submit_copy(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src,
 	return rc;
 }
 
+static int
+idxd_submit_dualcast(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
+		     uint64_t nbytes,
+		     spdk_accel_completion_cb cb)
+{
+	struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
+	struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
+	int rc = 0;
+
+	idxd_task->cb = cb;
+
+	if (chan->state == IDXD_CHANNEL_ACTIVE) {
+		rc = spdk_idxd_submit_dualcast(chan->chan, dst1, dst2, src, nbytes, idxd_done, idxd_task);
+	}
+
+	if (chan->state == IDXD_CHANNEL_PAUSED || rc == -EBUSY) {
+		struct idxd_op *op_to_queue;
+
+		/* Commpom prep. */
+		op_to_queue = _prep_queue_command(chan, idxd_done, idxd_task);
+		if (op_to_queue == NULL) {
+			return -ENOMEM;
+		}
+
+		/* Command specific. */
+		op_to_queue->dst = dst1;
+		op_to_queue->dst2 = dst2;
+		op_to_queue->src = src;
+		op_to_queue->nbytes = nbytes;
+		op_to_queue->op_code = IDXD_OPCODE_DUALCAST;
+
+		/* Queue the operation. */
+		TAILQ_INSERT_TAIL(&chan->queued_ops, op_to_queue, link);
+		return 0;
+
+	} else if (chan->state == IDXD_CHANNEL_ERROR) {
+		return -EINVAL;
+	}
+
+	return rc;
+}
+
 static int
 idxd_submit_compare(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2,
 		    uint64_t nbytes,
@@ -382,12 +429,14 @@ idxd_submit_crc32c(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void
 static uint64_t
 idxd_get_capabilities(void)
 {
-	return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C | ACCEL_COMPARE;
+	return ACCEL_COPY | ACCEL_FILL | ACCEL_CRC32C | ACCEL_COMPARE |
+	       ACCEL_DUALCAST;
 }
 
 static struct spdk_accel_engine idxd_accel_engine = {
 	.get_capabilities	= idxd_get_capabilities,
 	.copy			= idxd_submit_copy,
+	.dualcast		= idxd_submit_dualcast,
 	.compare		= idxd_submit_compare,
 	.fill			= idxd_submit_fill,
 	.crc32c			= idxd_submit_crc32c,