lib/idxd: add support for batching dualcast commands

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: Ia5b5a346ae5c8470d9792766dba2bcf68c6fe2fd
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2915
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

include/spdk/idxd.h

@@ -32,7 +32,7 @@
  */
 
 /** \file
- * IDXD accel engine driver public interface
+ * IDXD driver public interface
  */
 
 #ifndef SPDK_IDXD_H
@@ -193,7 +193,28 @@ int spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_bat
 			      void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg);
 
 /**
- * Build and submit a idxd memory copy request.
+ * Synchronous call to prepare a dualcast request into a previously initialized batch
+ *  created with spdk_idxd_batch_create(). The callback will be called when the dualcast
+ *  completes after the batch has been submitted by an asynchronous call to
+ *  spdk_idxd_batch_submit().
+ *
+ * \param chan IDXD channel to submit request.
+ * \param batch Handle provided when the batch was started with spdk_idxd_batch_create().
+ * \param dst1 First destination virtual address (must be 4K aligned).
+ * \param dst2 Second destination virtual address (must be 4K aligned).
+ * \param src Source virtual address.
+ * \param nbytes Number of bytes to copy.
+ * \param cb_fn Callback function which will be called when the request is complete.
+ * \param cb_arg Opaque value which will be passed back as the arg parameter in
+ * the completion callback.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,
+				  void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg);
+
+/**
+ * Build and submit an idxd memory copy request.
  *
  * This function will build the copy descriptor and then immediately submit
  * by writing to the proper device portal.
@@ -213,7 +234,7 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
 			  spdk_idxd_req_cb cb_fn, void *cb_arg);
 
 /**
- * Build and submit an idxd dual cast copy request.
+ * Build and submit an idxd dualcast request.
  *
  * This function will build the dual cast descriptor and then immediately submit
  * by writing to the proper device portal.

lib/idxd/idxd.c

@@ -1005,6 +1005,50 @@ spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_batch *
 	return 0;
 }
 
+int
+spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,
+			      void *dst1, void *dst2, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)
+{
+	struct idxd_hw_desc *desc;
+	struct idxd_comp *comp;
+
+	if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {
+		SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
+		return -EINVAL;
+	}
+
+	if (_does_batch_exist(batch, chan) == false) {
+		SPDK_ERRLOG("Attempt to add to a batch that doesn't exist\n.");
+		return -EINVAL;
+	}
+
+	if ((batch->cur_index - batch->start_index) == DESC_PER_BATCH) {
+		SPDK_ERRLOG("Attempt to add to a batch that is already full\n.");
+		return -ENOMEM;
+	}
+
+	desc = &chan->ring_ctrl.user_desc[batch->cur_index];
+	comp = &chan->ring_ctrl.user_completions[batch->cur_index];
+	SPDK_DEBUGLOG(SPDK_LOG_IDXD, "Prep batch %p index %u\n", batch, batch->cur_index);
+
+	batch->cur_index++;
+	assert(batch->cur_index > batch->start_index);
+
+	desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID | IDXD_FLAG_REQUEST_COMPLETION;
+	desc->opcode = IDXD_OPCODE_DUALCAST;
+	desc->src_addr = (uint64_t)src;
+	desc->dst_addr = (uint64_t)dst1;
+	desc->dest2 = (uint64_t)dst2;
+	desc->xfer_size = nbytes;
+
+	desc->completion_addr = (uint64_t)&comp->hw;
+	comp->cb_arg = cb_arg;
+	comp->cb_fn = cb_fn;
+	comp->batch = batch;
+
+	return 0;
+}
+
 static void
 _dump_error_reg(struct spdk_idxd_io_channel *chan)
 {
@@ -1070,6 +1114,7 @@ _spdk_idxd_process_batch_events(struct spdk_idxd_io_channel *chan)
 					status = comp->hw.result;
 				}
 				break;
+			case IDXD_OPCODE_DUALCAST:
 			case IDXD_OPCODE_MEMMOVE:
 				break;
 			default:

lib/idxd/spdk_idxd.map

@@ -7,6 +7,7 @@
 	spdk_idxd_probe;
 	spdk_idxd_detach;
 	spdk_idxd_batch_prep_copy;
+        spdk_idxd_batch_prep_dualcast;
 	spdk_idxd_batch_submit;
 	spdk_idxd_batch_create;
 	spdk_idxd_batch_get_max;

module/accel/idxd/accel_engine_idxd.c

@@ -47,6 +47,8 @@
 #include "spdk/util.h"
 #include "spdk/json.h"
 
+#define ALIGN_4K 0x1000
+
 static bool g_idxd_enable = false;
 uint32_t g_config_number;
 
@@ -494,12 +496,27 @@ idxd_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel
 					 idxd_done, idxd_task);
 }
 
+static int
+idxd_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *_batch,
+			 void *dst1, void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb)
+{
+	struct idxd_task *idxd_task = (struct idxd_task *)cb_arg;
+	struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
+	struct idxd_batch *batch = (struct idxd_batch *)_batch;
+
+	idxd_task->cb = cb;
+
+	return spdk_idxd_batch_prep_dualcast(chan->chan, batch, dst1, dst2, src, nbytes, idxd_done,
+					     idxd_task);
+}
+
 static struct spdk_accel_engine idxd_accel_engine = {
 	.get_capabilities	= idxd_get_capabilities,
 	.copy			= idxd_submit_copy,
 	.batch_get_max		= idxd_batch_get_max,
 	.batch_create		= idxd_batch_start,
 	.batch_prep_copy	= idxd_batch_prep_copy,
+	.batch_prep_dualcast	= idxd_batch_prep_dualcast,
 	.batch_submit		= idxd_batch_submit,
 	.dualcast		= idxd_submit_dualcast,
 	.compare		= idxd_submit_compare,