ioat: add APIs to only build descriptors

Add spdk_ioat_build_copy and spdk_ioat_build_fill
which mirror the existing spdk_ioat_submit_copy
and spdk_ioat_submit_fill.  These new functions
*only* build the descriptors in the ring - they do
not write the doorbell.  This enables batching
which can significantly improve performance by
reducing the number of MMIO writes.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: Ia3539f936924b7f833f4a7b963d06ffefa68379f

Reviewed-on: https://review.gerrithub.io/c/444973
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

include/spdk/ioat.h

@@ -108,7 +108,32 @@ int spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_
 void spdk_ioat_detach(struct spdk_ioat_chan *ioat);
 
 /**
- * Submit a DMA engine memory copy request.
+ * Build a DMA engine memory copy request.
+ *
+ * This function will build the descriptor in the channel's ring.  The
+ * caller must also explicitly call spdk_ioat_flush to submit the
+ * descriptor, possibly after building additional descriptors.
+ *
+ * \param chan I/OAT channel to build request.
+ * \param cb_arg Opaque value which will be passed back as the arg parameter in
+ * the completion callback.
+ * \param cb_fn Callback function which will be called when the request is complete.
+ * \param dst Destination virtual address.
+ * \param src Source virtual address.
+ * \param nbytes Number of bytes to copy.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_ioat_build_copy(struct spdk_ioat_chan *chan,
+			 void *cb_arg, spdk_ioat_req_cb cb_fn,
+			 void *dst, const void *src, uint64_t nbytes);
+
+/**
+ * Build and submit a DMA engine memory copy request.
+ *
+ * This function will build the descriptor in the channel's ring and then
+ * immediately submit it by writing the channel's doorbell.  Calling this
+ * function does not require a subsequent call to spdk_ioat_flush.
  *
  * \param chan I/OAT channel to submit request.
  * \param cb_arg Opaque value which will be passed back as the arg parameter in
@@ -125,7 +150,32 @@ int spdk_ioat_submit_copy(struct spdk_ioat_chan *chan,
 			  void *dst, const void *src, uint64_t nbytes);
 
 /**
- * Submit a DMA engine memory fill request.
+ * Build a DMA engine memory fill request.
+ *
+ * This function will build the descriptor in the channel's ring.  The
+ * caller must also explicitly call spdk_ioat_flush to submit the
+ * descriptor, possibly after building additional descriptors.
+ *
+ * \param chan I/OAT channel to build request.
+ * \param cb_arg Opaque value which will be passed back as the cb_arg parameter
+ * in the completion callback.
+ * \param cb_fn Callback function which will be called when the request is complete.
+ * \param dst Destination virtual address.
+ * \param fill_pattern Repeating eight-byte pattern to use for memory fill.
+ * \param nbytes Number of bytes to fill.
+ *
+ * \return 0 on success, negative errno on failure.
+ */
+int spdk_ioat_build_fill(struct spdk_ioat_chan *chan,
+			 void *cb_arg, spdk_ioat_req_cb cb_fn,
+			 void *dst, uint64_t fill_pattern, uint64_t nbytes);
+
+/**
+ * Build and submit a DMA engine memory fill request.
+ *
+ * This function will build the descriptor in the channel's ring and then
+ * immediately submit it by writing the channel's doorbell.  Calling this
+ * function does not require a subsequent call to spdk_ioat_flush.
  *
  * \param chan I/OAT channel to submit request.
  * \param cb_arg Opaque value which will be passed back as the cb_arg parameter

lib/ioat/ioat.c

@@ -584,7 +584,7 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat)
 }
 
 int
-spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
+spdk_ioat_build_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
 		     void *dst, const void *src, uint64_t nbytes)
 {
 	struct ioat_descriptor	*last_desc;
@@ -652,12 +652,26 @@ spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_c
 		return -ENOMEM;
 	}
 
+	return 0;
+}
+
+int
+spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
+		      void *dst, const void *src, uint64_t nbytes)
+{
+	int rc;
+
+	rc = spdk_ioat_build_copy(ioat, cb_arg, cb_fn, dst, src, nbytes);
+	if (rc != 0) {
+		return rc;
+	}
+
 	spdk_ioat_flush(ioat);
 	return 0;
 }
 
 int
-spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
+spdk_ioat_build_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
 		     void *dst, uint64_t fill_pattern, uint64_t nbytes)
 {
 	struct ioat_descriptor	*last_desc = NULL;
@@ -709,6 +723,20 @@ spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_c
 		return -ENOMEM;
 	}
 
+	return 0;
+}
+
+int
+spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
+		      void *dst, uint64_t fill_pattern, uint64_t nbytes)
+{
+	int rc;
+
+	rc = spdk_ioat_build_fill(ioat, cb_arg, cb_fn, dst, fill_pattern, nbytes);
+	if (rc != 0) {
+		return rc;
+	}
+
 	spdk_ioat_flush(ioat);
 	return 0;
 }