From c258d73feb41b80d06c91c89b4114811be124fde Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Fri, 15 Feb 2019 16:24:31 -0700 Subject: [PATCH] ioat: add APIs to only build descriptors Add spdk_ioat_build_copy and spdk_ioat_build_fill which mirror the existing spdk_ioat_submit_copy and spdk_ioat_submit_fill. These new functions *only* build the descriptors in the ring - they do not write the doorbell. This enables batching which can significantly improve performance by reducing the number of MMIO writes. Signed-off-by: Jim Harris Change-Id: Ia3539f936924b7f833f4a7b963d06ffefa68379f Reviewed-on: https://review.gerrithub.io/c/444973 Tested-by: SPDK CI Jenkins Reviewed-by: Darek Stojaczyk Reviewed-by: Shuhei Matsumoto Reviewed-by: Changpeng Liu --- include/spdk/ioat.h | 54 +++++++++++++++++++++++++++++++++++++++++++-- lib/ioat/ioat.c | 36 ++++++++++++++++++++++++++---- 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/include/spdk/ioat.h b/include/spdk/ioat.h index 99a670a2d..fc1750957 100644 --- a/include/spdk/ioat.h +++ b/include/spdk/ioat.h @@ -108,7 +108,32 @@ int spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_ void spdk_ioat_detach(struct spdk_ioat_chan *ioat); /** - * Submit a DMA engine memory copy request. + * Build a DMA engine memory copy request. + * + * This function will build the descriptor in the channel's ring. The + * caller must also explicitly call spdk_ioat_flush to submit the + * descriptor, possibly after building additional descriptors. + * + * \param chan I/OAT channel to build request. + * \param cb_arg Opaque value which will be passed back as the arg parameter in + * the completion callback. + * \param cb_fn Callback function which will be called when the request is complete. + * \param dst Destination virtual address. + * \param src Source virtual address. + * \param nbytes Number of bytes to copy. + * + * \return 0 on success, negative errno on failure. + */ +int spdk_ioat_build_copy(struct spdk_ioat_chan *chan, + void *cb_arg, spdk_ioat_req_cb cb_fn, + void *dst, const void *src, uint64_t nbytes); + +/** + * Build and submit a DMA engine memory copy request. + * + * This function will build the descriptor in the channel's ring and then + * immediately submit it by writing the channel's doorbell. Calling this + * function does not require a subsequent call to spdk_ioat_flush. * * \param chan I/OAT channel to submit request. * \param cb_arg Opaque value which will be passed back as the arg parameter in @@ -125,7 +150,32 @@ int spdk_ioat_submit_copy(struct spdk_ioat_chan *chan, void *dst, const void *src, uint64_t nbytes); /** - * Submit a DMA engine memory fill request. + * Build a DMA engine memory fill request. + * + * This function will build the descriptor in the channel's ring. The + * caller must also explicitly call spdk_ioat_flush to submit the + * descriptor, possibly after building additional descriptors. + * + * \param chan I/OAT channel to build request. + * \param cb_arg Opaque value which will be passed back as the cb_arg parameter + * in the completion callback. + * \param cb_fn Callback function which will be called when the request is complete. + * \param dst Destination virtual address. + * \param fill_pattern Repeating eight-byte pattern to use for memory fill. + * \param nbytes Number of bytes to fill. + * + * \return 0 on success, negative errno on failure. + */ +int spdk_ioat_build_fill(struct spdk_ioat_chan *chan, + void *cb_arg, spdk_ioat_req_cb cb_fn, + void *dst, uint64_t fill_pattern, uint64_t nbytes); + +/** + * Build and submit a DMA engine memory fill request. + * + * This function will build the descriptor in the channel's ring and then + * immediately submit it by writing the channel's doorbell. Calling this + * function does not require a subsequent call to spdk_ioat_flush. * * \param chan I/OAT channel to submit request. * \param cb_arg Opaque value which will be passed back as the cb_arg parameter diff --git a/lib/ioat/ioat.c b/lib/ioat/ioat.c index 7ff1d1271..b987250db 100644 --- a/lib/ioat/ioat.c +++ b/lib/ioat/ioat.c @@ -584,8 +584,8 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat) } int -spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, - void *dst, const void *src, uint64_t nbytes) +spdk_ioat_build_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, + void *dst, const void *src, uint64_t nbytes) { struct ioat_descriptor *last_desc; uint64_t remaining, op_size; @@ -652,13 +652,27 @@ spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_c return -ENOMEM; } + return 0; +} + +int +spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, + void *dst, const void *src, uint64_t nbytes) +{ + int rc; + + rc = spdk_ioat_build_copy(ioat, cb_arg, cb_fn, dst, src, nbytes); + if (rc != 0) { + return rc; + } + spdk_ioat_flush(ioat); return 0; } int -spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, - void *dst, uint64_t fill_pattern, uint64_t nbytes) +spdk_ioat_build_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, + void *dst, uint64_t fill_pattern, uint64_t nbytes) { struct ioat_descriptor *last_desc = NULL; uint64_t remaining, op_size; @@ -709,6 +723,20 @@ spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_c return -ENOMEM; } + return 0; +} + +int +spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, + void *dst, uint64_t fill_pattern, uint64_t nbytes) +{ + int rc; + + rc = spdk_ioat_build_fill(ioat, cb_arg, cb_fn, dst, fill_pattern, nbytes); + if (rc != 0) { + return rc; + } + spdk_ioat_flush(ioat); return 0; }