From d478edd4ee2029aa11ec03e914b079c7c8a8428d Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Tue, 8 Dec 2020 22:44:35 +0800 Subject: [PATCH] lib/nvmf: Use the async function to caculate the crc32 for header_digest. In this patch, we will use the accel library to do the crc32 work for the header digest when sending the pdu in the target side. For data digest support, will consider in the further patch. Signed-off-by: Ziye Yang Change-Id: I3169a158afd633f48bdbeb2cce1ed20e4141ae45 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5472 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- include/spdk_internal/nvme_tcp.h | 1 + lib/nvmf/tcp.c | 81 +++++++++++++++++++++++-------- mk/spdk.lib_deps.mk | 2 +- test/unit/lib/nvmf/tcp.c/tcp_ut.c | 11 +++++ 4 files changed, 75 insertions(+), 20 deletions(-) diff --git a/include/spdk_internal/nvme_tcp.h b/include/spdk_internal/nvme_tcp.h index ce1b08edc..f49db939a 100644 --- a/include/spdk_internal/nvme_tcp.h +++ b/include/spdk_internal/nvme_tcp.h @@ -101,6 +101,7 @@ struct nvme_tcp_pdu { bool has_hdgst; bool ddgst_enable; + uint32_t header_digest_crc32; uint8_t data_digest[SPDK_NVME_TCP_DIGEST_LEN]; uint8_t ch_valid_bytes; diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index b4929064a..a4d45f1ed 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -31,6 +31,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include "spdk/accel_engine.h" #include "spdk/stdinc.h" #include "spdk/crc32.h" #include "spdk/endian.h" @@ -251,6 +252,8 @@ struct spdk_nvmf_tcp_qpair { */ struct spdk_poller *timeout_poller; + struct spdk_io_channel *accel_channel; + TAILQ_ENTRY(spdk_nvmf_tcp_qpair) link; }; @@ -477,6 +480,9 @@ nvmf_tcp_qpair_destroy(struct spdk_nvmf_tcp_qpair *tqpair) nvmf_tcp_dump_qpair_req_contents(tqpair); } + if (tqpair->accel_channel) { + spdk_put_io_channel(tqpair->accel_channel); + } spdk_dma_free(tqpair->pdus); free(tqpair->reqs); spdk_free(tqpair->bufs); @@ -800,25 +806,12 @@ _pdu_write_done(void *_pdu, int err) } static void -nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, - struct nvme_tcp_pdu *pdu, - nvme_tcp_qpair_xfer_complete_cb cb_fn, - void *cb_arg) +_tcp_write_pdu(struct nvme_tcp_pdu *pdu) { - int hlen; - uint32_t crc32c; uint32_t mapped_length = 0; ssize_t rc; - - assert(&tqpair->pdu_in_progress != pdu); - - hlen = pdu->hdr.common.hlen; - - /* Header Digest */ - if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) { - crc32c = nvme_tcp_pdu_calc_header_digest(pdu); - MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + hlen, crc32c); - } + struct spdk_nvmf_tcp_qpair *tqpair = pdu->qpair; + uint32_t crc32c; /* Data Digest */ if (pdu->data_len > 0 && g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && tqpair->host_ddgst_enable) { @@ -826,9 +819,6 @@ nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, MAKE_DIGEST_WORD(pdu->data_digest, crc32c); } - pdu->cb_fn = cb_fn; - pdu->cb_arg = cb_arg; - pdu->sock_req.iovcnt = nvme_tcp_build_iovs(pdu->iov, SPDK_COUNTOF(pdu->iov), pdu, tqpair->host_hdgst_enable, tqpair->host_ddgst_enable, &mapped_length); @@ -848,6 +838,48 @@ nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, } } +static void +header_crc32_accel_done(void *cb_arg, int status) +{ + struct nvme_tcp_pdu *pdu = cb_arg; + + pdu->header_digest_crc32 = pdu->header_digest_crc32 ^ SPDK_CRC32C_XOR; + MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + pdu->hdr.common.hlen, pdu->header_digest_crc32); + if (spdk_unlikely(status)) { + SPDK_ERRLOG("Failed to finish the crc32 work\n"); + _pdu_write_done(pdu, status); + return; + } + + _tcp_write_pdu(pdu); +} + +static void +nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, + struct nvme_tcp_pdu *pdu, + nvme_tcp_qpair_xfer_complete_cb cb_fn, + void *cb_arg) +{ + int hlen; + + assert(&tqpair->pdu_in_progress != pdu); + + hlen = pdu->hdr.common.hlen; + pdu->cb_fn = cb_fn; + pdu->cb_arg = cb_arg; + pdu->qpair = tqpair; + + /* Header Digest */ + if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) { + spdk_accel_submit_crc32c(tqpair->accel_channel, &pdu->header_digest_crc32, + &pdu->hdr.raw, 0, + hlen, header_crc32_accel_done, pdu); + return; + } + + _tcp_write_pdu(pdu); +} + static int nvmf_tcp_qpair_init_mem_resource(struct spdk_nvmf_tcp_qpair *tqpair) { @@ -1649,6 +1681,17 @@ nvmf_tcp_icreq_handle(struct spdk_nvmf_tcp_transport *ttransport, /* Not fatal. */ } + if (tqpair->host_hdgst_enable) { + tqpair->accel_channel = spdk_accel_engine_get_io_channel(); + if (spdk_unlikely(!tqpair->accel_channel)) { + fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR; + error_offset = offsetof(struct spdk_nvme_tcp_ic_req, dgst); + SPDK_ERRLOG("Unabled to get accel_channel for tqpair=%p, failed to enable header digest\n", + tqpair); + goto end; + } + } + tqpair->cpda = spdk_min(ic_req->hpda, SPDK_NVME_TCP_CPDA_MAX); SPDK_DEBUGLOG(nvmf_tcp, "cpda of tqpair=(%p) is : %u\n", tqpair, tqpair->cpda); diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index 1d7b7d99c..7a70fe43e 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -79,7 +79,7 @@ DEPDIRS-event := log util thread $(JSON_LIBS) trace DEPDIRS-ftl := log util thread trace bdev DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev -DEPDIRS-nvmf := log sock util nvme thread $(JSON_LIBS) trace bdev +DEPDIRS-nvmf := accel log sock util nvme thread $(JSON_LIBS) trace bdev ifeq ($(CONFIG_RDMA),y) DEPDIRS-nvmf += rdma endif diff --git a/test/unit/lib/nvmf/tcp.c/tcp_ut.c b/test/unit/lib/nvmf/tcp.c/tcp_ut.c index b97f1848d..9b06633a0 100644 --- a/test/unit/lib/nvmf/tcp.c/tcp_ut.c +++ b/test/unit/lib/nvmf/tcp.c/tcp_ut.c @@ -222,6 +222,17 @@ DEFINE_STUB(nvmf_transport_req_free, (struct spdk_nvmf_request *req), 0); +DEFINE_STUB(spdk_accel_engine_get_io_channel, + struct spdk_io_channel *, + (void), + NULL); + +DEFINE_STUB(spdk_accel_submit_crc32c, + int, + (struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed, + uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg), + 0); + DEFINE_STUB(spdk_nvmf_bdev_ctrlr_nvme_passthru_admin, int, (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,