diff --git a/include/spdk_internal/nvme_tcp.h b/include/spdk_internal/nvme_tcp.h index ce1b08edc..f49db939a 100644 --- a/include/spdk_internal/nvme_tcp.h +++ b/include/spdk_internal/nvme_tcp.h @@ -101,6 +101,7 @@ struct nvme_tcp_pdu { bool has_hdgst; bool ddgst_enable; + uint32_t header_digest_crc32; uint8_t data_digest[SPDK_NVME_TCP_DIGEST_LEN]; uint8_t ch_valid_bytes; diff --git a/lib/nvmf/tcp.c b/lib/nvmf/tcp.c index b4929064a..a4d45f1ed 100644 --- a/lib/nvmf/tcp.c +++ b/lib/nvmf/tcp.c @@ -31,6 +31,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include "spdk/accel_engine.h" #include "spdk/stdinc.h" #include "spdk/crc32.h" #include "spdk/endian.h" @@ -251,6 +252,8 @@ struct spdk_nvmf_tcp_qpair { */ struct spdk_poller *timeout_poller; + struct spdk_io_channel *accel_channel; + TAILQ_ENTRY(spdk_nvmf_tcp_qpair) link; }; @@ -477,6 +480,9 @@ nvmf_tcp_qpair_destroy(struct spdk_nvmf_tcp_qpair *tqpair) nvmf_tcp_dump_qpair_req_contents(tqpair); } + if (tqpair->accel_channel) { + spdk_put_io_channel(tqpair->accel_channel); + } spdk_dma_free(tqpair->pdus); free(tqpair->reqs); spdk_free(tqpair->bufs); @@ -800,25 +806,12 @@ _pdu_write_done(void *_pdu, int err) } static void -nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, - struct nvme_tcp_pdu *pdu, - nvme_tcp_qpair_xfer_complete_cb cb_fn, - void *cb_arg) +_tcp_write_pdu(struct nvme_tcp_pdu *pdu) { - int hlen; - uint32_t crc32c; uint32_t mapped_length = 0; ssize_t rc; - - assert(&tqpair->pdu_in_progress != pdu); - - hlen = pdu->hdr.common.hlen; - - /* Header Digest */ - if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) { - crc32c = nvme_tcp_pdu_calc_header_digest(pdu); - MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + hlen, crc32c); - } + struct spdk_nvmf_tcp_qpair *tqpair = pdu->qpair; + uint32_t crc32c; /* Data Digest */ if (pdu->data_len > 0 && g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && tqpair->host_ddgst_enable) { @@ -826,9 +819,6 @@ nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, MAKE_DIGEST_WORD(pdu->data_digest, crc32c); } - pdu->cb_fn = cb_fn; - pdu->cb_arg = cb_arg; - pdu->sock_req.iovcnt = nvme_tcp_build_iovs(pdu->iov, SPDK_COUNTOF(pdu->iov), pdu, tqpair->host_hdgst_enable, tqpair->host_ddgst_enable, &mapped_length); @@ -848,6 +838,48 @@ nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, } } +static void +header_crc32_accel_done(void *cb_arg, int status) +{ + struct nvme_tcp_pdu *pdu = cb_arg; + + pdu->header_digest_crc32 = pdu->header_digest_crc32 ^ SPDK_CRC32C_XOR; + MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + pdu->hdr.common.hlen, pdu->header_digest_crc32); + if (spdk_unlikely(status)) { + SPDK_ERRLOG("Failed to finish the crc32 work\n"); + _pdu_write_done(pdu, status); + return; + } + + _tcp_write_pdu(pdu); +} + +static void +nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair, + struct nvme_tcp_pdu *pdu, + nvme_tcp_qpair_xfer_complete_cb cb_fn, + void *cb_arg) +{ + int hlen; + + assert(&tqpair->pdu_in_progress != pdu); + + hlen = pdu->hdr.common.hlen; + pdu->cb_fn = cb_fn; + pdu->cb_arg = cb_arg; + pdu->qpair = tqpair; + + /* Header Digest */ + if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) { + spdk_accel_submit_crc32c(tqpair->accel_channel, &pdu->header_digest_crc32, + &pdu->hdr.raw, 0, + hlen, header_crc32_accel_done, pdu); + return; + } + + _tcp_write_pdu(pdu); +} + static int nvmf_tcp_qpair_init_mem_resource(struct spdk_nvmf_tcp_qpair *tqpair) { @@ -1649,6 +1681,17 @@ nvmf_tcp_icreq_handle(struct spdk_nvmf_tcp_transport *ttransport, /* Not fatal. */ } + if (tqpair->host_hdgst_enable) { + tqpair->accel_channel = spdk_accel_engine_get_io_channel(); + if (spdk_unlikely(!tqpair->accel_channel)) { + fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR; + error_offset = offsetof(struct spdk_nvme_tcp_ic_req, dgst); + SPDK_ERRLOG("Unabled to get accel_channel for tqpair=%p, failed to enable header digest\n", + tqpair); + goto end; + } + } + tqpair->cpda = spdk_min(ic_req->hpda, SPDK_NVME_TCP_CPDA_MAX); SPDK_DEBUGLOG(nvmf_tcp, "cpda of tqpair=(%p) is : %u\n", tqpair, tqpair->cpda); diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index 1d7b7d99c..7a70fe43e 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -79,7 +79,7 @@ DEPDIRS-event := log util thread $(JSON_LIBS) trace DEPDIRS-ftl := log util thread trace bdev DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev -DEPDIRS-nvmf := log sock util nvme thread $(JSON_LIBS) trace bdev +DEPDIRS-nvmf := accel log sock util nvme thread $(JSON_LIBS) trace bdev ifeq ($(CONFIG_RDMA),y) DEPDIRS-nvmf += rdma endif diff --git a/test/unit/lib/nvmf/tcp.c/tcp_ut.c b/test/unit/lib/nvmf/tcp.c/tcp_ut.c index b97f1848d..9b06633a0 100644 --- a/test/unit/lib/nvmf/tcp.c/tcp_ut.c +++ b/test/unit/lib/nvmf/tcp.c/tcp_ut.c @@ -222,6 +222,17 @@ DEFINE_STUB(nvmf_transport_req_free, (struct spdk_nvmf_request *req), 0); +DEFINE_STUB(spdk_accel_engine_get_io_channel, + struct spdk_io_channel *, + (void), + NULL); + +DEFINE_STUB(spdk_accel_submit_crc32c, + int, + (struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed, + uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg), + 0); + DEFINE_STUB(spdk_nvmf_bdev_ctrlr_nvme_passthru_admin, int, (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,