lib/nvmf: Use the async function to caculate the crc32 for header_digest.

In this patch, we will use the accel library to do the crc32 work for the
header digest when sending the pdu in the target side.

For data digest support, will consider in the further patch.

Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Change-Id: I3169a158afd633f48bdbeb2cce1ed20e4141ae45
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5472
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

include/spdk_internal/nvme_tcp.h

@@ -101,6 +101,7 @@ struct nvme_tcp_pdu {
 
 	bool						has_hdgst;
 	bool						ddgst_enable;
+	uint32_t					header_digest_crc32;
 	uint8_t						data_digest[SPDK_NVME_TCP_DIGEST_LEN];
 
 	uint8_t						ch_valid_bytes;

lib/nvmf/tcp.c

@@ -31,6 +31,7 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "spdk/accel_engine.h"
 #include "spdk/stdinc.h"
 #include "spdk/crc32.h"
 #include "spdk/endian.h"
@@ -251,6 +252,8 @@ struct spdk_nvmf_tcp_qpair {
 	 */
 	struct spdk_poller			*timeout_poller;
 
+	struct spdk_io_channel			*accel_channel;
+
 	TAILQ_ENTRY(spdk_nvmf_tcp_qpair)	link;
 };
 
@@ -477,6 +480,9 @@ nvmf_tcp_qpair_destroy(struct spdk_nvmf_tcp_qpair *tqpair)
 		nvmf_tcp_dump_qpair_req_contents(tqpair);
 	}
 
+	if (tqpair->accel_channel) {
+		spdk_put_io_channel(tqpair->accel_channel);
+	}
 	spdk_dma_free(tqpair->pdus);
 	free(tqpair->reqs);
 	spdk_free(tqpair->bufs);
@@ -800,25 +806,12 @@ _pdu_write_done(void *_pdu, int err)
 }
 
 static void
-nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair,
+_tcp_write_pdu(struct nvme_tcp_pdu *pdu)
-			 struct nvme_tcp_pdu *pdu,
-			 nvme_tcp_qpair_xfer_complete_cb cb_fn,
-			 void *cb_arg)
 {
-	int hlen;
-	uint32_t crc32c;
 	uint32_t mapped_length = 0;
 	ssize_t rc;
-
+	struct spdk_nvmf_tcp_qpair *tqpair = pdu->qpair;
-	assert(&tqpair->pdu_in_progress != pdu);
+	uint32_t crc32c;
-
-	hlen = pdu->hdr.common.hlen;
-
-	/* Header Digest */
-	if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) {
-		crc32c = nvme_tcp_pdu_calc_header_digest(pdu);
-		MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + hlen, crc32c);
-	}
 
 	/* Data Digest */
 	if (pdu->data_len > 0 && g_nvme_tcp_ddgst[pdu->hdr.common.pdu_type] && tqpair->host_ddgst_enable) {
@@ -826,9 +819,6 @@ nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair,
 		MAKE_DIGEST_WORD(pdu->data_digest, crc32c);
 	}
 
-	pdu->cb_fn = cb_fn;
-	pdu->cb_arg = cb_arg;
-
 	pdu->sock_req.iovcnt = nvme_tcp_build_iovs(pdu->iov, SPDK_COUNTOF(pdu->iov), pdu,
 			       tqpair->host_hdgst_enable, tqpair->host_ddgst_enable,
 			       &mapped_length);
@@ -848,6 +838,48 @@ nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair,
 	}
 }
 
+static void
+header_crc32_accel_done(void *cb_arg, int status)
+{
+	struct nvme_tcp_pdu *pdu = cb_arg;
+
+	pdu->header_digest_crc32 = pdu->header_digest_crc32 ^ SPDK_CRC32C_XOR;
+	MAKE_DIGEST_WORD((uint8_t *)pdu->hdr.raw + pdu->hdr.common.hlen, pdu->header_digest_crc32);
+	if (spdk_unlikely(status)) {
+		SPDK_ERRLOG("Failed to finish the crc32 work\n");
+		_pdu_write_done(pdu, status);
+		return;
+	}
+
+	_tcp_write_pdu(pdu);
+}
+
+static void
+nvmf_tcp_qpair_write_pdu(struct spdk_nvmf_tcp_qpair *tqpair,
+			 struct nvme_tcp_pdu *pdu,
+			 nvme_tcp_qpair_xfer_complete_cb cb_fn,
+			 void *cb_arg)
+{
+	int hlen;
+
+	assert(&tqpair->pdu_in_progress != pdu);
+
+	hlen = pdu->hdr.common.hlen;
+	pdu->cb_fn = cb_fn;
+	pdu->cb_arg = cb_arg;
+	pdu->qpair = tqpair;
+
+	/* Header Digest */
+	if (g_nvme_tcp_hdgst[pdu->hdr.common.pdu_type] && tqpair->host_hdgst_enable) {
+		spdk_accel_submit_crc32c(tqpair->accel_channel, &pdu->header_digest_crc32,
+					 &pdu->hdr.raw, 0,
+					 hlen, header_crc32_accel_done, pdu);
+		return;
+	}
+
+	_tcp_write_pdu(pdu);
+}
+
 static int
 nvmf_tcp_qpair_init_mem_resource(struct spdk_nvmf_tcp_qpair *tqpair)
 {
@@ -1649,6 +1681,17 @@ nvmf_tcp_icreq_handle(struct spdk_nvmf_tcp_transport *ttransport,
 		/* Not fatal. */
 	}
 
+	if (tqpair->host_hdgst_enable) {
+		tqpair->accel_channel = spdk_accel_engine_get_io_channel();
+		if (spdk_unlikely(!tqpair->accel_channel)) {
+			fes = SPDK_NVME_TCP_TERM_REQ_FES_HDGST_ERROR;
+			error_offset = offsetof(struct spdk_nvme_tcp_ic_req, dgst);
+			SPDK_ERRLOG("Unabled to get accel_channel for tqpair=%p, failed to enable header digest\n",
+				    tqpair);
+			goto end;
+		}
+	}
+
 	tqpair->cpda = spdk_min(ic_req->hpda, SPDK_NVME_TCP_CPDA_MAX);
 	SPDK_DEBUGLOG(nvmf_tcp, "cpda of tqpair=(%p) is : %u\n", tqpair, tqpair->cpda);
 

mk/spdk.lib_deps.mk

@@ -79,7 +79,7 @@ DEPDIRS-event := log util thread $(JSON_LIBS) trace
 
 DEPDIRS-ftl := log util thread trace bdev
 DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev
-DEPDIRS-nvmf := log sock util nvme thread $(JSON_LIBS) trace bdev
+DEPDIRS-nvmf := accel log sock util nvme thread $(JSON_LIBS) trace bdev
 ifeq ($(CONFIG_RDMA),y)
 DEPDIRS-nvmf += rdma
 endif

test/unit/lib/nvmf/tcp.c/tcp_ut.c

@@ -222,6 +222,17 @@ DEFINE_STUB(nvmf_transport_req_free,
 	    (struct spdk_nvmf_request *req),
 	    0);
 
+DEFINE_STUB(spdk_accel_engine_get_io_channel,
+	    struct spdk_io_channel *,
+	    (void),
+	    NULL);
+
+DEFINE_STUB(spdk_accel_submit_crc32c,
+	    int,
+	    (struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed,
+	     uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg),
+	    0);
+
 DEFINE_STUB(spdk_nvmf_bdev_ctrlr_nvme_passthru_admin,
 	    int,
 	    (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,