bdev/nvme: Use the accelerated_engine

This patch really uses the accelrated engine.
Currently, we only offload the crc32c caculation,
but it can be extended.

Change-Id: If0e4c6a44b6e1e10e03f7eca355bed418d67326b
Signed-off-by: Ziye Yang <ziye.yang@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6760
Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

mk/spdk.lib_deps.mk

@@ -140,7 +140,7 @@ DEPDIRS-bdev_crypto := $(BDEV_DEPS_THREAD)
 DEPDIRS-bdev_delay := $(BDEV_DEPS_THREAD)
 DEPDIRS-bdev_iscsi := $(BDEV_DEPS_THREAD)
 DEPDIRS-bdev_null := $(BDEV_DEPS_THREAD)
-DEPDIRS-bdev_nvme = $(BDEV_DEPS_THREAD) nvme
+DEPDIRS-bdev_nvme = $(BDEV_DEPS_THREAD) accel nvme
 DEPDIRS-bdev_ocf := $(BDEV_DEPS_THREAD)
 DEPDIRS-bdev_passthru := $(BDEV_DEPS_THREAD)
 DEPDIRS-bdev_pmem := $(BDEV_DEPS_THREAD)

module/bdev/nvme/bdev_nvme.c

@@ -36,6 +36,7 @@
 #include "bdev_nvme.h"
 #include "bdev_ocssd.h"
 
+#include "spdk/accel_engine.h"
 #include "spdk/config.h"
 #include "spdk/endian.h"
 #include "spdk/bdev.h"
@@ -973,19 +974,54 @@ bdev_nvme_destroy_cb(void *io_device, void *ctx_buf)
 	spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_ch->group));
 }
 
+static void
+bdev_nvme_poll_group_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov,
+		uint32_t iov_cnt, uint32_t seed,
+		spdk_nvme_accel_completion_cb cb_fn, void *cb_arg)
+{
+	struct nvme_bdev_poll_group *group = ctx;
+	int rc;
+
+	assert(group->accel_channel != NULL);
+	assert(cb_fn != NULL);
+
+	rc = spdk_accel_submit_crc32cv(group->accel_channel, dst, iov, iov_cnt, seed, cb_fn, cb_arg);
+	if (rc) {
+		/* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */
+		if (rc == -ENOMEM || rc == -EINVAL) {
+			cb_fn(cb_arg, rc);
+		}
+		SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov);
+	}
+}
+
+static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = {
+	.table_size		= sizeof(struct spdk_nvme_accel_fn_table),
+	.submit_accel_crc32c	= bdev_nvme_poll_group_submit_accel_crc32c,
+};
+
 static int
 bdev_nvme_poll_group_create_cb(void *io_device, void *ctx_buf)
 {
 	struct nvme_bdev_poll_group *group = ctx_buf;
 
-	group->group = spdk_nvme_poll_group_create(group, NULL);
+	group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table);
 	if (group->group == NULL) {
 		return -1;
 	}
 
+	group->accel_channel = spdk_accel_engine_get_io_channel();
+	if (!group->accel_channel) {
+		spdk_nvme_poll_group_destroy(group->group);
+		SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n",
+			    group);
+		return -1;
+	}
+
 	group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us);
 
 	if (group->poller == NULL) {
+		spdk_put_io_channel(group->accel_channel);
 		spdk_nvme_poll_group_destroy(group->group);
 		return -1;
 	}
@@ -998,6 +1034,10 @@ bdev_nvme_poll_group_destroy_cb(void *io_device, void *ctx_buf)
 {
 	struct nvme_bdev_poll_group *group = ctx_buf;
 
+	if (group->accel_channel) {
+		spdk_put_io_channel(group->accel_channel);
+	}
+
 	spdk_poller_unregister(&group->poller);
 	if (spdk_nvme_poll_group_destroy(group->group)) {
 		SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module.");

module/bdev/nvme/common.h

@@ -124,6 +124,7 @@ struct nvme_bdev {
 
 struct nvme_bdev_poll_group {
 	struct spdk_nvme_poll_group		*group;
+	struct spdk_io_channel			*accel_channel;
 	struct spdk_poller			*poller;
 	bool					collect_spin_stat;
 	uint64_t				spin_ticks;

test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c

@@ -44,6 +44,8 @@
 
 #include "unit/lib/json_mock.c"
 
+static void *g_accel_p = (void *)0xdeadbeaf;
+
 DEFINE_STUB(spdk_nvme_probe_async, struct spdk_nvme_probe_ctx *,
 	    (const struct spdk_nvme_transport_id *trid, void *cb_ctx,
 	     spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb,
@@ -70,6 +72,15 @@ DEFINE_STUB_V(spdk_nvme_ctrlr_set_remove_cb, (struct spdk_nvme_ctrlr *ctrlr,
 
 DEFINE_STUB(spdk_nvme_ctrlr_get_flags, uint64_t, (struct spdk_nvme_ctrlr *ctrlr), 0);
 
+DEFINE_STUB(accel_engine_create_cb, int, (void *io_device, void *ctx_buf), 0);
+DEFINE_STUB_V(accel_engine_destroy_cb, (void *io_device, void *ctx_buf));
+
+struct spdk_io_channel *
+spdk_accel_engine_get_io_channel(void)
+{
+	return spdk_get_io_channel(g_accel_p);
+}
+
 void
 spdk_nvme_ctrlr_get_default_io_qpair_opts(struct spdk_nvme_ctrlr *ctrlr,
 		struct spdk_nvme_io_qpair_opts *opts, size_t opts_size)
@@ -150,6 +161,11 @@ DEFINE_STUB_V(bdev_ocssd_fini_ctrlr, (struct nvme_bdev_ctrlr *nvme_bdev_ctrlr));
 
 DEFINE_STUB_V(bdev_ocssd_handle_chunk_notification, (struct nvme_bdev_ctrlr *nvme_bdev_ctrlr));
 
+DEFINE_STUB(spdk_accel_submit_crc32cv, int, (struct spdk_io_channel *ch, uint32_t *dst,
+		struct iovec *iov,
+		uint32_t iov_cnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg), 0);
+
+
 struct ut_nvme_req {
 	uint16_t			opc;
 	spdk_nvme_cmd_cb		cb_fn;
@@ -2107,6 +2123,19 @@ test_bdev_unregister(void)
 	ut_detach_ctrlr(ctrlr);
 }
 
+static void
+init_accel(void)
+{
+	spdk_io_device_register(g_accel_p, accel_engine_create_cb, accel_engine_destroy_cb,
+				sizeof(int), "accel_p");
+}
+
+static void
+fini_accel(void)
+{
+	spdk_io_device_unregister(g_accel_p, NULL);
+}
+
 int
 main(int argc, const char **argv)
 {
@@ -2137,11 +2166,13 @@ main(int argc, const char **argv)
 	allocate_threads(3);
 	set_thread(0);
 	bdev_nvme_library_init();
+	init_accel();
 
 	CU_basic_run_tests();
 
 	set_thread(0);
 	bdev_nvme_library_fini();
+	fini_accel();
 	free_threads();
 
 	num_failures = CU_get_number_of_failures();