From d515bf739d0ab020d35efa9b2ea70e9cc665ddc6 Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Tue, 9 Mar 2021 01:43:10 +0800 Subject: [PATCH] bdev/nvme: Use the accelerated_engine This patch really uses the accelrated engine. Currently, we only offload the crc32c caculation, but it can be extended. Change-Id: If0e4c6a44b6e1e10e03f7eca355bed418d67326b Signed-off-by: Ziye Yang Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6760 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Changpeng Liu --- mk/spdk.lib_deps.mk | 2 +- module/bdev/nvme/bdev_nvme.c | 42 ++++++++++++++++++- module/bdev/nvme/common.h | 1 + .../lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c | 31 ++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index fe7bfac52..ece8233ff 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -140,7 +140,7 @@ DEPDIRS-bdev_crypto := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_delay := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_iscsi := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_null := $(BDEV_DEPS_THREAD) -DEPDIRS-bdev_nvme = $(BDEV_DEPS_THREAD) nvme +DEPDIRS-bdev_nvme = $(BDEV_DEPS_THREAD) accel nvme DEPDIRS-bdev_ocf := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_passthru := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_pmem := $(BDEV_DEPS_THREAD) diff --git a/module/bdev/nvme/bdev_nvme.c b/module/bdev/nvme/bdev_nvme.c index fbff171f4..620117def 100644 --- a/module/bdev/nvme/bdev_nvme.c +++ b/module/bdev/nvme/bdev_nvme.c @@ -36,6 +36,7 @@ #include "bdev_nvme.h" #include "bdev_ocssd.h" +#include "spdk/accel_engine.h" #include "spdk/config.h" #include "spdk/endian.h" #include "spdk/bdev.h" @@ -973,19 +974,54 @@ bdev_nvme_destroy_cb(void *io_device, void *ctx_buf) spdk_put_io_channel(spdk_io_channel_from_ctx(nvme_ch->group)); } +static void +bdev_nvme_poll_group_submit_accel_crc32c(void *ctx, uint32_t *dst, struct iovec *iov, + uint32_t iov_cnt, uint32_t seed, + spdk_nvme_accel_completion_cb cb_fn, void *cb_arg) +{ + struct nvme_bdev_poll_group *group = ctx; + int rc; + + assert(group->accel_channel != NULL); + assert(cb_fn != NULL); + + rc = spdk_accel_submit_crc32cv(group->accel_channel, dst, iov, iov_cnt, seed, cb_fn, cb_arg); + if (rc) { + /* For the two cases, spdk_accel_submit_crc32cv does not call the user's cb_fn */ + if (rc == -ENOMEM || rc == -EINVAL) { + cb_fn(cb_arg, rc); + } + SPDK_ERRLOG("Cannot complete the accelerated crc32c operation with iov=%p\n", iov); + } +} + +static struct spdk_nvme_accel_fn_table g_bdev_nvme_accel_fn_table = { + .table_size = sizeof(struct spdk_nvme_accel_fn_table), + .submit_accel_crc32c = bdev_nvme_poll_group_submit_accel_crc32c, +}; + static int bdev_nvme_poll_group_create_cb(void *io_device, void *ctx_buf) { struct nvme_bdev_poll_group *group = ctx_buf; - group->group = spdk_nvme_poll_group_create(group, NULL); + group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table); if (group->group == NULL) { return -1; } + group->accel_channel = spdk_accel_engine_get_io_channel(); + if (!group->accel_channel) { + spdk_nvme_poll_group_destroy(group->group); + SPDK_ERRLOG("Cannot get the accel_channel for bdev nvme polling group=%p\n", + group); + return -1; + } + group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us); if (group->poller == NULL) { + spdk_put_io_channel(group->accel_channel); spdk_nvme_poll_group_destroy(group->group); return -1; } @@ -998,6 +1034,10 @@ bdev_nvme_poll_group_destroy_cb(void *io_device, void *ctx_buf) { struct nvme_bdev_poll_group *group = ctx_buf; + if (group->accel_channel) { + spdk_put_io_channel(group->accel_channel); + } + spdk_poller_unregister(&group->poller); if (spdk_nvme_poll_group_destroy(group->group)) { SPDK_ERRLOG("Unable to destroy a poll group for the NVMe bdev module."); diff --git a/module/bdev/nvme/common.h b/module/bdev/nvme/common.h index 407789552..1590e0aed 100644 --- a/module/bdev/nvme/common.h +++ b/module/bdev/nvme/common.h @@ -124,6 +124,7 @@ struct nvme_bdev { struct nvme_bdev_poll_group { struct spdk_nvme_poll_group *group; + struct spdk_io_channel *accel_channel; struct spdk_poller *poller; bool collect_spin_stat; uint64_t spin_ticks; diff --git a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c index 01ea80bb9..90a518a22 100644 --- a/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c +++ b/test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c @@ -44,6 +44,8 @@ #include "unit/lib/json_mock.c" +static void *g_accel_p = (void *)0xdeadbeaf; + DEFINE_STUB(spdk_nvme_probe_async, struct spdk_nvme_probe_ctx *, (const struct spdk_nvme_transport_id *trid, void *cb_ctx, spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb, @@ -70,6 +72,15 @@ DEFINE_STUB_V(spdk_nvme_ctrlr_set_remove_cb, (struct spdk_nvme_ctrlr *ctrlr, DEFINE_STUB(spdk_nvme_ctrlr_get_flags, uint64_t, (struct spdk_nvme_ctrlr *ctrlr), 0); +DEFINE_STUB(accel_engine_create_cb, int, (void *io_device, void *ctx_buf), 0); +DEFINE_STUB_V(accel_engine_destroy_cb, (void *io_device, void *ctx_buf)); + +struct spdk_io_channel * +spdk_accel_engine_get_io_channel(void) +{ + return spdk_get_io_channel(g_accel_p); +} + void spdk_nvme_ctrlr_get_default_io_qpair_opts(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_io_qpair_opts *opts, size_t opts_size) @@ -150,6 +161,11 @@ DEFINE_STUB_V(bdev_ocssd_fini_ctrlr, (struct nvme_bdev_ctrlr *nvme_bdev_ctrlr)); DEFINE_STUB_V(bdev_ocssd_handle_chunk_notification, (struct nvme_bdev_ctrlr *nvme_bdev_ctrlr)); +DEFINE_STUB(spdk_accel_submit_crc32cv, int, (struct spdk_io_channel *ch, uint32_t *dst, + struct iovec *iov, + uint32_t iov_cnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg), 0); + + struct ut_nvme_req { uint16_t opc; spdk_nvme_cmd_cb cb_fn; @@ -2107,6 +2123,19 @@ test_bdev_unregister(void) ut_detach_ctrlr(ctrlr); } +static void +init_accel(void) +{ + spdk_io_device_register(g_accel_p, accel_engine_create_cb, accel_engine_destroy_cb, + sizeof(int), "accel_p"); +} + +static void +fini_accel(void) +{ + spdk_io_device_unregister(g_accel_p, NULL); +} + int main(int argc, const char **argv) { @@ -2137,11 +2166,13 @@ main(int argc, const char **argv) allocate_threads(3); set_thread(0); bdev_nvme_library_init(); + init_accel(); CU_basic_run_tests(); set_thread(0); bdev_nvme_library_fini(); + fini_accel(); free_threads(); num_failures = CU_get_number_of_failures();