From e5d5b3ebef2077c577c4f956c2be346e1138e711 Mon Sep 17 00:00:00 2001 From: Ziye Yang Date: Tue, 13 Apr 2021 18:36:46 +0800 Subject: [PATCH] idxd/rpc: Revise the rpc function to use kernel or user driver This patch is used to add the support for users to configure use kernel or userspace idxd library. Change-Id: Ie159b897bc9595894ad8f333168efaea6c2a3d78 Signed-off-by: Ziye Yang Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7332 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker Reviewed-by: Changpeng Liu Reviewed-by: Aleksey Marchuk --- CHANGELOG.md | 3 +++ doc/jsonrpc.md | 4 +++- include/spdk/idxd.h | 3 ++- lib/idxd/idxd.c | 9 +++++++-- module/accel/idxd/accel_engine_idxd.c | 7 +++++-- module/accel/idxd/accel_engine_idxd.h | 2 +- module/accel/idxd/accel_engine_idxd_rpc.c | 10 ++++++++-- scripts/rpc.py | 7 +++++-- scripts/rpc/idxd.py | 9 +++++++-- test/unit/lib/idxd/idxd.c/idxd_ut.c | 2 +- 10 files changed, 42 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af0f5c8da..d08fe4b7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,9 @@ Rados Cluster names. Revised a parameter `--stripe-size_kb` to `--stripe-size-kb` of `bdev_raid_create` method provided in `scripts/rpc.py` for consistency. +An new optional parameter `config_kernel_mode` was added to the RPC `idxd_scan_accel_engine`, +and this is used to enable using the kernel mode IDXD driver. + ### bdev Removed ZCOPY emulation: The bdev module can be checked to see if it supports ZCOPY diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 8f77919a2..12d2e0991 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -1407,6 +1407,7 @@ This feature is considered as experimental. Name | Optional | Type | Description ----------------------- | -------- | ----------- | ----------- config_number | Required | number | Pre-defined config # to use (ie 0, 1). See [docs.](https://spdk.io/doc/idxd.html) +config_kernel_mode | Optional | Boolean | If set, will use kernel idxd driver. ### Example @@ -1415,7 +1416,8 @@ Example request: ~~~ { "params": { - "config_number": 0 + "config_number": 0, + "config_kernel_mode": false }, "jsonrpc": "2.0", "method": "idxd_scan_accel_engine", diff --git a/include/spdk/idxd.h b/include/spdk/idxd.h index 80120844f..a1d034a2a 100644 --- a/include/spdk/idxd.h +++ b/include/spdk/idxd.h @@ -117,8 +117,9 @@ void spdk_idxd_detach(struct spdk_idxd_device *idxd); * Sets the IDXD configuration. * * \param config_number the configuration number for a valid IDXD config. + * \param kernel_mode true if using kernel driver. */ -void spdk_idxd_set_config(uint32_t config_number); +void spdk_idxd_set_config(uint32_t config_number, bool kernel_mode); /** * Return the max number of descriptors per batch for IDXD. diff --git a/lib/idxd/idxd.c b/lib/idxd/idxd.c index 0ee77a7ad..1589a9194 100644 --- a/lib/idxd/idxd.c +++ b/lib/idxd/idxd.c @@ -46,6 +46,7 @@ #define ALIGN_4K 0x1000 #define USERSPACE_DRIVER_NAME "user" +#define KERNEL_DRIVER_NAME "kernel" #define CHAN_PER_DEVICE(total_wq_size) ((total_wq_size >= 128) ? 8 : 4) /* * Need to limit how many completions we reap in one poller to avoid starving @@ -257,9 +258,13 @@ idxd_get_impl_by_name(const char *impl_name) /* Called via RPC to select a pre-defined configuration. */ void -spdk_idxd_set_config(uint32_t config_num) +spdk_idxd_set_config(uint32_t config_num, bool kernel_mode) { - g_idxd_impl = idxd_get_impl_by_name(USERSPACE_DRIVER_NAME); + if (kernel_mode) { + g_idxd_impl = idxd_get_impl_by_name(KERNEL_DRIVER_NAME); + } else { + g_idxd_impl = idxd_get_impl_by_name(USERSPACE_DRIVER_NAME); + } if (g_idxd_impl == NULL) { SPDK_ERRLOG("Cannot set the idxd implementation"); diff --git a/module/accel/idxd/accel_engine_idxd.c b/module/accel/idxd/accel_engine_idxd.c index c67c0d876..d9a010ff3 100644 --- a/module/accel/idxd/accel_engine_idxd.c +++ b/module/accel/idxd/accel_engine_idxd.c @@ -47,6 +47,7 @@ #include "spdk/json.h" static bool g_idxd_enable = false; +static bool g_kernel_mode = false; uint32_t g_config_number; static uint32_t g_batch_max; @@ -345,7 +346,7 @@ attach_cb(void *cb_ctx, struct spdk_idxd_device *idxd) } void -accel_engine_idxd_enable_probe(uint32_t config_number) +accel_engine_idxd_enable_probe(uint32_t config_number, bool kernel_mode) { if (config_number > IDXD_MAX_CONFIG_NUM) { SPDK_ERRLOG("Invalid config number, using default of 0\n"); @@ -353,8 +354,9 @@ accel_engine_idxd_enable_probe(uint32_t config_number) } g_config_number = config_number; + g_kernel_mode = kernel_mode; g_idxd_enable = true; - spdk_idxd_set_config(g_config_number); + spdk_idxd_set_config(g_config_number, g_kernel_mode); } static int @@ -410,6 +412,7 @@ accel_engine_idxd_write_config_json(struct spdk_json_write_ctx *w) spdk_json_write_named_string(w, "method", "idxd_scan_accel_engine"); spdk_json_write_named_object_begin(w, "params"); spdk_json_write_named_uint32(w, "config_number", g_config_number); + spdk_json_write_named_uint32(w, "config_kernel_mode", g_kernel_mode); spdk_json_write_object_end(w); spdk_json_write_object_end(w); } diff --git a/module/accel/idxd/accel_engine_idxd.h b/module/accel/idxd/accel_engine_idxd.h index dac6569a2..214f5a1ab 100644 --- a/module/accel/idxd/accel_engine_idxd.h +++ b/module/accel/idxd/accel_engine_idxd.h @@ -38,6 +38,6 @@ #define IDXD_MAX_DEVICES 16 -void accel_engine_idxd_enable_probe(uint32_t config_number); +void accel_engine_idxd_enable_probe(uint32_t config_number, bool kernel_mode); #endif /* SPDK_ACCEL_ENGINE_IDXD_H */ diff --git a/module/accel/idxd/accel_engine_idxd_rpc.c b/module/accel/idxd/accel_engine_idxd_rpc.c index 48da4729e..bae5c893c 100644 --- a/module/accel/idxd/accel_engine_idxd_rpc.c +++ b/module/accel/idxd/accel_engine_idxd_rpc.c @@ -41,10 +41,12 @@ struct rpc_idxd_scan_accel_engine { uint32_t config_number; + bool config_kernel_mode; }; static const struct spdk_json_object_decoder rpc_idxd_scan_accel_engine_decoder[] = { {"config_number", offsetof(struct rpc_idxd_scan_accel_engine, config_number), spdk_json_decode_uint32}, + {"config_kernel_mode", offsetof(struct rpc_idxd_scan_accel_engine, config_kernel_mode), spdk_json_decode_bool, true}, }; static void @@ -64,9 +66,13 @@ rpc_idxd_scan_accel_engine(struct spdk_jsonrpc_request *request, } } - SPDK_NOTICELOG("Enabling IDXD with config #%u\n", req.config_number); - accel_engine_idxd_enable_probe(req.config_number); + if (req.config_kernel_mode) { + SPDK_NOTICELOG("Enabling IDXD kernel with config #%u\n", req.config_number); + } else { + SPDK_NOTICELOG("Enabling IDXD with config #%u\n", req.config_number); + } + accel_engine_idxd_enable_probe(req.config_number, req.config_kernel_mode); spdk_jsonrpc_send_bool_response(request, true); } SPDK_RPC_REGISTER("idxd_scan_accel_engine", rpc_idxd_scan_accel_engine, SPDK_RPC_STARTUP) diff --git a/scripts/rpc.py b/scripts/rpc.py index 895ade25a..3bb24b084 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2373,12 +2373,15 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse # idxd def idxd_scan_accel_engine(args): - rpc.idxd.idxd_scan_accel_engine(args.client, config_number=args.config_number) + rpc.idxd.idxd_scan_accel_engine(args.client, config_number=args.config_number, + config_kernel_mode=args.config_kernel_mode) p = subparsers.add_parser('idxd_scan_accel_engine', help='Set config and enable idxd accel engine offload.') p.add_argument('-c', '--config-number', help="""Pre-defined configuration number to use. See docs.""", type=int) - p.set_defaults(func=idxd_scan_accel_engine) + p.add_argument('-k', '--config-kernel-mode', help='Use Kernel mode idxd', + action='store_true', dest='config_kernel_mode') + p.set_defaults(func=idxd_scan_accel_engine, config_kernel_mode=None) # opal def bdev_nvme_opal_init(args): diff --git a/scripts/rpc/idxd.py b/scripts/rpc/idxd.py index 3e076c68e..cdcafa645 100644 --- a/scripts/rpc/idxd.py +++ b/scripts/rpc/idxd.py @@ -1,8 +1,13 @@ -def idxd_scan_accel_engine(client, config_number): +def idxd_scan_accel_engine(client, config_number=None, config_kernel_mode=None): """Scan and enable IDXD accel engine. Args: config_number: Pre-defined configuration number, see docs. + config_kernel_mode: Use kernel IDXD driver. (optional) """ - params = {'config_number': config_number} + params = {} + + params['config_number'] = config_number + if config_kernel_mode is not None: + params['config_kernel_mode'] = config_kernel_mode return client.call('idxd_scan_accel_engine', params) diff --git a/test/unit/lib/idxd/idxd.c/idxd_ut.c b/test/unit/lib/idxd/idxd.c/idxd_ut.c index 2607fe7c1..a6943e660 100644 --- a/test/unit/lib/idxd/idxd.c/idxd_ut.c +++ b/test/unit/lib/idxd/idxd.c/idxd_ut.c @@ -54,7 +54,7 @@ test_spdk_idxd_set_config(void) { g_dev_cfg = NULL; - spdk_idxd_set_config(0); + spdk_idxd_set_config(false, 0); SPDK_CU_ASSERT_FATAL(g_dev_cfg != NULL); CU_ASSERT(memcmp(&g_dev_cfg0, g_dev_cfg, sizeof(struct device_config)) == 0);