From a4463cbc64e41103d2e18ddc666a7e6200dc702f Mon Sep 17 00:00:00 2001 From: Marcin Spiewak Date: Wed, 29 Mar 2023 11:44:50 +0000 Subject: [PATCH] app: use --lcores to map ids greater than 128 Fixes #2812 This patch adds support for '--lcores ' parameter in spdk. This parameter allow mapping of the lcores to CPU IDs, if the system contains CPUs with IDs greater or equal to 128 (RTE_MAX_LCORE). Such CPUs can not be directly included in core mask specified in '-m ' parameter, as the dpdk rejects cores if IDs are greater than 127. The only way to use them in spdk is to map lcore to CPU using --lcores parameters specified in command line. --lcores and -m parameters are mutually exclusive. Please use only one of them. Examples: build/bin/nvmf_tgt --lcores 0@130 build/bin/nvmf_tgt --lcores 0@150,1@151 build/bin/nvmf_tgt --lcores "(5-7)@(10-12)" build/bin/nvmf_tgt --lcores "(5-7)@(136,138,140)" Change-Id: Ia92be4499c8daaa936b1a4357d52ae303d6f3eb1 Signed-off-by: Marcin Spiewak Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17403 Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Aleksey Marchuk Reviewed-by: Shuhei Matsumoto --- lib/env_dpdk/init.c | 41 ++++++++++++++++++++++++++++------------- lib/event/app.c | 28 +++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/lib/env_dpdk/init.c b/lib/env_dpdk/init.c index 2625a8290..445a68111 100644 --- a/lib/env_dpdk/init.c +++ b/lib/env_dpdk/init.c @@ -225,19 +225,34 @@ build_eal_cmdline(const struct spdk_env_opts *opts) } } - /* - * Set the coremask: - * - * - if it starts with '-', we presume it's literal EAL arguments such - * as --lcores. - * - * - if it starts with '[', we presume it's a core list to use with the - * -l option. - * - * - otherwise, it's a CPU mask of the form "0xff.." as expected by the - * -c option. - */ - if (opts->core_mask[0] == '-') { + /* Either lcore_map or core_mask must be set. If both, or none specified, fail */ + if ((opts->core_mask == NULL) == (opts->lcore_map == NULL)) { + if (opts->core_mask && opts->lcore_map) { + fprintf(stderr, + "Both, lcore map and core mask are provided, while only one can be set\n"); + } else { + fprintf(stderr, "Core mask or lcore map must be specified\n"); + } + free_args(args, argcount); + return -1; + } + + if (opts->lcore_map) { + /* If lcore list is set, generate --lcores parameter */ + args = push_arg(args, &argcount, _sprintf_alloc("--lcores=%s", opts->lcore_map)); + } else if (opts->core_mask[0] == '-') { + /* + * Set the coremask: + * + * - if it starts with '-', we presume it's literal EAL arguments such + * as --lcores. + * + * - if it starts with '[', we presume it's a core list to use with the + * -l option. + * + * - otherwise, it's a CPU mask of the form "0xff.." as expected by the + * -c option. + */ args = push_arg(args, &argcount, _sprintf_alloc("%s", opts->core_mask)); } else if (opts->core_mask[0] == '[') { char *l_arg = _sprintf_alloc("-l %s", opts->core_mask + 1); diff --git a/lib/event/app.c b/lib/event/app.c index 8ba31843c..b298e151f 100644 --- a/lib/event/app.c +++ b/lib/event/app.c @@ -129,6 +129,8 @@ static const struct option g_cmdline_options[] = { {"vfio-vf-token", required_argument, NULL, ENV_VF_TOKEN_OPT_IDX}, #define MSG_MEMPOOL_SIZE_OPT_IDX 270 {"msg-mempool-size", required_argument, NULL, MSG_MEMPOOL_SIZE_OPT_IDX}, +#define LCORES_OPT_IDX 271 + {"lcores", required_argument, NULL, LCORES_OPT_IDX}, }; static void @@ -203,7 +205,6 @@ spdk_app_opts_init(struct spdk_app_opts *opts, size_t opts_size) SET_FIELD(mem_size, SPDK_APP_DPDK_DEFAULT_MEM_SIZE); SET_FIELD(main_core, SPDK_APP_DPDK_DEFAULT_MAIN_CORE); SET_FIELD(mem_channel, SPDK_APP_DPDK_DEFAULT_MEM_CHANNEL); - SET_FIELD(reactor_mask, SPDK_APP_DPDK_DEFAULT_CORE_MASK); SET_FIELD(base_virtaddr, SPDK_APP_DPDK_DEFAULT_BASE_VIRTADDR); SET_FIELD(print_level, SPDK_APP_DEFAULT_LOG_PRINT_LEVEL); SET_FIELD(rpc_addr, SPDK_DEFAULT_RPC_ADDR); @@ -326,6 +327,7 @@ app_setup_env(struct spdk_app_opts *opts) env_opts.name = opts->name; env_opts.core_mask = opts->reactor_mask; + env_opts.lcore_map = opts->lcore_map; env_opts.shm_id = opts->shm_id; env_opts.mem_channel = opts->mem_channel; env_opts.main_core = opts->main_core; @@ -497,6 +499,7 @@ app_copy_opts(struct spdk_app_opts *opts, struct spdk_app_opts *opts_user, size_ SET_FIELD(json_config_ignore_errors); SET_FIELD(rpc_addr); SET_FIELD(reactor_mask); + SET_FIELD(lcore_map); SET_FIELD(tpoint_group_mask); SET_FIELD(shm_id); SET_FIELD(shutdown_cb); @@ -669,6 +672,11 @@ spdk_app_start(struct spdk_app_opts *opts_user, spdk_msg_fn start_fn, return 1; } + if (!(opts->lcore_map || opts->reactor_mask)) { + /* Set default CPU mask */ + opts->reactor_mask = SPDK_APP_DPDK_DEFAULT_CORE_MASK; + } + tty = ttyname(STDERR_FILENO); if (opts->print_level > SPDK_LOG_WARN && isatty(STDERR_FILENO) && @@ -860,6 +868,13 @@ usage(void (*app_usage)(void)) printf(" -h, --help show this usage\n"); printf(" -i, --shm-id shared memory ID (optional)\n"); printf(" -m, --cpumask core mask (like 0xF) or core list of '[]' embraced (like [0,1,10]) for DPDK\n"); + printf(" --lcores lcore to CPU mapping list. The list is in the format:\n"); + printf(" [<,lcores[@CPUs]>...]\n"); + printf(" lcores and cpus list are grouped by '(' and ')', e.g '--lcores \"(5-7)@(10-12)\"'\n"); + printf(" Within the group, '-' is used for range separator,\n"); + printf(" ',' is used for single number separator.\n"); + printf(" '( )' can be omitted for single element group,\n"); + printf(" '@' can be omitted if cpus and lcores have the same value\n"); printf(" -n, --mem-channels channel number of memory channels used for DPDK\n"); printf(" -p, --main-core main (primary) core for DPDK\n"); printf(" -r, --rpc-socket RPC listen address (default %s)\n", SPDK_DEFAULT_RPC_ADDR); @@ -1001,8 +1016,19 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, } break; case CPUMASK_OPT_IDX: + if (opts->lcore_map) { + SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); + goto out; + } opts->reactor_mask = optarg; break; + case LCORES_OPT_IDX: + if (opts->reactor_mask) { + SPDK_ERRLOG("lcore map and core mask can't be set simultaneously\n"); + goto out; + } + opts->lcore_map = optarg; + break; case DISABLE_CPUMASK_LOCKS_OPT_IDX: g_disable_cpumask_locks = true; break;