From eb76afe78b5c56281a8d0c987bb38b44186fd043 Mon Sep 17 00:00:00 2001 From: Jin Yu Date: Wed, 17 Jun 2020 01:26:30 +0800 Subject: [PATCH] event: add iova-mode parameter Export iova-mode parameters in spdk which is useful in VM environment. Change-Id: I3f4756b2c3b6cf5d1964a50bbf63f9c596997696 Signed-off-by: Jin Yu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2910 Community-CI: Mellanox Build Bot Community-CI: Broadcom CI Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Changpeng Liu --- include/spdk/env.h | 1 + include/spdk/event.h | 1 + lib/env_dpdk/init.c | 47 +++++++++++++++++++++++++------------------- lib/event/Makefile | 2 +- lib/event/app.c | 9 ++++++++- 5 files changed, 38 insertions(+), 22 deletions(-) diff --git a/include/spdk/env.h b/include/spdk/env.h index 4d522b941..4777d3842 100644 --- a/include/spdk/env.h +++ b/include/spdk/env.h @@ -84,6 +84,7 @@ struct spdk_env_opts { const char *hugedir; struct spdk_pci_addr *pci_blacklist; struct spdk_pci_addr *pci_whitelist; + const char *iova_mode; uint64_t base_virtaddr; /** Opaque context for use of the env implementation. */ diff --git a/include/spdk/event.h b/include/spdk/event.h index 484bfccca..98392cf7c 100644 --- a/include/spdk/event.h +++ b/include/spdk/event.h @@ -113,6 +113,7 @@ struct spdk_app_opts { size_t num_pci_addr; struct spdk_pci_addr *pci_blacklist; struct spdk_pci_addr *pci_whitelist; + const char *iova_mode; /* DEPRECATED. No longer has any effect. * diff --git a/lib/env_dpdk/init.c b/lib/env_dpdk/init.c index eb0d7baa8..20fb05211 100644 --- a/lib/env_dpdk/init.c +++ b/lib/env_dpdk/init.c @@ -413,36 +413,43 @@ build_eal_cmdline(const struct spdk_env_opts *opts) #ifdef __linux__ - /* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa, - * but DPDK guesses it should be iova-mode=va. Add a check and force - * iova-mode=pa here. */ - if (rte_vfio_noiommu_is_enabled()) { - args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); + if (opts->iova_mode) { + args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=%s", opts->iova_mode)); if (args == NULL) { return -1; } - } + } else { + /* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa, + * but DPDK guesses it should be iova-mode=va. Add a check and force + * iova-mode=pa here. */ + if (rte_vfio_noiommu_is_enabled()) { + args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); + if (args == NULL) { + return -1; + } + } #if defined(__x86_64__) - /* DPDK by default guesses that it should be using iova-mode=va so that it can - * support running as an unprivileged user. However, some systems (especially - * virtual machines) don't have an IOMMU capable of handling the full virtual - * address space and DPDK doesn't currently catch that. Add a check in SPDK - * and force iova-mode=pa here. */ - if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) { + /* DPDK by default guesses that it should be using iova-mode=va so that it can + * support running as an unprivileged user. However, some systems (especially + * virtual machines) don't have an IOMMU capable of handling the full virtual + * address space and DPDK doesn't currently catch that. Add a check in SPDK + * and force iova-mode=pa here. */ + if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) { + args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); + if (args == NULL) { + return -1; + } + } +#elif defined(__PPC64__) + /* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly + * auto-detect at the moment, so we'll just force it here. */ args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); if (args == NULL) { return -1; } - } -#elif defined(__PPC64__) - /* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly - * auto-detect at the moment, so we'll just force it here. */ - args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa")); - if (args == NULL) { - return -1; - } #endif + } /* Set the base virtual address - it must be an address that is not in the diff --git a/lib/event/Makefile b/lib/event/Makefile index 9be0da351..d36cd2d55 100644 --- a/lib/event/Makefile +++ b/lib/event/Makefile @@ -34,7 +34,7 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk -SO_VER := 3 +SO_VER := 4 SO_MINOR := 0 LIBNAME = event diff --git a/lib/event/app.c b/lib/event/app.c index ad1a4ef81..3d56fb644 100644 --- a/lib/event/app.c +++ b/lib/event/app.c @@ -110,7 +110,7 @@ static const struct option g_cmdline_options[] = { {"version", no_argument, NULL, VERSION_OPT_IDX}, #define PCI_BLACKLIST_OPT_IDX 'B' {"pci-blacklist", required_argument, NULL, PCI_BLACKLIST_OPT_IDX}, -#define LOGFLAG_OPT_IDX 'L' +#define LOGFLAG_OPT_IDX 'L' {"logflag", required_argument, NULL, LOGFLAG_OPT_IDX}, #define HUGE_UNLINK_OPT_IDX 'R' {"huge-unlink", no_argument, NULL, HUGE_UNLINK_OPT_IDX}, @@ -130,6 +130,8 @@ static const struct option g_cmdline_options[] = { {"json", required_argument, NULL, JSON_CONFIG_OPT_IDX}, #define JSON_CONFIG_IGNORE_INIT_ERRORS_IDX 263 {"json-ignore-init-errors", no_argument, NULL, JSON_CONFIG_IGNORE_INIT_ERRORS_IDX}, +#define IOVA_MODE_OPT_IDX 264 + {"iova-mode", required_argument, NULL, IOVA_MODE_OPT_IDX}, }; /* Global section */ @@ -507,6 +509,7 @@ app_setup_env(struct spdk_app_opts *opts) env_opts.pci_blacklist = opts->pci_blacklist; env_opts.pci_whitelist = opts->pci_whitelist; env_opts.env_context = opts->env_context; + env_opts.iova_mode = opts->iova_mode; rc = spdk_env_init(&env_opts); free(env_opts.pci_blacklist); @@ -761,6 +764,7 @@ usage(void (*app_usage)(void)) printf(" -W, --pci-whitelist \n"); printf(" pci addr to whitelist (-B and -W cannot be used at the same time)\n"); printf(" --huge-dir use a specific hugetlbfs mount to reserve memory from\n"); + printf(" --iova-mode set IOVA mode ('pa' for IOVA_PA and 'va' for IOVA_VA)\n"); printf(" --num-trace-entries number of trace entries for each core, must be power of 2. (default %d)\n", SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES); spdk_log_usage(stdout, "-L"); @@ -974,6 +978,9 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, case HUGE_DIR_OPT_IDX: opts->hugedir = optarg; break; + case IOVA_MODE_OPT_IDX: + opts->iova_mode = optarg; + break; case NUM_TRACE_ENTRIES_OPT_IDX: tmp = spdk_strtoll(optarg, 0); if (tmp <= 0) {