event: add iova-mode parameter

Export iova-mode parameters in spdk which is useful in
VM environment.

Change-Id: I3f4756b2c3b6cf5d1964a50bbf63f9c596997696
Signed-off-by: Jin Yu <jin.yu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2910
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

include/spdk/env.h

@@ -84,6 +84,7 @@ struct spdk_env_opts {
 	const char		*hugedir;
 	struct spdk_pci_addr	*pci_blacklist;
 	struct spdk_pci_addr	*pci_whitelist;
+	const char		*iova_mode;
 	uint64_t		base_virtaddr;
 
 	/** Opaque context for use of the env implementation. */

include/spdk/event.h

@@ -113,6 +113,7 @@ struct spdk_app_opts {
 	size_t			num_pci_addr;
 	struct spdk_pci_addr	*pci_blacklist;
 	struct spdk_pci_addr	*pci_whitelist;
+	const char		*iova_mode;
 
 	/* DEPRECATED. No longer has any effect.
 	 *

lib/env_dpdk/init.c

@@ -413,36 +413,43 @@ build_eal_cmdline(const struct spdk_env_opts *opts)
 
 #ifdef __linux__
 
-	/* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa,
+	if (opts->iova_mode) {
-	 * but DPDK guesses it should be iova-mode=va. Add a check and force
+		args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=%s", opts->iova_mode));
-	 * iova-mode=pa here. */
-	if (rte_vfio_noiommu_is_enabled()) {
-		args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
 		if (args == NULL) {
 			return -1;
 		}
-	}
+	} else {
+		/* When using vfio with enable_unsafe_noiommu_mode=Y, we need iova-mode=pa,
+		 * but DPDK guesses it should be iova-mode=va. Add a check and force
+		 * iova-mode=pa here. */
+		if (rte_vfio_noiommu_is_enabled()) {
+			args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
+			if (args == NULL) {
+				return -1;
+			}
+		}
 
 #if defined(__x86_64__)
-	/* DPDK by default guesses that it should be using iova-mode=va so that it can
+		/* DPDK by default guesses that it should be using iova-mode=va so that it can
-	 * support running as an unprivileged user. However, some systems (especially
+		 * support running as an unprivileged user. However, some systems (especially
-	 * virtual machines) don't have an IOMMU capable of handling the full virtual
+		 * virtual machines) don't have an IOMMU capable of handling the full virtual
-	 * address space and DPDK doesn't currently catch that. Add a check in SPDK
+		 * address space and DPDK doesn't currently catch that. Add a check in SPDK
-	 * and force iova-mode=pa here. */
+		 * and force iova-mode=pa here. */
-	if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) {
+		if (get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) {
+			args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
+			if (args == NULL) {
+				return -1;
+			}
+		}
+#elif defined(__PPC64__)
+		/* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly
+		 * auto-detect at the moment, so we'll just force it here. */
 		args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
 		if (args == NULL) {
 			return -1;
 		}
-	}
-#elif defined(__PPC64__)
-	/* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly
-	 * auto-detect at the moment, so we'll just force it here. */
-	args = push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
-	if (args == NULL) {
-		return -1;
-	}
 #endif
+	}
 
 
 	/* Set the base virtual address - it must be an address that is not in the

lib/event/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 3
+SO_VER := 4
 SO_MINOR := 0
 
 LIBNAME = event

lib/event/app.c

@@ -110,7 +110,7 @@ static const struct option g_cmdline_options[] = {
 	{"version",			no_argument,		NULL, VERSION_OPT_IDX},
 #define PCI_BLACKLIST_OPT_IDX	'B'
 	{"pci-blacklist",		required_argument,	NULL, PCI_BLACKLIST_OPT_IDX},
-#define LOGFLAG_OPT_IDX	'L'
+#define LOGFLAG_OPT_IDX		'L'
 	{"logflag",			required_argument,	NULL, LOGFLAG_OPT_IDX},
 #define HUGE_UNLINK_OPT_IDX	'R'
 	{"huge-unlink",			no_argument,		NULL, HUGE_UNLINK_OPT_IDX},
@@ -130,6 +130,8 @@ static const struct option g_cmdline_options[] = {
 	{"json",			required_argument,	NULL, JSON_CONFIG_OPT_IDX},
 #define JSON_CONFIG_IGNORE_INIT_ERRORS_IDX	263
 	{"json-ignore-init-errors",	no_argument,		NULL, JSON_CONFIG_IGNORE_INIT_ERRORS_IDX},
+#define IOVA_MODE_OPT_IDX	264
+	{"iova-mode",			required_argument,	NULL, IOVA_MODE_OPT_IDX},
 };
 
 /* Global section */
@@ -507,6 +509,7 @@ app_setup_env(struct spdk_app_opts *opts)
 	env_opts.pci_blacklist = opts->pci_blacklist;
 	env_opts.pci_whitelist = opts->pci_whitelist;
 	env_opts.env_context = opts->env_context;
+	env_opts.iova_mode = opts->iova_mode;
 
 	rc = spdk_env_init(&env_opts);
 	free(env_opts.pci_blacklist);
@@ -761,6 +764,7 @@ usage(void (*app_usage)(void))
 	printf(" -W, --pci-whitelist <bdf>\n");
 	printf("                           pci addr to whitelist (-B and -W cannot be used at the same time)\n");
 	printf("      --huge-dir <path>    use a specific hugetlbfs mount to reserve memory from\n");
+	printf("      --iova-mode <pa/va>  set IOVA mode ('pa' for IOVA_PA and 'va' for IOVA_VA)\n");
 	printf("      --num-trace-entries <num>   number of trace entries for each core, must be power of 2. (default %d)\n",
 	       SPDK_APP_DEFAULT_NUM_TRACE_ENTRIES);
 	spdk_log_usage(stdout, "-L");
@@ -974,6 +978,9 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts,
 		case HUGE_DIR_OPT_IDX:
 			opts->hugedir = optarg;
 			break;
+		case IOVA_MODE_OPT_IDX:
+			opts->iova_mode = optarg;
+			break;
 		case NUM_TRACE_ENTRIES_OPT_IDX:
 			tmp = spdk_strtoll(optarg, 0);
 			if (tmp <= 0) {