From 97b0f7733fee749f9355d019f14b6d9316b7ce0a Mon Sep 17 00:00:00 2001
From: Ben Walker <benjamin.walker@intel.com>
Date: Tue, 19 Nov 2019 11:28:14 -0700
Subject: [PATCH] env: Check supported iommu address width before using
 iova-mode=va

DPDK by default guesses that it should be using iova-mode=va
so that it can support running as an unprivileged user. However,
some systems (especially virtual machines) don't have an IOMMU capable
of handling the full virtual address space and DPDK doesn't
currently catch that. Add a check in SPDK and force iova-mode=pa
here.

Change-Id: Ib3a5691a584190feaab4b9064b5a500e361328f2
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/475149
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
---
 include/spdk/stdinc.h |  1 +
 lib/env_dpdk/init.c   | 84 ++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/include/spdk/stdinc.h b/include/spdk/stdinc.h
index f200223be..65820d58e 100644
--- a/include/spdk/stdinc.h
+++ b/include/spdk/stdinc.h
@@ -57,6 +57,7 @@ extern "C" {
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 #include <time.h>
 
 /* POSIX */
diff --git a/lib/env_dpdk/init.c b/lib/env_dpdk/init.c
index 58cb363ca..9fed7c00f 100644
--- a/lib/env_dpdk/init.c
+++ b/lib/env_dpdk/init.c
@@ -175,6 +175,75 @@ spdk_push_arg(char *args[], int *argcount, char *arg)
 	return tmp;
 }
 
+#if defined(__linux__) && defined(__x86_64__)
+
+/* TODO: Can likely get this value from rlimits in the future */
+#define SPDK_IOMMU_VA_REQUIRED_WIDTH 48
+#define VTD_CAP_MGAW_SHIFT 16
+#define VTD_CAP_MGAW_MASK (0x3F << VTD_CAP_MGAW_SHIFT)
+
+static int
+spdk_get_iommu_width(void)
+{
+	DIR *dir;
+	FILE *file;
+	struct dirent *entry;
+	char mgaw_path[64];
+	char buf[64];
+	char *end;
+	long long int val;
+	int width, tmp;
+
+	dir = opendir("/sys/devices/virtual/iommu/");
+	if (dir == NULL) {
+		return -EINVAL;
+	}
+
+	width = 0;
+
+	while ((entry = readdir(dir)) != NULL) {
+		/* Find directories named "dmar0", "dmar1", etc */
+		if (strncmp(entry->d_name, "dmar", sizeof("dmar") - 1) != 0) {
+			continue;
+		}
+
+		tmp = snprintf(mgaw_path, sizeof(mgaw_path), "/sys/devices/virtual/iommu/%s/intel-iommu/cap",
+			       entry->d_name);
+		if ((unsigned)tmp >= sizeof(mgaw_path)) {
+			continue;
+		}
+
+		file = fopen(mgaw_path, "r");
+		if (file == NULL) {
+			continue;
+		}
+
+		if (fgets(buf, sizeof(buf), file) == NULL) {
+			fclose(file);
+			continue;
+		}
+
+		val = strtoll(buf, &end, 16);
+		if (val == LLONG_MIN || val == LLONG_MAX) {
+			fclose(file);
+			continue;
+		}
+
+		tmp = ((val & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1;
+		if (width == 0 || tmp < width) {
+			width = tmp;
+		}
+
+		fclose(file);
+	}
+
+	closedir(dir);
+
+	return width;
+}
+
+#endif
+
 static int
 spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
 {
@@ -337,7 +406,19 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
 
 #ifdef __linux__
 
-#ifdef __PPC64__
+#if defined(__x86_64__)
+	/* DPDK by default guesses that it should be using iova-mode=va so that it can
+	 * support running as an unprivileged user. However, some systems (especially
+	 * virtual machines) don't have an IOMMU capable of handling the full virtual
+	 * address space and DPDK doesn't currently catch that. Add a check in SPDK
+	 * and force iova-mode=pa here. */
+	if (spdk_get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) {
+		args = spdk_push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
+		if (args == NULL) {
+			return -1;
+		}
+	}
+#elif defined(__PPC64__)
 	/* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly
 	 * auto-detect at the moment, so we'll just force it here. */
 	args = spdk_push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
@@ -346,6 +427,7 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
 	}
 #endif
 
+
 	/* Set the base virtual address - it must be an address that is not in the
 	 * ASAN shadow region, otherwise ASAN-enabled builds will ignore the
 	 * mmap hint.