From 70c703bb4910d8165943efcda5a192f853bfb918 Mon Sep 17 00:00:00 2001 From: Nikos Dragazis Date: Wed, 6 Feb 2019 03:24:33 +0200 Subject: [PATCH] vtophys: add vfio no-IOMMU mode support Currently, SPDK does not support vfio in no-IOMMU mode. However, it seems quite easy to extend the vtophys code to add support for this. vfio in no-IOMMU mode does not support DMA remapping. This implies that physical DMA addresses are used instead of IOVAs. This patch checks whether the vfio no-IOMMU mode is enabled using function rte_vfio_noiommu_is_enabled() from the DPDK RTE vfio interface. In this case, physical addresses are used for the DMA mappings. This is the same code path for the DMA translations as when the uio is used as a kernel driver. Change-Id: I6fb3c849a345c6f2f2b4141dddb8c17be2581495 Signed-off-by: Nikos Dragazis Reviewed-on: https://review.gerrithub.io/c/441061 Reviewed-by: Darek Stojaczyk Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins --- lib/env_dpdk/vtophys.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/env_dpdk/vtophys.c b/lib/env_dpdk/vtophys.c index 60b4d89af..b296ee798 100644 --- a/lib/env_dpdk/vtophys.c +++ b/lib/env_dpdk/vtophys.c @@ -77,6 +77,7 @@ struct spdk_vfio_dma_map { struct vfio_cfg { int fd; bool enabled; + bool noiommu_enabled; unsigned device_ref; TAILQ_HEAD(, spdk_vfio_dma_map) maps; pthread_mutex_t mutex; @@ -85,6 +86,7 @@ struct vfio_cfg { static struct vfio_cfg g_vfio = { .fd = -1, .enabled = false, + .noiommu_enabled = false, .device_ref = 0, .maps = TAILQ_HEAD_INITIALIZER(g_vfio.maps), .mutex = PTHREAD_MUTEX_INITIALIZER @@ -357,7 +359,7 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, if (paddr == SPDK_VTOPHYS_ERROR) { /* This is not an address that DPDK is managing. */ #if SPDK_VFIO_ENABLED - if (g_vfio.enabled) { + if (g_vfio.enabled && !g_vfio.noiommu_enabled) { /* We'll use the virtual address as the iova. DPDK * currently uses physical addresses as the iovas (or counts * up from 0 if it can't get physical addresses), so @@ -400,7 +402,7 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, * This is not an address that DPDK is managing. If vfio is enabled, * we need to unmap the range from the IOMMU */ - if (g_vfio.enabled) { + if (g_vfio.enabled && !g_vfio.noiommu_enabled) { uint64_t buffer_len = VALUE_2MB; paddr = spdk_mem_map_translate(map, (uint64_t)vaddr, &buffer_len); if (buffer_len != VALUE_2MB) { @@ -462,6 +464,12 @@ has_iommu_groups(void) return count > 2; } +static bool +spdk_vfio_noiommu_enabled(void) +{ + return rte_vfio_noiommu_is_enabled(); +} + static void spdk_vtophys_iommu_init(void) { @@ -471,7 +479,13 @@ spdk_vtophys_iommu_init(void) DIR *dir; struct dirent *d; - if (!spdk_vfio_enabled() || !has_iommu_groups()) { + if (!spdk_vfio_enabled()) { + return; + } + + if (spdk_vfio_noiommu_enabled()) { + g_vfio.noiommu_enabled = true; + } else if (!has_iommu_groups()) { return; }