From dd7cd80c6d992f09a8fa40cc8953eecaadd9574f Mon Sep 17 00:00:00 2001 From: Ben Walker Date: Mon, 9 Dec 2019 14:33:09 -0700 Subject: [PATCH] env/dpdk: Detect DPDK's iova mode Match DPDK's iova assignment strategy so there are never any conflicts. Change-Id: I3863487f9bd247c40edbf0d0d3a8c880bdad1708 Signed-off-by: Ben Walker Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/477362 Tested-by: SPDK CI Jenkins Community-CI: SPDK CI Jenkins Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- lib/env_dpdk/memory.c | 79 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/lib/env_dpdk/memory.c b/lib/env_dpdk/memory.c index 492682164..5812c0c01 100644 --- a/lib/env_dpdk/memory.c +++ b/lib/env_dpdk/memory.c @@ -1004,13 +1004,16 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, if (paddr == SPDK_VTOPHYS_ERROR) { /* This is not an address that DPDK is managing. */ #if SPDK_VFIO_ENABLED - if (spdk_iommu_is_enabled()) { - /* We'll use the virtual address as the iova. DPDK - * currently uses physical addresses as the iovas (or counts - * up from 0 if it can't get physical addresses), so - * the range of user space virtual addresses and physical - * addresses will never overlap. - */ + enum rte_iova_mode iova_mode; + +#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) + iova_mode = rte_eal_iova_mode(); +#else + iova_mode = rte_eal_get_configuration()->iova_mode; +#endif + + if (spdk_iommu_is_enabled() && iova_mode == RTE_IOVA_VA) { + /* We'll use the virtual address as the iova to match DPDK. */ paddr = (uint64_t)vaddr; rc = vtophys_iommu_map_dma((uint64_t)vaddr, paddr, len); if (rc) { @@ -1062,6 +1065,17 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, DEBUG_PRINT("invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr); return -EINVAL; } +#if SPDK_VFIO_ENABLED + /* If the IOMMU is on, but DPDK is using iova-mode=pa, we want to register this memory + * with the IOMMU using the physical address to match. */ + if (spdk_iommu_is_enabled()) { + rc = vtophys_iommu_map_dma((uint64_t)vaddr, paddr, VALUE_2MB); + if (rc) { + DEBUG_PRINT("Unable to assign vaddr %p to paddr 0x%" PRIx64 "\n", vaddr, paddr); + return -EFAULT; + } + } +#endif rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr); if (rc != 0) { @@ -1101,13 +1115,50 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, */ if (spdk_iommu_is_enabled()) { uint64_t buffer_len = len; - paddr = spdk_mem_map_translate(map, (uint64_t)vaddr, &buffer_len); - if (buffer_len != len) { - return -EINVAL; - } - rc = vtophys_iommu_unmap_dma(paddr, len); - if (rc) { - return -EFAULT; + uint8_t *va = vaddr; + enum rte_iova_mode iova_mode; + +#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) + iova_mode = rte_eal_iova_mode(); +#else + iova_mode = rte_eal_get_configuration()->iova_mode; +#endif + /* + * In virtual address mode, the region is contiguous and can be done in + * one unmap. + */ + if (iova_mode == RTE_IOVA_VA) { + paddr = spdk_mem_map_translate(map, (uint64_t)va, &buffer_len); + if (buffer_len != len || paddr != (uintptr_t)va) { + DEBUG_PRINT("Unmapping %p with length %lu failed because " + "translation had address 0x%" PRIx64 " and length %lu\n", + va, len, paddr, buffer_len); + return -EINVAL; + } + rc = vtophys_iommu_unmap_dma(paddr, len); + if (rc) { + DEBUG_PRINT("Failed to iommu unmap paddr 0x%" PRIx64 "\n", paddr); + return -EFAULT; + } + } else if (iova_mode == RTE_IOVA_PA) { + /* Get paddr for each 2MB chunk in this address range */ + while (buffer_len > 0) { + paddr = spdk_mem_map_translate(map, (uint64_t)va, NULL); + + if (paddr == SPDK_VTOPHYS_ERROR || buffer_len < VALUE_2MB) { + DEBUG_PRINT("could not get phys addr for %p\n", va); + return -EFAULT; + } + + rc = vtophys_iommu_unmap_dma(paddr, VALUE_2MB); + if (rc) { + DEBUG_PRINT("Failed to iommu unmap paddr 0x%" PRIx64 "\n", paddr); + return -EFAULT; + } + + va += VALUE_2MB; + buffer_len -= VALUE_2MB; + } } } }