From 35a331a949ab15ec010181d8c5aa97e520a7cfed Mon Sep 17 00:00:00 2001 From: Stephen Bates Date: Mon, 5 Feb 2018 17:25:14 -0700 Subject: [PATCH] vtophys: Add vtophys_get_paddr_pci() Add a new virtual to physical (vtophys) method for spdk_vtophys_notify() that works for PCI memory (namely NVMe CMBs and PMRs). This new method searches all the BARs on all the detected PCI devices to see if the vaddr resides inside any of them. Change-Id: I68afbeffd958cf40c1e8652e13da5531811b522b Signed-off-by: Stephen Bates Reviewed-on: https://review.gerrithub.io/398872 Reviewed-by: Jim Harris Reviewed-by: Dariusz Stojaczyk Reviewed-by: Daniel Verkamp Tested-by: SPDK Automated Test System --- lib/env_dpdk/vtophys.c | 52 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/lib/env_dpdk/vtophys.c b/lib/env_dpdk/vtophys.c index 92aa256d1..49cbe215e 100644 --- a/lib/env_dpdk/vtophys.c +++ b/lib/env_dpdk/vtophys.c @@ -240,12 +240,40 @@ vtophys_get_paddr_pagemap(uint64_t vaddr) return paddr; } +/* Try to get the paddr from pci devices */ +static uint64_t +vtophys_get_paddr_pci(uint64_t vaddr) +{ + uintptr_t paddr; + struct rte_pci_device *dev; + struct rte_mem_resource *res; + unsigned r; + +#if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 2) + FOREACH_DEVICE_ON_PCIBUS(dev) { +#else + TAILQ_FOREACH(dev, &pci_device_list, next) { +#endif + for (r = 0; r < PCI_MAX_RESOURCE; r++) { + res = &dev->mem_resource[r]; + if (res->phys_addr && vaddr >= (uint64_t)res->addr && + vaddr < (uint64_t)res->addr + res->len) { + paddr = res->phys_addr + (vaddr - (uint64_t)res->addr); + DEBUG_PRINT("%s: %p -> %p\n", __func__, (void *)vaddr, + (void *)paddr); + return paddr; + } + } + } + return SPDK_VTOPHYS_ERROR; +} + static int spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, enum spdk_mem_map_notify_action action, void *vaddr, size_t len) { - int rc = 0; + int rc = 0, pci_phys = 0; uint64_t paddr; if ((uintptr_t)vaddr & ~MASK_256TB) { @@ -286,13 +314,18 @@ spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, /* Get the physical address from /proc/self/pagemap. */ paddr = vtophys_get_paddr_pagemap((uint64_t)vaddr); if (paddr == SPDK_VTOPHYS_ERROR) { - DEBUG_PRINT("could not get phys addr for %p\n", vaddr); - return -EFAULT; + /* Get the physical address from PCI devices */ + paddr = vtophys_get_paddr_pci((uint64_t)vaddr); + if (paddr == SPDK_VTOPHYS_ERROR) { + DEBUG_PRINT("could not get phys addr for %p\n", vaddr); + return -EFAULT; + } + pci_phys = 1; } } } - - if (paddr & MASK_2MB) { + /* Since PCI paddr can break the 2MiB physical alginment skip this check for that. */ + if (!pci_phys && (paddr & MASK_2MB)) { DEBUG_PRINT("invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr); return -EINVAL; } @@ -476,8 +509,13 @@ spdk_vtophys(void *buf) /* * SPDK_VTOPHYS_ERROR has all bits set, so if the lookup returned SPDK_VTOPHYS_ERROR, * we will still bitwise-or it with the buf offset below, but the result will still be - * SPDK_VTOPHYS_ERROR. + * SPDK_VTOPHYS_ERROR. However now that we do + rather than | (due to PCI vtophys being + * unaligned) we must now check the return value before addition. */ SPDK_STATIC_ASSERT(SPDK_VTOPHYS_ERROR == UINT64_C(-1), "SPDK_VTOPHYS_ERROR should be all 1s"); - return paddr_2mb | ((uint64_t)buf & MASK_2MB); + if (paddr_2mb == SPDK_VTOPHYS_ERROR) { + return SPDK_VTOPHYS_ERROR; + } else { + return paddr_2mb + ((uint64_t)buf & MASK_2MB); + } }