From 9aaccfe3d7c3e7d5cdbba789e165334163bf882d Mon Sep 17 00:00:00 2001 From: Pawel Wodkowski Date: Wed, 5 Apr 2017 19:50:47 +0200 Subject: [PATCH] vhost: enable VFIO Vhost needs to register memory given by guest in VFIO container to be able to do any DMA using this memory. Currently DPDK doesn't provide any interface to handle guest memory, so for now lets find container fd in /proc/self/fd/ directory and provide some VFIO internal API that finally should extend DPDK API. Change-Id: Iee9d496367ccd61219068fc0eadc17e786ff0731 Signed-off-by: Pawel Wodkowski --- doc/vhost/getting_started.md | 5 - lib/vhost/Makefile | 2 +- lib/vhost/vhost.c | 58 +++------ lib/vhost/vhost_iommu.c | 234 +++++++++++++++++++++++++++++++++++ lib/vhost/vhost_iommu.h | 61 +++++++++ 5 files changed, 316 insertions(+), 44 deletions(-) create mode 100644 lib/vhost/vhost_iommu.c create mode 100644 lib/vhost/vhost_iommu.h diff --git a/doc/vhost/getting_started.md b/doc/vhost/getting_started.md index 9cfa02a40..9db22a6ce 100644 --- a/doc/vhost/getting_started.md +++ b/doc/vhost/getting_started.md @@ -149,11 +149,6 @@ assigned to vm. # Known bugs and limitations {#vhost_bugs} -## VFIO driver is not supported -Currently, VFIO driver is not supported by vhost library. Please use UIO -driver if running SPDK vhost app. Otherwise any IO to physical device -(eg. IOAT or NVMe) will fail. Support for vhost with VFIO is in progress. - ## Hot plug is not supported Hot plug is not supported in vhost yet. Event queue path doesn't handle that case. While hot plug will be just ignored, hot removal might cause segmentation diff --git a/lib/vhost/Makefile b/lib/vhost/Makefile index e91528e72..605404880 100644 --- a/lib/vhost/Makefile +++ b/lib/vhost/Makefile @@ -38,7 +38,7 @@ CFLAGS += -I. CFLAGS += -Irte_vhost CFLAGS += $(ENV_CFLAGS) -C_SRCS = task.c vhost.c vhost_rpc.c +C_SRCS = task.c vhost.c vhost_rpc.c vhost_iommu.c LIBNAME = vhost diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index 3ed31e378..f3cad48d2 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -57,6 +57,7 @@ #include "spdk/vhost.h" #include "task.h" +#include "vhost_iommu.h" static uint32_t g_num_ctrlrs[RTE_MAX_LCORE]; @@ -72,6 +73,9 @@ static char dev_dirname[PATH_MAX] = ""; struct spdk_vaddr_region { void *vaddr; uint64_t len; + + uint64_t host_user_addr; + uint64_t host_user_size; }; /* @@ -647,6 +651,8 @@ add_vdev_cb(void *arg1, void *arg2) { struct spdk_vhost_scsi_ctrlr *vdev = arg1; struct virtio_memory_region *region; + struct spdk_vaddr_region *vregion; + uint64_t start, end, len; uint32_t i; for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { @@ -658,16 +664,20 @@ add_vdev_cb(void *arg1, void *arg2) SPDK_NOTICELOG("Started poller for vhost controller %s on lcore %d\n", vdev->name, vdev->lcore); vdev->nregions = vdev->dev->mem->nregions; for (i = 0; i < vdev->nregions; i++) { - uint64_t start, end, len; region = &vdev->dev->mem->regions[i]; start = FLOOR_2MB(region->mmap_addr); end = CEIL_2MB(region->mmap_addr + region->mmap_size); len = end - start; - vdev->region[i].vaddr = (void *)start; - vdev->region[i].len = len; - SPDK_NOTICELOG("Registering VM memory for vtophys translation - 0x%jx len:0x%jx\n", - start, len); - spdk_mem_register(vdev->region[i].vaddr, vdev->region[i].len); + vregion = &vdev->region[i]; + vregion->vaddr = (void *)start; + vregion->len = len; + vregion->host_user_addr = region->host_user_addr; + vregion->host_user_size = region->size; + + SPDK_NOTICELOG("Registering VM memory for vtophys translation - %p len:0x%jx\n", + vdev->region[i].vaddr, vdev->region[i].len); + spdk_mem_register(vregion->vaddr, vregion->len); + spdk_iommu_mem_register(vregion->host_user_addr, vregion->host_user_size); } spdk_poller_register(&vdev->requestq_poller, vdev_worker, vdev, vdev->lcore, 0); @@ -680,6 +690,7 @@ static void remove_vdev_cb(void *arg1, void *arg2) { struct spdk_vhost_scsi_ctrlr *vdev = arg1; + struct spdk_vaddr_region *reg; uint32_t i; for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) { @@ -691,7 +702,9 @@ remove_vdev_cb(void *arg1, void *arg2) SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", vdev->name); for (i = 0; i < vdev->nregions; i++) { - spdk_mem_unregister(vdev->region[i].vaddr, vdev->region[i].len); + reg = &vdev->region[i]; + spdk_iommu_mem_unregister(reg->host_user_addr, reg->host_user_size); + spdk_mem_unregister(reg->vaddr, reg->len); } vdev->nregions = 0; @@ -1123,28 +1136,6 @@ session_start(void *arg) return NULL; } -/** - * Check if /sys/kernel/iommu_groups exists and is not empty - */ -static bool -has_iommu_groups(void) -{ - struct dirent *d; - int iommu_count = 0; - DIR *dir = opendir("/sys/kernel/iommu_groups"); - - if (dir == NULL) { - return false; - } - - while (iommu_count < 3 && (d = readdir(dir)) != NULL) { - ++iommu_count; - } - - closedir(dir); - return iommu_count > 2; /* there will always be ./ and ../ entries */ -} - void spdk_vhost_startup(void *arg1, void *arg2) { @@ -1168,15 +1159,6 @@ spdk_vhost_startup(void *arg1, void *arg2) if (ret != 0) rte_exit(EXIT_FAILURE, "Cannot construct vhost controllers\n"); - if (has_iommu_groups()) { - RTE_LOG(WARNING, VHOST_CONFIG, - "Currently VFIO driver is not supported by vhost library\n" - "Although guest might be able to boot and see devices, it will not be able\n" - "to do any IO to physical devices (Malloc with IOAT, NVMe).\n" - "Please use uio_pci_generic driver with vhost app/library.\n" - "See documentation for more information.\n"); - } - rte_vhost_driver_callback_register(&virtio_net_device_ops); if (pthread_create(&tid, NULL, &session_start, NULL) < 0) diff --git a/lib/vhost/vhost_iommu.c b/lib/vhost/vhost_iommu.c new file mode 100644 index 000000000..977139ba4 --- /dev/null +++ b/lib/vhost/vhost_iommu.c @@ -0,0 +1,234 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include "spdk_internal/log.h" +#include "vhost_iommu.h" + +static struct { + int need_init; + int container_fd; +} vfio_cfg = { 1, -1 }; + +/* Internal DPDK function forward declaration */ +int pci_vfio_is_enabled(void); + +/* Discover DPDK vfio container fd. This is to be removed if DPDK API + * provides interface for memory registration in VFIO container. + * + * Return -1 on error, 0 on success (VFIO is used or not) + */ +static int +vfio_cfg_init(void) +{ + char proc_fd_path[PATH_MAX + 1]; + char link_path[PATH_MAX + 1]; + const char vfio_path[] = "/dev/vfio/vfio"; + const int vfio_path_len = sizeof(vfio_path) - 1; + DIR *dir; + struct dirent *d; + + if (!vfio_cfg.need_init) { + return 0; + } + + vfio_cfg.need_init = 0; + if (!pci_vfio_is_enabled()) { + return 0; + } + + dir = opendir("/proc/self/fd"); + while ((d = readdir(dir)) != NULL) { + if (d->d_type != DT_LNK) + continue; + + snprintf(proc_fd_path, sizeof(proc_fd_path), "/proc/self/fd/%s", d->d_name); + if (readlink(proc_fd_path, link_path, sizeof(link_path)) != vfio_path_len) + continue; + + if (memcmp(link_path, vfio_path, vfio_path_len) == 0) { + sscanf(d->d_name, "%d", &vfio_cfg.container_fd); + break; + } + } + + closedir(dir); + + if (vfio_cfg.container_fd < 0) { + SPDK_ERRLOG("Failed to discover DPDK VFIO container fd.\n"); + return -1; + } + + return 0; +} + +static int +vfio_pci_memory_region_map(int vfio_container_fd, uint64_t vaddr, uint64_t phys_addr, uint64_t size) +{ + struct vfio_iommu_type1_dma_map dma_map; + int ret; + + dma_map.argsz = sizeof(dma_map); + dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + dma_map.vaddr = vaddr; + dma_map.iova = phys_addr; + dma_map.size = size; + + SPDK_TRACELOG(SPDK_TRACE_VHOST_VFIO, "MAP vaddr:%p phys:%p len:%#"PRIx64"\n", (void *)vaddr, + (void *)phys_addr, size); + ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map); + + if (ret) { + SPDK_ERRLOG("Cannot set up DMA mapping, error %d (%s)\n", errno, strerror(errno)); + } + + return ret; +} + +static int +vfio_pci_memory_region_unmap(int vfio_container_fd, uint64_t phys_addr, uint64_t size) +{ + struct vfio_iommu_type1_dma_unmap dma_unmap; + int ret; + + dma_unmap.argsz = sizeof(dma_unmap); + dma_unmap.flags = 0; + dma_unmap.iova = phys_addr; + dma_unmap.size = size; + + SPDK_TRACELOG(SPDK_TRACE_VHOST_VFIO, "UNMAP phys:%p len:%#"PRIx64"\n", (void *)phys_addr, size); + ret = ioctl(vfio_container_fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap); + + if (ret) { + SPDK_ERRLOG("Cannot clear DMA mapping, error %d (%s)\n", errno, strerror(errno)); + } + + return ret; +} + +static int +vfio_pci_memory_region_op(uint64_t vaddr, uint64_t phys_addr, uint64_t size, int op) +{ + + if (vfio_cfg.container_fd == -1) { + return 0; + } + + if (op == VFIO_IOMMU_MAP_DMA) { + return vfio_pci_memory_region_map(vfio_cfg.container_fd, vaddr, phys_addr, size); + } else { + return vfio_pci_memory_region_unmap(vfio_cfg.container_fd, phys_addr, size); + } +} + + +#define SHIFT_2MB 21 /* (1 << 21) == 2MB */ +#define MASK_2MB ((1ULL << SHIFT_2MB) - 1) + +static int +spdk_vfio_mem_op(uint64_t addr, uint64_t len, int dma_op) +{ + const uint64_t len_2mb = 1 << SHIFT_2MB; + uint64_t vaddr, vend, phaddr, phend, vlen; + int ret = 0; + + if (vfio_cfg_init() != 0) { + return -1; + } + + if (vfio_cfg.container_fd == -1) { + return 0; + } + + vaddr = addr; + while (len > 0) { + vlen = spdk_min(len_2mb - (vaddr & MASK_2MB), len); + vend = vaddr + vlen; + + phaddr = spdk_vtophys((void *)vaddr); + phend = spdk_vtophys((void *)(vend - 1)); + + if (phaddr == SPDK_VTOPHYS_ERROR || phend == SPDK_VTOPHYS_ERROR || + phend - phaddr > vlen - 1) { + SPDK_ERRLOG("Invalid memory region addr: %p len:%"PRIu64" " + "spdk_vtophys(%p) = %p spdk_vtophys(%p) = %p\n", + (void *)addr, len, (void *)vaddr, (void *)phaddr, + (void *)vend, (void *)phend); + ret = -1; + break; + } + + ret = vfio_pci_memory_region_op(vaddr, phaddr, vlen, dma_op); + if (ret) { + SPDK_ERRLOG("Failed to %s region region vaddr=%p phys_addr=%p len=%#"PRIx64"\n", + (dma_op == VFIO_IOMMU_MAP_DMA ? "map" : "unmap"), (void *)vaddr, + (void *)phaddr, vlen); + break; + } + + vaddr += vlen; + len -= vlen; + + assert(len == 0 || (vaddr & MASK_2MB) == 0); + } + + if (ret) { + spdk_vfio_mem_op(addr, vaddr - addr, VFIO_IOMMU_UNMAP_DMA); + } + + return ret; +} + +int spdk_iommu_mem_register(uint64_t addr, uint64_t len) +{ + return spdk_vfio_mem_op(addr, len, VFIO_IOMMU_MAP_DMA); +} + +int spdk_iommu_mem_unregister(uint64_t addr, uint64_t len) +{ + return spdk_vfio_mem_op(addr, len, VFIO_IOMMU_UNMAP_DMA); +} + +SPDK_LOG_REGISTER_TRACE_FLAG("vhost_vfio", SPDK_TRACE_VHOST_VFIO) diff --git a/lib/vhost/vhost_iommu.h b/lib/vhost/vhost_iommu.h new file mode 100644 index 000000000..9739c6925 --- /dev/null +++ b/lib/vhost/vhost_iommu.h @@ -0,0 +1,61 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_VHOST_IOMMU_H +#define SPDK_VHOST_IOMMU_H + +#include + +/** + * Register given memory block in currently used IOMMU. If no IOMMU is used this + * function do nothing but still should be called. + * + * \param addr Start of memory block + * \param len Length of memory block. + * \return 0 on success, -1 on error. + */ +int spdk_iommu_mem_register(uint64_t addr, uint64_t len); + +/** + * Unregister previously registered memory block in currently used IOMMU. If no + * IOMMU is used this function do nothing but still should be called. + * + * \note This functiom might fail for invalid memory block. + * + * \param addr Start of memory block + * \param len Length of memory block. + * \return 0 on success, -1 on error. + */ +int spdk_iommu_mem_unregister(uint64_t addr, uint64_t len); + +#endif /* SPDK_VHOST_IOMMU_H */