ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

env_dpdk: add dpdk_pci_device_get_mem_resource

Browse Source 
This allows eliminating dpdk_pci_device_vtophys and
dpdk_pci_device_map_bar, reducing the amount of
code we need to maintain in the per-DPDK version
implementations.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I73d15eb75bf7fe8340d85494425e15651fec5425
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14722
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Krzysztof Karas <krzysztof.karas@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
This commit is contained in:
Jim Harris 2022-09-27 17:12:19 +00:00 committed by Tomasz Zawadzki
parent 5be703ef35
commit c7f5010984
6 changed files with 57 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
lib/env_dpdk/memory.c
View File

@ -11,6 +11,8 @@
#include <rte_config.h>
#include <rte_memory.h>
#include <rte_eal_memconfig.h>
#include <rte_dev.h>
#include <rte_pci.h>
#include "spdk_internal/assert.h"
@ -978,6 +980,38 @@ vtophys_get_paddr_pagemap(uint64_t vaddr)
return paddr;
}
static uint64_t
pci_device_vtophys(struct rte_pci_device *dev, uint64_t vaddr, size_t len)
{
struct rte_mem_resource *res;
uint64_t paddr;
unsigned r;
for (r = 0; r < PCI_MAX_RESOURCE; r++) {
res = dpdk_pci_device_get_mem_resource(dev, r);
if (res->phys_addr == 0 || vaddr < (uint64_t)res->addr ||
(vaddr + len) >= (uint64_t)res->addr + res->len) {
continue;
}
#if VFIO_ENABLED
if (spdk_iommu_is_enabled() && rte_eal_iova_mode() == RTE_IOVA_VA) {
/*
* The IOMMU is on and we're using IOVA == VA. The BAR was
* automatically registered when it was mapped, so just return
* the virtual address here.
*/
return vaddr;
}
#endif
paddr = res->phys_addr + (vaddr - (uint64_t)res->addr);
return paddr;
}
return SPDK_VTOPHYS_ERROR;
}
/* Try to get the paddr from pci devices */
static uint64_t
vtophys_get_paddr_pci(uint64_t vaddr, size_t len)
@ -989,7 +1023,7 @@ vtophys_get_paddr_pci(uint64_t vaddr, size_t len)
pthread_mutex_lock(&g_vtophys_pci_devices_mutex);
TAILQ_FOREACH(vtophys_dev, &g_vtophys_pci_devices, tailq) {
dev = vtophys_dev->pci_device;
paddr = dpdk_pci_device_vtophys(dev, vaddr, len);
paddr = pci_device_vtophys(dev, vaddr, len);
if (paddr != SPDK_VTOPHYS_ERROR) {
pthread_mutex_unlock(&g_vtophys_pci_devices_mutex);
return paddr;

9
lib/env_dpdk/pci.c
View File

@ -70,7 +70,14 @@ static int
map_bar_rte(struct spdk_pci_device *device, uint32_t bar,
void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
{
return dpdk_pci_device_map_bar(device->dev_handle, bar, mapped_addr, phys_addr, size);
struct rte_mem_resource *res;
res = dpdk_pci_device_get_mem_resource(device->dev_handle, bar);
*mapped_addr = res->addr;
*phys_addr = (uint64_t)res->phys_addr;
*size = (uint64_t)res->len;
return 0;
}
static int

13
lib/env_dpdk/pci_dpdk.c
View File

@ -43,10 +43,10 @@ dpdk_pci_init(void)
return 0;
}
uint64_t
dpdk_pci_device_vtophys(struct rte_pci_device *dev, uint64_t vaddr, size_t len)
struct rte_mem_resource *
dpdk_pci_device_get_mem_resource(struct rte_pci_device *dev, uint32_t bar)
{
return g_dpdk_fn_table->pci_device_vtophys(dev, vaddr, len);
return g_dpdk_fn_table->pci_device_get_mem_resource(dev, bar);
}
const char *
@ -79,13 +79,6 @@ dpdk_pci_device_get_numa_node(struct rte_pci_device *_dev)
return g_dpdk_fn_table->pci_device_get_numa_node(_dev);
}
int
dpdk_pci_device_map_bar(struct rte_pci_device *dev, uint32_t bar,
void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
{
return g_dpdk_fn_table->pci_device_map_bar(dev, bar, mapped_addr, phys_addr, size);
}
int
dpdk_pci_device_read_config(struct rte_pci_device *dev, void *value, uint32_t len, uint32_t offset)
{

7
lib/env_dpdk/pci_dpdk.h
View File

@ -26,14 +26,12 @@ struct rte_pci_driver;
struct rte_device;
struct dpdk_fn_table {
uint64_t (*pci_device_vtophys)(struct rte_pci_device *dev, uint64_t vaddr, size_t len);
struct rte_mem_resource *(*pci_device_get_mem_resource)(struct rte_pci_device *dev, uint32_t bar);
const char *(*pci_device_get_name)(struct rte_pci_device *);
struct rte_devargs *(*pci_device_get_devargs)(struct rte_pci_device *);
struct rte_pci_addr *(*pci_device_get_addr)(struct rte_pci_device *);
struct rte_pci_id *(*pci_device_get_id)(struct rte_pci_device *);
int (*pci_device_get_numa_node)(struct rte_pci_device *_dev);
int (*pci_device_map_bar)(struct rte_pci_device *dev, uint32_t bar,
void **mapped_addr, uint64_t *phys_addr, uint64_t *size);
int (*pci_device_read_config)(struct rte_pci_device *dev, void *value, uint32_t len,
uint32_t offset);
int (*pci_device_write_config)(struct rte_pci_device *dev, void *value, uint32_t len,
@ -54,14 +52,13 @@ struct dpdk_fn_table {
int dpdk_pci_init(void);
struct rte_mem_resource *dpdk_pci_device_get_mem_resource(struct rte_pci_device *dev, uint32_t bar);
uint64_t dpdk_pci_device_vtophys(struct rte_pci_device *dev, uint64_t vaddr, size_t len);
const char *dpdk_pci_device_get_name(struct rte_pci_device *);
struct rte_devargs *dpdk_pci_device_get_devargs(struct rte_pci_device *);
struct rte_pci_addr *dpdk_pci_device_get_addr(struct rte_pci_device *);
struct rte_pci_id *dpdk_pci_device_get_id(struct rte_pci_device *);
int dpdk_pci_device_get_numa_node(struct rte_pci_device *_dev);
int dpdk_pci_device_map_bar(struct rte_pci_device *dev, uint32_t bar,
void **mapped_addr, uint64_t *phys_addr, uint64_t *size);
int dpdk_pci_device_read_config(struct rte_pci_device *dev, void *value, uint32_t len,
uint32_t offset);
int dpdk_pci_device_write_config(struct rte_pci_device *dev, void *value, uint32_t len,

47
lib/env_dpdk/pci_dpdk_2207.c
View File

@ -15,36 +15,15 @@ SPDK_STATIC_ASSERT(offsetof(struct spdk_pci_driver, driver_buf) == 0, "driver_bu
SPDK_STATIC_ASSERT(offsetof(struct spdk_pci_driver, driver) >= sizeof(struct rte_pci_driver),
"driver_buf not big enough");
static uint64_t
pci_device_vtophys_2207(struct rte_pci_device *dev, uint64_t vaddr, size_t len)
static struct rte_mem_resource *
pci_device_get_mem_resource_2207(struct rte_pci_device *dev, uint32_t bar)
{
struct rte_mem_resource *res;
uint64_t paddr;
unsigned r;
for (r = 0; r < PCI_MAX_RESOURCE; r++) {
res = &dev->mem_resource[r];
if (res->phys_addr == 0 || vaddr < (uint64_t)res->addr ||
(vaddr + len) >= (uint64_t)res->addr + res->len) {
continue;
if (bar >= PCI_MAX_RESOURCE) {
assert(false);
return NULL;
}
#if VFIO_ENABLED
if (spdk_iommu_is_enabled() && rte_eal_iova_mode() == RTE_IOVA_VA) {
/*
* The IOMMU is on and we're using IOVA == VA. The BAR was
* automatically registered when it was mapped, so just return
* the virtual address here.
*/
return vaddr;
}
#endif
paddr = res->phys_addr + (vaddr - (uint64_t)res->addr);
return paddr;
}
return SPDK_VTOPHYS_ERROR;
return &dev->mem_resource[bar];
}
static const char *
@ -77,17 +56,6 @@ pci_device_get_numa_node_2207(struct rte_pci_device *_dev)
return _dev->device.numa_node;
}
static int
pci_device_map_bar_2207(struct rte_pci_device *dev, uint32_t bar,
void **mapped_addr, uint64_t *phys_addr, uint64_t *size)
{
*mapped_addr = dev->mem_resource[bar].addr;
*phys_addr = (uint64_t)dev->mem_resource[bar].phys_addr;
*size = (uint64_t)dev->mem_resource[bar].len;
return 0;
}
static int
pci_device_read_config_2207(struct rte_pci_device *dev, void *value, uint32_t len, uint32_t offset)
{
@ -243,13 +211,12 @@ device_scan_allowed_2207(struct rte_device *dev)
}
struct dpdk_fn_table fn_table_2207 = {
.pci_device_vtophys = pci_device_vtophys_2207,
.pci_device_get_mem_resource = pci_device_get_mem_resource_2207,
.pci_device_get_name = pci_device_get_name_2207,
.pci_device_get_devargs = pci_device_get_devargs_2207,
.pci_device_get_addr = pci_device_get_addr_2207,
.pci_device_get_id = pci_device_get_id_2207,
.pci_device_get_numa_node = pci_device_get_numa_node_2207,
.pci_device_map_bar = pci_device_map_bar_2207,
.pci_device_read_config = pci_device_read_config_2207,
.pci_device_write_config = pci_device_write_config_2207,
.pci_driver_register = pci_driver_register_2207,

4
test/env/memory/memory_ut.c vendored
View File

@ -29,8 +29,8 @@ DEFINE_STUB(rte_vfio_noiommu_is_enabled, int, (void), 0);
DEFINE_STUB(rte_memseg_get_fd_thread_unsafe, int, (const struct rte_memseg *ms), 0);
DEFINE_STUB(rte_memseg_get_fd_offset_thread_unsafe, int,
(const struct rte_memseg *ms, size_t *offset), 0);
DEFINE_STUB(dpdk_pci_device_vtophys, uint64_t, (struct rte_pci_device *dev, uint64_t vaddr,
size_t len), 0);
DEFINE_STUB(dpdk_pci_device_get_mem_resource, struct rte_mem_resource *,
(struct rte_pci_device *dev, uint32_t bar), 0);
static int
test_mem_map_notify(void *cb_ctx, struct spdk_mem_map *map,