From df4d03f10745cbf1cabaaf4ed1d00711dc83aaa5 Mon Sep 17 00:00:00 2001 From: Darek Stojaczyk Date: Sun, 2 Dec 2018 12:38:13 +0100 Subject: [PATCH] env: encapsulate spdk_pci_device fields In order to populate our PCI device list with devices located behind the VMD, we'll need to fill out those device structures from within a special VMD driver. That driver will base on PCI configuration and BAR accesses, but definitely not on DPDK. We want to put the VMD driver outside of the env lib, so we're about to provide it with a direct access to the device struct. Before we do that, let's group all the env-internal fields into an extra struct "internal". The spdk_pci_device struct does actually depend on DPDK now as it contains an `rte_pci_device *dev_handle` field, but we can easily break that dependency. The field is only used as an arguement to DPDK functions, so we can change its type to void* and let the implicit type conversion do the magic. After all, the VMD driver will potentially use it to store its (non-DPDK) data as well. Change-Id: I425d6dfa7af13e022f5377ceaff39efbd4a01b3d Signed-off-by: Darek Stojaczyk Reviewed-on: https://review.gerrithub.io/435799 Tested-by: SPDK CI Jenkins Chandler-Test-Pool: SPDK Automated Test System Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- lib/env_dpdk/env_internal.h | 11 +++++++---- lib/env_dpdk/pci.c | 31 ++++++++++++++++--------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/lib/env_dpdk/env_internal.h b/lib/env_dpdk/env_internal.h index 872914f71..2febefaaf 100644 --- a/lib/env_dpdk/env_internal.h +++ b/lib/env_dpdk/env_internal.h @@ -79,13 +79,16 @@ __attribute__((constructor)) static void pci_drv ## _register(void) \ } struct spdk_pci_device { - struct rte_pci_device *dev_handle; - struct spdk_pci_driver *driver; + void *dev_handle; struct spdk_pci_addr addr; struct spdk_pci_id id; int socket_id; - bool attached; - TAILQ_ENTRY(spdk_pci_device) tailq; + + struct _spdk_pci_device_internal { + struct spdk_pci_driver *driver; + bool attached; + TAILQ_ENTRY(spdk_pci_device) tailq; + } internal; }; struct spdk_pci_driver { diff --git a/lib/env_dpdk/pci.c b/lib/env_dpdk/pci.c index 0f3d4c53a..1532bd353 100644 --- a/lib/env_dpdk/pci.c +++ b/lib/env_dpdk/pci.c @@ -108,7 +108,6 @@ spdk_pci_device_init(struct rte_pci_driver *_drv, } dev->dev_handle = _dev; - dev->driver = driver; dev->addr.domain = _dev->addr.domain; dev->addr.bus = _dev->addr.bus; @@ -120,16 +119,18 @@ spdk_pci_device_init(struct rte_pci_driver *_drv, dev->id.subdevice_id = _dev->id.subsystem_device_id; dev->socket_id = _dev->device.numa_node; + dev->internal.driver = driver; + if (driver->cb_fn != NULL) { rc = driver->cb_fn(driver->cb_arg, dev); if (rc != 0) { free(dev); return rc; } - dev->attached = true; + dev->internal.attached = true; } - TAILQ_INSERT_TAIL(&g_pci_devices, dev, tailq); + TAILQ_INSERT_TAIL(&g_pci_devices, dev, internal.tailq); spdk_vtophys_pci_device_added(dev->dev_handle); return 0; } @@ -139,19 +140,19 @@ spdk_pci_device_fini(struct rte_pci_device *_dev) { struct spdk_pci_device *dev; - TAILQ_FOREACH(dev, &g_pci_devices, tailq) { + TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { if (dev->dev_handle == _dev) { break; } } - if (dev == NULL || dev->attached) { + if (dev == NULL || dev->internal.attached) { /* The device might be still referenced somewhere in SPDK. */ return -1; } spdk_vtophys_pci_device_removed(dev->dev_handle); - TAILQ_REMOVE(&g_pci_devices, dev, tailq); + TAILQ_REMOVE(&g_pci_devices, dev, internal.tailq); free(dev); return 0; @@ -162,8 +163,8 @@ spdk_pci_device_detach(struct spdk_pci_device *dev) { struct rte_pci_device *device = dev->dev_handle; - assert(dev->attached); - dev->attached = false; + assert(dev->internal.attached); + dev->internal.attached = false; #if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0) char bdf[32]; int i = 0, rc; @@ -204,21 +205,21 @@ spdk_pci_device_attach(struct spdk_pci_driver *driver, pthread_mutex_lock(&g_pci_mutex); - TAILQ_FOREACH(dev, &g_pci_devices, tailq) { + TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) { break; } } - if (dev != NULL && dev->driver == driver) { - if (dev->attached) { + if (dev != NULL && dev->internal.driver == driver) { + if (dev->internal.attached) { pthread_mutex_unlock(&g_pci_mutex); return -1; } rc = enum_cb(enum_ctx, dev); if (rc == 0) { - dev->attached = true; + dev->internal.attached = true; } pthread_mutex_unlock(&g_pci_mutex); return rc; @@ -278,14 +279,14 @@ spdk_pci_enumerate(struct spdk_pci_driver *driver, pthread_mutex_lock(&g_pci_mutex); - TAILQ_FOREACH(dev, &g_pci_devices, tailq) { - if (dev->attached || dev->driver != driver) { + TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) { + if (dev->internal.attached || dev->internal.driver != driver) { continue; } rc = enum_cb(enum_ctx, dev); if (rc == 0) { - dev->attached = true; + dev->internal.attached = true; } else if (rc < 0) { pthread_mutex_unlock(&g_pci_mutex); return -1;