diff --git a/lib/env_dpdk/env_internal.h b/lib/env_dpdk/env_internal.h index 3b3f4f462..872914f71 100644 --- a/lib/env_dpdk/env_internal.h +++ b/lib/env_dpdk/env_internal.h @@ -72,6 +72,12 @@ extern struct rte_pci_bus rte_pci_bus; #define MASK_4KB ((1ULL << SHIFT_4KB) - 1) #define VALUE_4KB (1 << SHIFT_4KB) +#define SPDK_PMD_REGISTER_PCI(pci_drv) \ +__attribute__((constructor)) static void pci_drv ## _register(void) \ +{ \ + spdk_pci_driver_register(&pci_drv); \ +} + struct spdk_pci_device { struct rte_pci_device *dev_handle; struct spdk_pci_driver *driver; @@ -83,12 +89,14 @@ struct spdk_pci_device { }; struct spdk_pci_driver { - struct rte_pci_driver driver; - spdk_pci_enum_cb cb_fn; - void *cb_arg; - bool is_registered; + struct rte_pci_driver driver; + spdk_pci_enum_cb cb_fn; + void *cb_arg; + bool is_registered; + TAILQ_ENTRY(spdk_pci_driver) tailq; }; +void spdk_pci_driver_register(struct spdk_pci_driver *driver); int spdk_pci_device_init(struct rte_pci_driver *driver, struct rte_pci_device *device); int spdk_pci_device_fini(struct rte_pci_device *device); @@ -96,6 +104,7 @@ int spdk_pci_enumerate(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb, int spdk_pci_device_attach(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb, void *enum_ctx, struct spdk_pci_addr *pci_address); +void spdk_pci_init(void); int spdk_mem_map_init(void); int spdk_vtophys_init(void); diff --git a/lib/env_dpdk/init.c b/lib/env_dpdk/init.c index 8cb3ef286..ddf04ada6 100644 --- a/lib/env_dpdk/init.c +++ b/lib/env_dpdk/init.c @@ -396,6 +396,8 @@ int spdk_env_init(const struct spdk_env_opts *opts) spdk_env_unlink_shared_files(); } + spdk_pci_init(); + if (spdk_mem_map_init() < 0) { fprintf(stderr, "Failed to allocate mem_map\n"); return -1; diff --git a/lib/env_dpdk/pci.c b/lib/env_dpdk/pci.c index 9aed2336e..8d395273e 100644 --- a/lib/env_dpdk/pci.c +++ b/lib/env_dpdk/pci.c @@ -42,6 +42,40 @@ static pthread_mutex_t g_pci_mutex = PTHREAD_MUTEX_INITIALIZER; static TAILQ_HEAD(, spdk_pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices); +static TAILQ_HEAD(, spdk_pci_driver) g_pci_drivers = TAILQ_HEAD_INITIALIZER(g_pci_drivers); + +void +spdk_pci_driver_register(struct spdk_pci_driver *driver) +{ + TAILQ_INSERT_TAIL(&g_pci_drivers, driver, tailq); +} + +void +spdk_pci_init(void) +{ +#if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0) + struct spdk_pci_driver *driver; + + /* We need to pre-register pci drivers for the pci devices to be + * attachable in multi-process with DPDK 18.11+. + * + * DPDK 18.11+ does its best to ensure all devices are equally + * attached or detached in all processes within a shared memory group. + * For SPDK it means that if a device is hotplugged in the primary, + * then DPDK will automatically send an IPC hotplug request to all other + * processes. Those other processes may not have the same SPDK PCI + * driver registered and may fail to attach the device. DPDK will send + * back the failure status, and the the primary process will also fail + * to hotplug the device. To prevent that, we need to pre-register the + * pci drivers here. + */ + TAILQ_FOREACH(driver, &g_pci_drivers, tailq) { + assert(!driver->is_registered); + driver->is_registered = true; + rte_pci_register(&driver->driver); + } +#endif +} int spdk_pci_device_init(struct rte_pci_driver *_drv, diff --git a/lib/env_dpdk/pci_ioat.c b/lib/env_dpdk/pci_ioat.c index 95359eee9..80d9e2498 100644 --- a/lib/env_dpdk/pci_ioat.c +++ b/lib/env_dpdk/pci_ioat.c @@ -114,3 +114,5 @@ spdk_pci_ioat_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx) { return spdk_pci_enumerate(&g_ioat_pci_drv, enum_cb, enum_ctx); } + +SPDK_PMD_REGISTER_PCI(g_ioat_pci_drv); diff --git a/lib/env_dpdk/pci_nvme.c b/lib/env_dpdk/pci_nvme.c index b46dda411..83da3e588 100644 --- a/lib/env_dpdk/pci_nvme.c +++ b/lib/env_dpdk/pci_nvme.c @@ -76,3 +76,5 @@ spdk_pci_nvme_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx) { return spdk_pci_enumerate(&g_nvme_pci_drv, enum_cb, enum_ctx); } + +SPDK_PMD_REGISTER_PCI(g_nvme_pci_drv); diff --git a/lib/env_dpdk/pci_virtio.c b/lib/env_dpdk/pci_virtio.c index 007dbc008..5ef92af62 100644 --- a/lib/env_dpdk/pci_virtio.c +++ b/lib/env_dpdk/pci_virtio.c @@ -71,3 +71,5 @@ spdk_pci_virtio_enumerate(spdk_pci_enum_cb enum_cb, void *enum_ctx) { return spdk_pci_enumerate(&g_virtio_pci_drv, enum_cb, enum_ctx); } + +SPDK_PMD_REGISTER_PCI(g_virtio_pci_drv);