From 79b5618168b2b56a4ae71a4b6f10dfaccc63c19e Mon Sep 17 00:00:00 2001 From: Darek Stojaczyk Date: Wed, 3 Jul 2019 05:21:16 +0000 Subject: [PATCH] env_dpdk/pci: don't defer device detach while on the dpdk thread By making dpdk device detach asynchronous we have actually broken some cases where devices are re-attached immediately after and fail since they were not detached yet. We'll need to make detach synchronous again, and for that we'll wait for the background dpdk thread to perform all necessary actions before we return from spdk_pci_device_detach(). However, device detach could be triggered from the very same dpdk background thread as well. Waiting there would cause a deadlock, so now we'll schedule asynchronous device detach to the dpdk thread only if we're not on that thread already. This patch itself serves also as an optimization. Change-Id: I86b7ac1b669169eee3325de2d28c2cc313e7d901 Signed-off-by: Darek Stojaczyk Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/460285 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Shuhei Matsumoto --- lib/env_dpdk/pci.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/env_dpdk/pci.c b/lib/env_dpdk/pci.c index 25695ad7c..d8459d485 100644 --- a/lib/env_dpdk/pci.c +++ b/lib/env_dpdk/pci.c @@ -46,6 +46,8 @@ */ #define DPDK_HOTPLUG_RETRY_COUNT 4 +/* DPDK alarm/interrupt thread */ +static pthread_t g_dpdk_tid; static pthread_mutex_t g_pci_mutex = PTHREAD_MUTEX_INITIALIZER; static TAILQ_HEAD(, spdk_pci_device) g_pci_devices = TAILQ_HEAD_INITIALIZER(g_pci_devices); /* devices hotplugged on a dpdk thread */ @@ -125,7 +127,7 @@ spdk_detach_rte(struct spdk_pci_device *dev) * again while we go asynchronous, so we explicitly forbid that. */ dev->internal.pending_removal = true; - if (spdk_process_is_primary()) { + if (spdk_process_is_primary() && !pthread_equal(g_dpdk_tid, pthread_self())) { rte_eal_alarm_set(10, spdk_detach_rte_cb, dev->dev_handle); } else { spdk_detach_rte_cb(dev->dev_handle); @@ -198,6 +200,12 @@ cleanup_pci_devices(void) pthread_mutex_unlock(&g_pci_mutex); } +static void +_get_alarm_thread_cb(void *unused) +{ + g_dpdk_tid = pthread_self(); +} + void spdk_pci_init(void) { @@ -230,6 +238,11 @@ spdk_pci_init(void) rte_dev_event_callback_register(NULL, spdk_pci_device_rte_hotremove, NULL); } #endif + + rte_eal_alarm_set(1, _get_alarm_thread_cb, NULL); + /* alarms are executed in order, so this one will be always executed + * before any real hotremove alarms and we don't need to wait for it. + */ } void