From 9a59463b3bb8192ccc7927ac106ad8619067fc74 Mon Sep 17 00:00:00 2001 From: Darek Stojaczyk Date: Tue, 20 Nov 2018 15:05:13 +0100 Subject: [PATCH] pci: handle detaching a device in secondary processes Upon detaching a device in a secondary process, DPDK 18.11 will try to detach it from the primary process as well. SPDK doesn't support such hot-detach and will reject it in the primary process. That will cause the secondary process to also reject its detach. The device in the secondary process will be still there in DPDK, but for SPDK it will remain inaccessible - neither attach, nor enumerate will work on it. To fix it, we make our attach and enumerate functions always check the process local list of devices probed by DPDK, but not attached in SPDK. Looking at the patch from a different perspective, it simply introduces error handling for the DPDK detach function. If a device failed to detach, we'll now maintain it locally in SPDK to make it attach-able again. Change-Id: I8c509a571bea7a9fb413c9c2bfd64c62ad91074b Signed-off-by: Darek Stojaczyk Reviewed-on: https://review.gerrithub.io/434413 Tested-by: SPDK CI Jenkins Chandler-Test-Pool: SPDK Automated Test System Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- lib/env_dpdk/pci.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lib/env_dpdk/pci.c b/lib/env_dpdk/pci.c index 1c3fa5600..e4439345d 100644 --- a/lib/env_dpdk/pci.c +++ b/lib/env_dpdk/pci.c @@ -137,6 +137,7 @@ spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx, spdk_pci_enum_cb enum_cb, void *enum_ctx, struct spdk_pci_addr *pci_address) { + struct spdk_pci_device *dev; int rc; #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) char bdf[32]; @@ -153,6 +154,26 @@ spdk_pci_device_attach(struct spdk_pci_enum_ctx *ctx, pthread_mutex_lock(&g_pci_mutex); + TAILQ_FOREACH(dev, &g_pci_devices, tailq) { + if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) { + break; + } + } + + if (dev != NULL && dev->enum_ctx == ctx) { + if (dev->attached) { + pthread_mutex_unlock(&g_pci_mutex); + return -1; + } + + rc = enum_cb(enum_ctx, dev); + if (rc == 0) { + dev->attached = true; + } + pthread_mutex_unlock(&g_pci_mutex); + return rc; + } + if (!ctx->is_registered) { ctx->is_registered = true; #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4) @@ -191,8 +212,25 @@ spdk_pci_enumerate(struct spdk_pci_enum_ctx *ctx, spdk_pci_enum_cb enum_cb, void *enum_ctx) { + struct spdk_pci_device *dev; + int rc; + pthread_mutex_lock(&g_pci_mutex); + TAILQ_FOREACH(dev, &g_pci_devices, tailq) { + if (dev->attached || dev->enum_ctx != ctx) { + continue; + } + + rc = enum_cb(enum_ctx, dev); + if (rc == 0) { + dev->attached = true; + } else if (rc < 0) { + pthread_mutex_unlock(&g_pci_mutex); + return -1; + } + } + if (!ctx->is_registered) { ctx->is_registered = true; #if RTE_VERSION >= RTE_VERSION_NUM(17, 05, 0, 4)