env/pci: add detach() callback to pci_device_provider

This makes it possible to notify other PCI device providers (VMD) that a
PCI device is no longer used.  The VMD will driver will unhook that
device and free any resources tied to it.

Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I42752afbb371a1d33972dac50fd679f68d05b597
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13887
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tom Nabarro <tom.nabarro@intel.com>

include/spdk/env.h

@@ -1182,6 +1182,15 @@ struct spdk_pci_device_provider {
 	 */
 	int (*attach_cb)(const struct spdk_pci_addr *addr);
 
+	/**
+	 * Callback executed to detach a given PCI device.  The provider to detach the device is
+	 * selected based on the type of the device and the name of the provider (i.e. dev->type ==
+	 * provider->name).
+	 *
+	 * \param dev PCI device to detach.
+	 */
+	void (*detach_cb)(struct spdk_pci_device *dev);
+
 	TAILQ_ENTRY(spdk_pci_device_provider) tailq;
 };
 

lib/env_dpdk/pci.c

@@ -121,6 +121,13 @@ detach_rte_cb(void *_dev)
 	remove_rte_dev(_dev);
 }
 
+/* if it's a physical device we need to deal with DPDK on
+ * a different process and we can't just unset one flag
+ * here. We also want to stop using any device resources
+ * so that the device isn't "in use" by the userspace driver
+ * once we detach it. This would allow attaching the device
+ * to a different process, or to a kernel driver like nvme.
+ */
 static void
 detach_rte(struct spdk_pci_device *dev)
 {
@@ -525,23 +532,23 @@ pci_device_fini(struct rte_pci_device *_dev)
 void
 spdk_pci_device_detach(struct spdk_pci_device *dev)
 {
+	struct spdk_pci_device_provider *provider;
+
 	assert(dev->internal.attached);
 
 	if (dev->internal.claim_fd >= 0) {
 		spdk_pci_device_unclaim(dev);
 	}
 
-	dev->internal.attached = false;
+	TAILQ_FOREACH(provider, &g_pci_device_providers, tailq) {
-	if (strcmp(dev->type, "pci") == 0) {
+		if (strcmp(dev->type, provider->name) == 0) {
-		/* if it's a physical device we need to deal with DPDK on
+			break;
-		 * a different process and we can't just unset one flag
-		 * here. We also want to stop using any device resources
-		 * so that the device isn't "in use" by the userspace driver
-		 * once we detach it. This would allow attaching the device
-		 * to a different process, or to a kernel driver like nvme.
-		 */
-		detach_rte(dev);
 		}
+	}
+
+	assert(provider != NULL);
+	dev->internal.attached = false;
+	provider->detach_cb(dev);
 
 	cleanup_pci_devices();
 }
@@ -633,6 +640,7 @@ pci_attach_rte(const struct spdk_pci_addr *addr)
 static struct spdk_pci_device_provider g_pci_rte_provider = {
 	.name = "pci",
 	.attach_cb = pci_attach_rte,
+	.detach_cb = detach_rte,
 };
 
 SPDK_PCI_REGISTER_DEVICE_PROVIDER(pci, &g_pci_rte_provider);

lib/vmd/vmd.c

@@ -1478,9 +1478,21 @@ vmd_attach_device(const struct spdk_pci_addr *addr)
 	return -ENODEV;
 }
 
+static void
+vmd_detach_device(struct spdk_pci_device *pci_dev)
+{
+	struct vmd_pci_device *dev = SPDK_CONTAINEROF(pci_dev, struct vmd_pci_device, pci);
+
+	assert(strcmp(spdk_pci_device_get_type(pci_dev), "vmd") == 0);
+	assert(vmd_find_device(&pci_dev->addr) != NULL);
+
+	vmd_remove_device(dev);
+}
+
 static struct spdk_pci_device_provider g_vmd_device_provider = {
 	.name = "vmd",
 	.attach_cb = vmd_attach_device,
+	.detach_cb = vmd_detach_device,
 };
 
 SPDK_PCI_REGISTER_DEVICE_PROVIDER(vmd, &g_vmd_device_provider);

test/env/pci/pci_ut.c

@@ -99,6 +99,25 @@ ut_enum_cb(void *ctx, struct spdk_pci_device *dev)
 	return 0;
 }
 
+static int
+ut_attach_cb(const struct spdk_pci_addr *addr)
+{
+	return -ENODEV;
+}
+
+static void
+ut_detach_cb(struct spdk_pci_device *dev)
+{
+}
+
+static struct spdk_pci_device_provider g_ut_provider = {
+	.name = "custom",
+	.attach_cb = ut_attach_cb,
+	.detach_cb = ut_detach_cb,
+};
+
+SPDK_PCI_REGISTER_DEVICE_PROVIDER(ut, &g_ut_provider);
+
 static void
 pci_hook_test(void)
 {