rte_virtio: convert to spdk pci_virtio layer

Replaced direct DPDK calls with SPDK env/pci
equivalents. This patch also makes the PCI scan
happen only conditionally when user has
configured proper field in the bdev_virtio config
file.

Change-Id: Ib6ad81d0b421b20ad0cd9d02cb40b2213af823e6
Signed-off-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/379489
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>

lib/bdev/virtio/bdev_virtio.c

@@ -590,6 +590,7 @@ bdev_virtio_initialize(void)
 	char *type, *path;
 	uint32_t i;
 	int rc = 0;
+	bool scan_pci = false;
 
 	if (sp == NULL) {
 		goto out;
@@ -608,38 +609,42 @@ bdev_virtio_initialize(void)
 				continue;
 			}
 			vdev = virtio_user_dev_init(path, 1, 512);
+			if (vdev == NULL) {
+				goto out;
+			}
 		} else if (!strcmp("Pci", type)) {
-			vdev = get_pci_virtio_hw();
+			scan_pci = true;
 		} else {
 			SPDK_ERRLOG("Invalid type %s specified for index %d\n", type, i);
 			continue;
 		}
 	}
 
-	if (vdev == NULL) {
+	if (scan_pci) {
-		goto out;
+		vtpci_init();
 	}
 
-	base = spdk_dma_zmalloc(sizeof(*base), 64, NULL);
+	TAILQ_FOREACH(vdev, &g_virtio_driver.init_ctrlrs, tailq) {
-	if (base == NULL) {
+		base = spdk_dma_zmalloc(sizeof(*base), 64, NULL);
-		SPDK_ERRLOG("couldn't allocate memory for scsi target scan.\n");
+		if (base == NULL) {
-		rc = -1;
+			SPDK_ERRLOG("couldn't allocate memory for scsi target scan.\n");
-		goto out;
+			rc = -1;
+			goto out;
+		}
+
+		/* TODO check rc, add virtio_dev_deinit() */
+		virtio_init_device(vdev, VIRTIO_SCSI_DEV_SUPPORTED_FEATURES);
+		virtio_dev_start(vdev);
+
+		base->vdev = vdev;
+		TAILQ_INIT(&base->found_disks);
+
+		spdk_bdev_poller_start(&base->scan_poller, bdev_scan_poll, base,
+				       spdk_env_get_current_core(), 0);
+
+		scan_target(base);
 	}
 
-	/* TODO check rc, add virtio_dev_deinit() */
-	virtio_init_device(vdev, VIRTIO_SCSI_DEV_SUPPORTED_FEATURES);
-	virtio_dev_start(vdev);
-
-	TAILQ_INSERT_TAIL(&g_virtio_driver.init_ctrlrs, vdev, tailq);
-
-	base->vdev = vdev;
-	TAILQ_INIT(&base->found_disks);
-
-	spdk_bdev_poller_start(&base->scan_poller, bdev_scan_poll, base,
-			       spdk_env_get_current_core(), 0);
-
-	scan_target(base);
 	return 0;
 
 out:

lib/bdev/virtio/rte_virtio/virtio_dev.c

@@ -58,14 +58,6 @@
 #include "virtio_logs.h"
 #include "virtio_queue.h"
 
-/*
- * The set of PCI devices this driver supports
- */
-static const struct rte_pci_id pci_id_virtio_map[] = {
-	{ RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_DEVICEID_SCSI_MODERN) },
-	{ .vendor_id = 0, /* sentinel */ },
-};
-
 static uint16_t
 virtio_get_nr_vq(struct virtio_dev *dev)
 {
@@ -353,70 +345,3 @@ virtio_dev_start(struct virtio_dev *vdev)
 
 	return 0;
 }
-
-static struct virtio_hw *g_pci_hw = NULL;
-
-struct virtio_dev *
-get_pci_virtio_hw(void)
-{
-	int ret;
-
-	printf("%s[%d] %p\n", __func__, __LINE__, g_pci_hw);
-	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
-		PMD_DRV_LOG(ERR, "rte secondary process support is not implemented yet");
-		return NULL;
-	}
-
-	ret = vtpci_init(g_pci_hw->pci_dev, &g_pci_hw->vdev);
-	if (ret)
-		return NULL;
-
-	return &g_pci_hw->vdev;
-}
-
-static int virtio_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
-	struct rte_pci_device *pci_dev)
-{
-	struct virtio_hw *hw;
-
-	hw = calloc(1, sizeof(*hw));
-	hw->vdev.is_hw = 1;
-	hw->pci_dev = (struct spdk_pci_device *) pci_dev;
-
-	g_pci_hw = hw;
-
-	printf("%s[%d]\n", __func__, __LINE__);
-	return 0;
-}
-
-static int virtio_pci_remove(struct rte_pci_device *pci_dev)
-{
-	printf("%s[%d]\n", __func__, __LINE__);
-	return 0;
-}
-
-static struct rte_pci_driver rte_virtio_pmd = {
-	.driver = {
-		.name = "net_virtio",
-	},
-	.id_table = pci_id_virtio_map,
-	.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
-	.probe = virtio_pci_probe,
-	.remove = virtio_pci_remove,
-};
-
-RTE_INIT(rte_virtio_pmd_init);
-static void
-rte_virtio_pmd_init(void)
-{
-	if (rte_eal_iopl_init() != 0) {
-		PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD");
-		return;
-	}
-
-	rte_pci_register(&rte_virtio_pmd);
-}
-
-RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__);
-RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map);
-RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio");

lib/bdev/virtio/rte_virtio/virtio_pci.c

@@ -689,28 +689,27 @@ next:
 	return 0;
 }
 
-/*
+static int
- * Return -1:
+pci_enum_virtio_probe_cb(void *ctx, struct spdk_pci_device *pci_dev)
- *   if there is error mapping with VFIO/UIO.
- *   if port map error when driver type is KDRV_NONE.
- *   if whitelisted but driver type is KDRV_UNKNOWN.
- * Return 1 if kernel driver is managing the device.
- * Return 0 on success.
- */
-int
-vtpci_init(struct spdk_pci_device *dev, struct virtio_dev *vdev)
 {
-	struct virtio_hw *hw = virtio_dev_get_hw(vdev);
+	struct virtio_hw *hw;
+	struct virtio_dev *vdev;
+
+	hw = calloc(1, sizeof(*hw));
+	vdev = &hw->vdev;
+	vdev->is_hw = 1;
+	hw->pci_dev = pci_dev;
 
 	/*
 	 * Try if we can succeed reading virtio pci caps, which exists
 	 * only on modern pci device. If failed, we fallback to legacy
 	 * virtio handling.
 	 */
-	if (virtio_read_caps(dev, hw) == 0) {
+	if (virtio_read_caps(pci_dev, hw) == 0) {
 		PMD_INIT_LOG(INFO, "modern virtio pci detected.");
 		VTPCI_OPS(vdev) = &modern_ops;
 		vdev->modern = 1;
+		TAILQ_INSERT_TAIL(&g_virtio_driver.init_ctrlrs, vdev, tailq);
 		return 0;
 	}
 
@@ -732,5 +731,17 @@ vtpci_init(struct spdk_pci_device *dev, struct virtio_dev *vdev)
 	VTPCI_OPS(vdev) = &legacy_ops;
 	vdev->modern   = 0;
 
+	TAILQ_INSERT_TAIL(&g_virtio_driver.init_ctrlrs, vdev, tailq);
 	return 0;
 }
+
+int
+vtpci_init(void)
+{
+	if (!spdk_process_is_primary()) {
+		PMD_INIT_LOG(INFO, "virtio_pci secondary process support is not implemented yet.");
+		return 0;
+	}
+
+	return spdk_pci_virtio_enumerate(pci_enum_virtio_probe_cb, NULL);
+}

lib/bdev/virtio/rte_virtio/virtio_pci.h

@@ -259,10 +259,7 @@ vtpci_with_feature(struct virtio_dev *dev, uint64_t bit)
 	return (dev->guest_features & (1ULL << bit)) != 0;
 }
 
-/*
+int vtpci_init(void);
- * Function declaration from virtio_pci.c
- */
-int vtpci_init(struct spdk_pci_device *dev, struct virtio_dev *vdev);
 void vtpci_reset(struct virtio_dev *);
 
 void vtpci_reinit_complete(struct virtio_dev *);

lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.c

@@ -272,6 +272,7 @@ virtio_user_dev_init(char *path, int queues, int queue_size)
 		goto err;
 	}
 
+	TAILQ_INSERT_TAIL(&g_virtio_driver.init_ctrlrs, vdev, tailq);
 	return vdev;
 
 err: