bdev/virtio: add API to attach particular PCI devices

This is just the API introduction. See subsequent
patches for its RPC usage.

Change-Id: Iadb7c9bf6a56ab4330c9f2215c6006a2935d208d
Signed-off-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/394445
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

include/spdk_internal/virtio.h

@@ -46,6 +46,7 @@
 #include "spdk/json.h"
 #include "spdk/io_channel.h"
 #include "spdk/pci_ids.h"
+#include "spdk/env.h"
 
 /**
  * The maximum virtqueue size is 2^15. Use that value as the end of
@@ -444,6 +445,20 @@ void virtio_dev_dump_json_config(struct virtio_dev *vdev, struct spdk_json_write
 int virtio_pci_dev_enumerate(virtio_pci_create_cb enum_cb, void *enum_ctx,
 			     uint16_t pci_device_id);
 
+/**
+ * Attach a PCI Virtio device of given type.
+ *
+ * \param create_cb callback to create a virtio_dev.
+ * If virtio_dev is has been created, the callback should return 0.
+ * Returning any other value will cause the PCI context to be freed,
+ * making it unusable.
+ * \param enum_ctx additional opaque context to be passed into `enum_cb`
+ * \param pci_device_id PCI Device ID of devices to iterate through
+ * \param pci_addr PCI address of the device to attach
+ */
+int virtio_pci_dev_attach(virtio_pci_create_cb create_cb, void *enum_ctx,
+			  uint16_t pci_device_id, struct spdk_pci_addr *pci_addr);
+
 /**
  * Connect to a vhost-user device and init corresponding virtio_dev struct.
  * The virtio_dev will have to be freed with \c virtio_dev_free.

lib/bdev/virtio/bdev_virtio.h

@@ -35,6 +35,7 @@
 #define SPDK_BDEV_VIRTIO_H
 
 #include "spdk/bdev.h"
+#include "spdk/env.h"
 
 /**
  * Callback for creating virtio bdevs.
@@ -77,9 +78,27 @@ typedef void (*bdev_virtio_remove_cb)(void *ctx, int errnum);
  * \return zero on success (device scan is started) or negative error code.
  * In case of error the \c cb_fn is not called.
  */
-int bdev_virtio_scsi_dev_create(const char *name, const char *path,
+int bdev_virtio_user_scsi_dev_create(const char *name, const char *path,
-				unsigned num_queues, unsigned queue_size,
+				     unsigned num_queues, unsigned queue_size,
-				bdev_virtio_create_cb cb_fn, void *cb_arg);
+				     bdev_virtio_create_cb cb_fn, void *cb_arg);
+
+/**
+ * Attach virtio-pci device. This creates a Virtio SCSI device with the same
+ * capabilities as the vhost-user equivalent. The device will be automatically
+ * scanned for exposed SCSI targets. This will result in creating possibly multiple
+ * Virtio SCSI bdevs - one for each target. Currently only one LUN per target is
+ * detected - LUN0. Note that the bdev creation is run asynchronously in the
+ * background. After it's finished, the `cb_fn` callback is called.
+ *
+ * \param pci_addr PCI address of the device to attach
+ * \param cb_fn function to be called after scanning all targets on the virtio
+ * device. It's optional, can be NULL. See \c bdev_virtio_create_cb.
+ * \param cb_arg argument for the `cb_fn`
+ * \return zero on success (device scan is started) or negative error code.
+ * In case of error the \c cb_fn is not called.
+ */
+int bdev_virtio_pci_scsi_dev_create(struct spdk_pci_addr *pci_addr,
+				    bdev_virtio_create_cb cb_fn, void *cb_arg);
 
 /**
  * Remove a Virtio device with given name. This will destroy all bdevs exposed

lib/bdev/virtio/bdev_virtio_rpc.c

@@ -121,8 +121,8 @@ spdk_rpc_create_virtio_user_scsi_bdev(struct spdk_jsonrpc_request *request,
 	}
 
 	req->request = request;
-	rc = bdev_virtio_scsi_dev_create(req->name, req->path, req->vq_count, req->vq_size,
+	rc = bdev_virtio_user_scsi_dev_create(req->name, req->path, req->vq_count, req->vq_size,
-					 rpc_create_virtio_user_scsi_bdev_cb, req);
+					      rpc_create_virtio_user_scsi_bdev_cb, req);
 	if (rc < 0) {
 		goto invalid;
 	}

lib/bdev/virtio/bdev_virtio_scsi.c

@@ -292,8 +292,8 @@ virtio_scsi_dev_init(struct virtio_scsi_dev *svdev, uint16_t max_queues)
 	return 0;
 }
 
-static int
+static struct virtio_scsi_dev *
-virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
+virtio_pci_scsi_dev_create(struct virtio_pci_ctx *pci_ctx)
 {
 	static int pci_dev_counter = 0;
 	struct virtio_scsi_dev *svdev;
@@ -305,14 +305,14 @@ virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
 	svdev = calloc(1, sizeof(*svdev));
 	if (svdev == NULL) {
 		SPDK_ERRLOG("virtio device calloc failed\n");
-		return -1;
+		return NULL;
 	}
 
 	vdev = &svdev->vdev;
 	name = spdk_sprintf_alloc("VirtioScsi%"PRIu32, pci_dev_counter++);
 	if (name == NULL) {
 		free(vdev);
-		return -1;
+		return NULL;
 	}
 
 	rc = virtio_pci_dev_init(vdev, name, pci_ctx);
@@ -320,7 +320,7 @@ virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
 
 	if (rc != 0) {
 		free(svdev);
-		return -1;
+		return NULL;
 	}
 
 	virtio_dev_read_dev_config(vdev, offsetof(struct virtio_scsi_config, num_queues),
@@ -330,9 +330,10 @@ virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
 	if (rc != 0) {
 		virtio_dev_destruct(vdev);
 		free(svdev);
+		return NULL;
 	}
 
-	return rc;
+	return svdev;
 }
 
 static struct virtio_scsi_dev *
@@ -1443,6 +1444,15 @@ _virtio_scsi_dev_scan_next(struct virtio_scsi_scan_base *base)
 	return 0;
 }
 
+static int
+virtio_pci_scsi_dev_enumerate_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
+{
+	struct virtio_scsi_dev *svdev;
+
+	svdev = virtio_pci_scsi_dev_create(pci_ctx);
+	return svdev == NULL ? -1 : 0;
+}
+
 static int
 bdev_virtio_process_config(void)
 {
@@ -1502,7 +1512,7 @@ bdev_virtio_process_config(void)
 
 	enable_pci = spdk_conf_section_get_boolval(sp, "Enable", false);
 	if (enable_pci) {
-		rc = virtio_pci_dev_enumerate(virtio_pci_scsi_dev_create_cb, NULL,
+		rc = virtio_pci_dev_enumerate(virtio_pci_scsi_dev_enumerate_cb, NULL,
 					      PCI_DEVICE_ID_VIRTIO_SCSI_MODERN);
 	}
 
@@ -1790,8 +1800,9 @@ bdev_virtio_finish(void)
 }
 
 int
-bdev_virtio_scsi_dev_create(const char *base_name, const char *path, unsigned num_queues,
+bdev_virtio_user_scsi_dev_create(const char *base_name, const char *path,
-			    unsigned queue_size, bdev_virtio_create_cb cb_fn, void *cb_arg)
+				 unsigned num_queues, unsigned queue_size,
+				 bdev_virtio_create_cb cb_fn, void *cb_arg)
 {
 	struct virtio_scsi_dev *svdev;
 	int rc;
@@ -1809,6 +1820,44 @@ bdev_virtio_scsi_dev_create(const char *base_name, const char *path, unsigned nu
 	return rc;
 }
 
+struct bdev_virtio_pci_dev_create_ctx {
+	bdev_virtio_create_cb cb_fn;
+	void *cb_arg;
+};
+
+static int
+bdev_virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx, void *ctx)
+{
+	struct virtio_scsi_dev *svdev;
+	struct bdev_virtio_pci_dev_create_ctx *create_ctx = ctx;
+	int rc;
+
+	svdev = virtio_pci_scsi_dev_create(pci_ctx);
+	if (svdev == NULL) {
+		return -1;
+	}
+
+	rc = virtio_scsi_dev_scan(svdev, create_ctx->cb_fn, create_ctx->cb_arg);
+	if (rc) {
+		virtio_scsi_dev_remove(svdev, NULL, NULL);
+	}
+
+	return rc;
+}
+
+int
+bdev_virtio_pci_scsi_dev_create(struct spdk_pci_addr *pci_addr,
+				bdev_virtio_create_cb cb_fn, void *cb_arg)
+{
+	struct bdev_virtio_pci_dev_create_ctx create_ctx;
+
+	create_ctx.cb_fn = cb_fn;
+	create_ctx.cb_arg = cb_arg;
+
+	return virtio_pci_dev_attach(bdev_virtio_pci_scsi_dev_create_cb, &create_ctx,
+				     PCI_DEVICE_ID_VIRTIO_SCSI_MODERN, pci_addr);
+}
+
 void
 bdev_virtio_scsi_dev_remove(const char *name, bdev_virtio_remove_cb cb_fn, void *cb_arg)
 {

lib/virtio/virtio_pci.c

@@ -508,6 +508,24 @@ virtio_pci_dev_enumerate(virtio_pci_create_cb enum_cb, void *enum_ctx,
 	return spdk_pci_virtio_enumerate(virtio_pci_dev_probe_cb, &ctx);
 }
 
+int
+virtio_pci_dev_attach(virtio_pci_create_cb enum_cb, void *enum_ctx,
+		      uint16_t pci_device_id, struct spdk_pci_addr *pci_address)
+{
+	struct virtio_pci_probe_ctx ctx;
+
+	if (!spdk_process_is_primary()) {
+		SPDK_WARNLOG("virtio_pci secondary process support is not implemented yet.\n");
+		return 0;
+	}
+
+	ctx.enum_cb = enum_cb;
+	ctx.enum_ctx = enum_ctx;
+	ctx.device_id = pci_device_id;
+
+	return spdk_pci_virtio_device_attach(virtio_pci_dev_probe_cb, &ctx, pci_address);
+}
+
 int
 virtio_pci_dev_init(struct virtio_dev *vdev, const char *name,
 		    struct virtio_pci_ctx *pci_ctx)