From 8b260d5c923a367328bfe41eadfd6f494996a2dc Mon Sep 17 00:00:00 2001 From: Changpeng Liu Date: Thu, 21 Jul 2022 16:36:41 +0800 Subject: [PATCH] virtio/vfio_user: add virtio_scsi device support Example usage using `bdevperf` as the client: Start `spdk_tgt` with created virtio_scsi device: 1. scripts/rpc.py bdev_malloc_create -b malloc0 $((512)) 512 2. scripts/rpc.py vfu_virtio_create_scsi_endpoint vfu.0 --cpumask 0x1 --num-io-queues=4 \ --qsize=128 --packed-ring 3. scripts/rpc.py vfu_virtio_scsi_add_target vfu.0 --scsi-target-num=0 --bdev-name malloc0 Start `bdevperf`: 1. test/bdev/bdevperf/bdevperf -r /var/tmp/spdk.sock.1 -g -s 2048 -q 128 -o 4096 \ -w randread -t 30 -m 0x2 2. scripts/rpc.py -s /var/tmp/spdk.sock.1 bdev_virtio_attach_controller --dev-type scsi \ --trtype vfio-user --traddr vfu.0 VirtioScsi0 3. test/bdev/bdevperf/bdevperf.py -s /var/tmp/spdk.sock.1 perform_tests Change-Id: Id95908a5243e9a224a9bd709a22b87740c50b835 Signed-off-by: Changpeng Liu Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13897 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Ben Walker Reviewed-by: Konrad Sztyber --- module/bdev/virtio/bdev_virtio.h | 21 +++++++++++ module/bdev/virtio/bdev_virtio_rpc.c | 6 ++-- module/bdev/virtio/bdev_virtio_scsi.c | 52 +++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/module/bdev/virtio/bdev_virtio.h b/module/bdev/virtio/bdev_virtio.h index 36b5a6e08..8bcb13e73 100644 --- a/module/bdev/virtio/bdev_virtio.h +++ b/module/bdev/virtio/bdev_virtio.h @@ -53,6 +53,27 @@ int bdev_virtio_user_scsi_dev_create(const char *name, const char *path, unsigned num_queues, unsigned queue_size, bdev_virtio_create_cb cb_fn, void *cb_arg); +/** + * Connect to a vfio-user Unix domain socket and create a Virtio SCSI device. + * If the connection is successful, the device will be automatically scanned. + * The scan consists of probing the targets on the device and will result in + * creating possibly multiple Virtio SCSI bdevs - one for each target. Currently + * only one LUN per target is detected - LUN0. Note that the bdev creation is + * run asynchronously in the background. After it's finished, the `cb_fn` + * callback is called. + * + * \param name name for the virtio device. It will be inherited by all created + * bdevs, which are named in the following format: t + * \param path path to the socket + * \param cb_fn function to be called after scanning all targets on the virtio + * device. It's optional, can be NULL. See \c bdev_virtio_create_cb. + * \param cb_arg argument for the `cb_fn` + * \return zero on success (device scan is started) or negative error code. + * In case of error the \c cb_fn is not called. + */ +int bdev_vfio_user_scsi_dev_create(const char *base_name, const char *path, + bdev_virtio_create_cb cb_fn, void *cb_arg); + /** * Attach virtio-pci device. This creates a Virtio SCSI device with the same * capabilities as the vhost-user equivalent. The device will be automatically diff --git a/module/bdev/virtio/bdev_virtio_rpc.c b/module/bdev/virtio/bdev_virtio_rpc.c index 17fed3ad3..8155fd5ea 100644 --- a/module/bdev/virtio/bdev_virtio_rpc.c +++ b/module/bdev/virtio/bdev_virtio_rpc.c @@ -186,7 +186,7 @@ rpc_bdev_virtio_attach_controller(struct spdk_jsonrpc_request *request, struct rpc_bdev_virtio_attach_controller_ctx *req; struct spdk_bdev *bdev = NULL; struct spdk_pci_addr pci_addr; - int rc; + int rc = 0; req = calloc(1, sizeof(*req)); if (!req) { @@ -248,9 +248,11 @@ rpc_bdev_virtio_attach_controller(struct spdk_jsonrpc_request *request, } else if (strcmp(req->dev_type, "scsi") == 0) { if (strcmp(req->trtype, "pci") == 0) { rc = bdev_virtio_pci_scsi_dev_create(req->name, &pci_addr, rpc_create_virtio_dev_cb, req); - } else { + } else if (strcmp(req->trtype, "user") == 0) { rc = bdev_virtio_user_scsi_dev_create(req->name, req->traddr, req->vq_count, req->vq_size, rpc_create_virtio_dev_cb, req); + } else if (strcmp(req->trtype, "vfio-user") == 0) { + rc = bdev_vfio_user_scsi_dev_create(req->name, req->traddr, rpc_create_virtio_dev_cb, req); } if (rc < 0) { diff --git a/module/bdev/virtio/bdev_virtio_scsi.c b/module/bdev/virtio/bdev_virtio_scsi.c index 308590c36..0bf9f7557 100644 --- a/module/bdev/virtio/bdev_virtio_scsi.c +++ b/module/bdev/virtio/bdev_virtio_scsi.c @@ -1806,6 +1806,58 @@ bdev_virtio_user_scsi_dev_create(const char *base_name, const char *path, return rc; } +int +bdev_vfio_user_scsi_dev_create(const char *base_name, const char *path, + bdev_virtio_create_cb cb_fn, void *cb_arg) +{ + struct virtio_scsi_dev *svdev; + uint32_t num_queues = 0; + int rc; + + svdev = calloc(1, sizeof(*svdev)); + if (svdev == NULL) { + SPDK_ERRLOG("calloc failed for virtio device %s: %s\n", base_name, path); + return -ENOMEM; + } + + rc = virtio_vfio_user_dev_init(&svdev->vdev, base_name, path); + if (rc != 0) { + SPDK_ERRLOG("Failed to create %s as virtio device\n", path); + free(svdev); + return -EFAULT; + } + + rc = virtio_dev_read_dev_config(&svdev->vdev, offsetof(struct virtio_scsi_config, num_queues), + &num_queues, sizeof(num_queues)); + if (rc) { + SPDK_ERRLOG("%s: config read failed: %s\n", base_name, spdk_strerror(-rc)); + virtio_dev_destruct(&svdev->vdev); + free(svdev); + return rc; + } + + if (num_queues < SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED) { + SPDK_ERRLOG("%s: invalid num_queues %u\n", base_name, num_queues); + virtio_dev_destruct(&svdev->vdev); + free(svdev); + return -EINVAL; + } + + rc = virtio_scsi_dev_init(svdev, num_queues, VIRTIO_SCSI_DEV_SUPPORTED_FEATURES); + if (rc != 0) { + virtio_dev_destruct(&svdev->vdev); + free(svdev); + return -EFAULT; + } + + rc = virtio_scsi_dev_scan(svdev, cb_fn, cb_arg); + if (rc) { + virtio_scsi_dev_remove(svdev, NULL, NULL); + } + + return rc; +} + struct bdev_virtio_pci_dev_create_ctx { const char *name; bdev_virtio_create_cb cb_fn;