From 97a3b814e04a4add9e37b5e08352f2238ea85f97 Mon Sep 17 00:00:00 2001 From: Dariusz Stojaczyk Date: Tue, 17 Oct 2017 17:51:48 +0200 Subject: [PATCH] rte_virtio: fix vdev->max_queues We interpeted max request queue count as number of all queues. It did not take into account eventq and controlq. This patch also fixes overall max_queues negotiation for modern PCI devices. Fixes 8b0a4a3 ("bdev_virtio: implement multiqueue") Change-Id: I834cb973772ca5946ac26d18bd3eeb2783f48ea9 Signed-off-by: Dariusz Stojaczyk Reviewed-on: https://review.gerrithub.io/382816 Tested-by: SPDK Automated Test System Reviewed-by: Jim Harris Reviewed-by: Daniel Verkamp --- lib/bdev/virtio/bdev_virtio.c | 3 ++- lib/bdev/virtio/rte_virtio/virtio_pci.c | 17 +++++++++++------ lib/bdev/virtio/rte_virtio/virtio_pci.h | 3 +++ .../rte_virtio/virtio_user/virtio_user_dev.c | 7 ++++--- .../rte_virtio/virtio_user/virtio_user_dev.h | 16 +++++++++++++++- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/lib/bdev/virtio/bdev_virtio.c b/lib/bdev/virtio/bdev_virtio.c index 40a90377f..e363bd68d 100644 --- a/lib/bdev/virtio/bdev_virtio.c +++ b/lib/bdev/virtio/bdev_virtio.c @@ -875,7 +875,8 @@ bdev_virtio_process_config(void) num_queues = 1; } - vdev = virtio_user_dev_init(path, num_queues + 2, 512); + vdev = virtio_user_dev_init(path, num_queues, 512, + SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED); if (vdev == NULL) { rc = -1; goto out; diff --git a/lib/bdev/virtio/rte_virtio/virtio_pci.c b/lib/bdev/virtio/rte_virtio/virtio_pci.c index 5c2475e2a..d3bef109a 100644 --- a/lib/bdev/virtio/rte_virtio/virtio_pci.c +++ b/lib/bdev/virtio/rte_virtio/virtio_pci.c @@ -617,6 +617,15 @@ next: return 0; } +static void +virtio_dev_pci_init(struct virtio_dev *vdev) +{ + vtpci_read_dev_config(vdev, offsetof(struct virtio_scsi_config, num_queues), + &vdev->max_queues, sizeof(vdev->max_queues)); + vdev->max_queues += SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED; + TAILQ_INSERT_TAIL(&g_virtio_driver.init_ctrlrs, vdev, tailq); +} + static int pci_enum_virtio_probe_cb(void *ctx, struct spdk_pci_device *pci_dev) { @@ -661,7 +670,7 @@ pci_enum_virtio_probe_cb(void *ctx, struct spdk_pci_device *pci_dev) goto err; } vdev->modern = 1; - TAILQ_INSERT_TAIL(&g_virtio_driver.init_ctrlrs, vdev, tailq); + virtio_dev_pci_init(vdev); return 0; } @@ -685,11 +694,7 @@ pci_enum_virtio_probe_cb(void *ctx, struct spdk_pci_device *pci_dev) goto err; } vdev->modern = 0; - - vtpci_read_dev_config(vdev, offsetof(struct virtio_scsi_config, num_queues), - &vdev->max_queues, sizeof(vdev->max_queues)); - - TAILQ_INSERT_TAIL(&g_virtio_driver.init_ctrlrs, vdev, tailq); + virtio_dev_pci_init(vdev); return 0; err: diff --git a/lib/bdev/virtio/rte_virtio/virtio_pci.h b/lib/bdev/virtio/rte_virtio/virtio_pci.h index 5841a9b28..1ce1d6962 100644 --- a/lib/bdev/virtio/rte_virtio/virtio_pci.h +++ b/lib/bdev/virtio/rte_virtio/virtio_pci.h @@ -50,6 +50,9 @@ struct virtqueue; /* Extra status define for readability */ #define VIRTIO_CONFIG_S_RESET 0 +/* Number of non-request queues - eventq and controlq */ +#define SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED 2 + struct virtio_pci_ops { void (*read_dev_cfg)(struct virtio_dev *hw, size_t offset, void *dst, int len); diff --git a/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.c b/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.c index 2b47f7556..ad526ca29 100644 --- a/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.c +++ b/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.c @@ -176,7 +176,8 @@ virtio_user_dev_setup(struct virtio_user_dev *dev) } struct virtio_dev * -virtio_user_dev_init(char *path, uint16_t requested_queues, uint32_t queue_size) +virtio_user_dev_init(char *path, uint16_t requested_queues, + uint32_t queue_size, uint16_t fixed_queue_num) { struct virtio_dev *vdev; struct virtio_user_dev *dev; @@ -211,12 +212,12 @@ virtio_user_dev_init(char *path, uint16_t requested_queues, uint32_t queue_size) } if (requested_queues > max_queues) { - SPDK_ERRLOG("requested %"PRIu16" queues but only %"PRIu64" available\n", + SPDK_ERRLOG("requested %"PRIu16" request queues but only %"PRIu64" available\n", requested_queues, max_queues); goto err; } - vdev->max_queues = requested_queues; + vdev->max_queues = fixed_queue_num + requested_queues; if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER, NULL) < 0) { spdk_strerror_r(errno, err_str, sizeof(err_str)); diff --git a/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.h b/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.h index cec72756d..d00823210 100644 --- a/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.h +++ b/lib/bdev/virtio/rte_virtio/virtio_user/virtio_user_dev.h @@ -62,7 +62,21 @@ struct virtio_user_dev { int virtio_user_start_device(struct virtio_user_dev *dev); int virtio_user_stop_device(struct virtio_user_dev *dev); -struct virtio_dev *virtio_user_dev_init(char *path, uint16_t requested_queues, uint32_t queue_size); + +/** + * Connect to a vhost-user device and create corresponding virtio_dev. + * + * \param path path to the Unix domain socket of the vhost-user device + * \param requested_queues maximum number of request queues that this + * device will support + * \param queue_size size of each of the queues + * \param fixed_queue_num number of queues preceeding the first + * request queue. For Virtio-SCSI this is equal to 2, as there are + * additional event and control queues. + * \return virtio device + */ +struct virtio_dev *virtio_user_dev_init(char *path, uint16_t requested_queues, + uint32_t queue_size, uint16_t fixed_queue_num); void virtio_user_dev_uninit(struct virtio_user_dev *dev); #endif