ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
fe2d64cecf
Spdk/module/vfu_device/vfu_virtio_internal.h
paul luse a6dbe3721e update Intel copyright notices 
per Intel policy to include file commit date using git cmd
below.  The policy does not apply to non-Intel (C) notices.

git log --follow -C90% --format=%ad --date default <file> | tail -1

and then pull just the 4 digit year from the result.

Intel copyrights were not added to files where Intel either had
no contribution ot the contribution lacked substance (ie license
header updates, formatting changes, etc).  Contribution date used
"--follow -C95%" to get the most accurate date.

Note that several files in this patch didn't end the license/(c)
block with a blank comment line so these were added as the vast
majority of files do have this last blank line.  Simply there for
consistency.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: Id5b7ce4f658fe87132f14139ead58d6e285c04d4
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15192
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Community-CI: Mellanox Build Bot
2022-11-10 08:28:53 +00:00

409 lines
12 KiB
C
Raw Blame History

/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (C) 2022 Intel Corporation.
* All rights reserved.
*/
#ifndef _VFU_VIRTIO_INTERNAL_H
#define _VFU_VIRTIO_INTERNAL_H
#include <linux/virtio_config.h>
#include <linux/virtio_ring.h>
#include <linux/virtio_pci.h>
#include "spdk/vfu_target.h"
#define VIRTIO_HOST_SUPPORTED_FEATURES ((1ULL << VIRTIO_F_VERSION_1) | \
(1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \
(1ULL << VIRTIO_F_RING_PACKED))
/* virtio device layout:
*
* region 1: MSI-X Table
* region 2: MSI-X PBA
* region 4: virtio modern memory 64bits BAR
* Common configuration 0x0 - 0x1000
* ISR access 0x1000 - 0x2000
* Device specific configuration 0x2000 - 0x3000
* Notifications 0x3000 - 0x4000
*/
#define VIRTIO_PCI_COMMON_CFG_OFFSET (0x0)
#define VIRTIO_PCI_COMMON_CFG_LENGTH (0x1000)
#define VIRTIO_PCI_ISR_ACCESS_OFFSET (VIRTIO_PCI_COMMON_CFG_OFFSET + VIRTIO_PCI_COMMON_CFG_LENGTH)
#define VIRTIO_PCI_ISR_ACCESS_LENGTH (0x1000)
#define VIRTIO_PCI_SPECIFIC_CFG_OFFSET (VIRTIO_PCI_ISR_ACCESS_OFFSET + VIRTIO_PCI_ISR_ACCESS_LENGTH)
#define VIRTIO_PCI_SPECIFIC_CFG_LENGTH (0x1000)
#define VIRTIO_PCI_NOTIFICATIONS_OFFSET (VIRTIO_PCI_SPECIFIC_CFG_OFFSET + VIRTIO_PCI_SPECIFIC_CFG_LENGTH)
#define VIRTIO_PCI_NOTIFICATIONS_LENGTH (0x1000)
#define VIRTIO_PCI_BAR4_LENGTH (VIRTIO_PCI_NOTIFICATIONS_OFFSET + VIRTIO_PCI_NOTIFICATIONS_LENGTH)
#define VIRTIO_DEV_MAX_IOVS (129)
/* Maximum number of requests which can be processed one time */
#define VIRTIO_DEV_VRING_MAX_REQS (32)
/* Maximum number of queues can be supported by virtio device */
#define VIRTIO_DEV_MAX_VQS (64)
/* Default queue size */
#define VIRTIO_VQ_DEFAULT_SIZE (128)
/* Maximum queue size */
#define VIRTIO_VQ_MAX_SIZE (1024)
struct vfu_virtio_endpoint;
struct vfu_virtio_req;
struct virtio_pci_cfg {
/* Common PCI configuration */
uint32_t guest_feat_lo;
uint32_t guest_feat_hi;
/* Negotiated feature bits */
uint64_t guest_features;
uint32_t host_feature_select;
uint32_t guest_feature_select;
uint16_t msix_config;
uint8_t device_status;
uint8_t config_generation;
uint16_t queue_select;
/* ISR access */
uint8_t isr;
};
enum vfu_vq_state {
VFU_VQ_CREATED = 0,
VFU_VQ_ACTIVE,
VFU_VQ_INACTIVE,
};
struct q_mapping {
/* iov of local process mapping. */
struct iovec iov;
/* Stored sg, needed for unmap. */
dma_sg_t *sg;
/* physical address */
uint64_t phys_addr;
/* virtual address */
union {
void *addr;
struct vring_desc *desc;
struct vring_packed_desc *desc_packed;
struct vring_avail *avail;
struct vring_packed_desc_event *driver_event;
struct vring_used *used;
struct vring_packed_desc_event *device_event;
};
/* size in bytes */
uint64_t len;
};
struct vfu_virtio_vq {
/* Read Only */
uint16_t id;
uint16_t qsize;
bool enabled;
uint16_t vector;
enum vfu_vq_state q_state;
STAILQ_HEAD(, vfu_virtio_req) free_reqs;
uint32_t desc_lo;
uint32_t desc_hi;
uint32_t avail_lo;
uint32_t avail_hi;
uint32_t used_lo;
uint32_t used_hi;
struct q_mapping avail;
struct q_mapping used;
struct q_mapping desc;
uint16_t last_avail_idx;
uint16_t last_used_idx;
struct {
/* To mark a descriptor as available in packed ring
* Equal to avail_wrap_counter in spec.
*/
uint8_t avail_phase : 1;
/* To mark a descriptor as used in packed ring
* Equal to used_wrap_counter in spec.
*/
uint8_t used_phase : 1;
uint8_t padding : 5;
bool packed_ring : 1;
} packed;
/* Request count from last event */
uint16_t used_req_cnt;
/* Next time when we need to send event */
uint64_t next_event_time;
};
struct vfu_virtio_dev {
char name[SPDK_VFU_MAX_NAME_LEN];
/* RO for Guest Driver */
uint16_t num_queues;
/* Supported feature bits by host driver, RO for Guest Driver */
uint64_t host_features;
struct virtio_pci_cfg cfg;
struct vfu_virtio_vq vqs[VIRTIO_DEV_MAX_VQS];
struct vfu_virtio_endpoint *virtio_endpoint;
/* VIRTIO_DEV_MAX_VQS * 3 worth of dma_sg_size() */
uint8_t sg[];
};
struct vfu_virtio_ops {
uint64_t (*get_device_features)(struct vfu_virtio_endpoint *virtio_endpoint);
struct vfu_virtio_req *(*alloc_req)(struct vfu_virtio_endpoint *virtio_endpoint,
struct vfu_virtio_vq *vq);
void (*free_req)(struct vfu_virtio_endpoint *virtio_endpoint, struct vfu_virtio_vq *vq,
struct vfu_virtio_req *req);
int (*exec_request)(struct vfu_virtio_endpoint *virtio_endpoint, struct vfu_virtio_vq *vq,
struct vfu_virtio_req *req);
int (*get_config)(struct vfu_virtio_endpoint *virtio_endpoint, char *buf, uint64_t offset,
uint64_t count);
int (*set_config)(struct vfu_virtio_endpoint *virtio_endpoint, char *buf, uint64_t offset,
uint64_t count);
int (*start_device)(struct vfu_virtio_endpoint *virtio_endpoint);
int (*stop_device)(struct vfu_virtio_endpoint *virtio_endpoint);
};
struct vfu_virtio_endpoint {
struct vfu_virtio_dev *dev;
int devmem_fd;
volatile uint32_t *doorbells;
uint16_t num_queues;
uint16_t qsize;
bool packed_ring;
uint32_t coalescing_delay_us;
struct spdk_vfu_endpoint *endpoint;
struct spdk_thread *thread;
struct vfu_virtio_ops virtio_ops;
/* quiesce poller */
uint32_t io_outstanding;
bool quiesce_in_progress;
struct spdk_poller *quiesce_poller;
};
struct vfu_virtio_req {
struct vfu_virtio_dev *dev;
struct vfu_virtio_vq *vq;
STAILQ_ENTRY(vfu_virtio_req) link;
uint32_t payload_size;
uint32_t used_len;
/* split vring */
uint16_t req_idx;
/* packed vring */
uint16_t buffer_id;
uint16_t num_descs;
uint16_t iovcnt;
struct iovec iovs[VIRTIO_DEV_MAX_IOVS + 1];
uint8_t desc_writeable[VIRTIO_DEV_MAX_IOVS + 1];
struct iovec *indirect_iov;
dma_sg_t *indirect_sg;
/* VIRIO_DEV_MAX_IOVS + 1 worth of dma_sg_size() */
uint8_t sg[];
};
static inline bool
virtio_guest_has_feature(struct vfu_virtio_dev *dev, uint32_t feature_bit)
{
assert(feature_bit <= 64);
return !!(dev->cfg.guest_features & (1ULL << feature_bit));
}
static inline uint64_t
virtio_queue_desc_size(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq)
{
return sizeof(struct vring_desc) * vq->qsize;
}
static inline uint64_t
virtio_queue_avail_size(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq)
{
uint16_t event_size;
if (virtio_guest_has_feature(dev, VIRTIO_F_RING_PACKED)) {
return sizeof(struct vring_packed_desc_event);
}
event_size = virtio_guest_has_feature(dev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return (sizeof(struct vring_avail) + sizeof(uint16_t) * vq->qsize
+ event_size);
}
static inline uint64_t
virtio_queue_used_size(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq)
{
uint16_t event_size;
if (virtio_guest_has_feature(dev, VIRTIO_F_RING_PACKED)) {
return sizeof(struct vring_packed_desc_event);
}
event_size = virtio_guest_has_feature(dev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
return (sizeof(struct vring_used) + sizeof(struct vring_used_elem) * vq->qsize
+ event_size);
}
static inline bool
virtio_queue_event_is_suppressed(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq)
{
bool is_suppressed = false;
if (virtio_guest_has_feature(dev, VIRTIO_F_RING_PACKED)) {
is_suppressed = vq->avail.driver_event->flags & VRING_PACKED_EVENT_FLAG_DISABLE;
} else {
is_suppressed = vq->avail.avail->flags & VRING_AVAIL_F_NO_INTERRUPT;
}
return is_suppressed;
}
static inline bool
virtio_dev_is_started(struct vfu_virtio_dev *dev)
{
return !!(dev->cfg.device_status & VIRTIO_CONFIG_S_DRIVER_OK);
}
static inline bool
virtio_vring_split_desc_is_indirect(struct vring_desc *desc)
{
return !!(desc->flags & VRING_DESC_F_INDIRECT);
}
static inline bool
virtio_vring_packed_desc_is_indirect(struct vring_packed_desc *desc)
{
return !!(desc->flags & VRING_DESC_F_INDIRECT);
}
static inline bool
virtio_vring_split_desc_is_wr(struct vring_desc *desc)
{
return !!(desc->flags & VRING_DESC_F_WRITE);
}
static inline bool
virtio_vring_packed_desc_is_wr(struct vring_packed_desc *desc)
{
return !!(desc->flags & VRING_DESC_F_WRITE);
}
static inline bool
virtio_vring_packed_is_avail(struct vring_packed_desc *desc, bool avail_phase)
{
bool avail_flag, used_flag;
uint16_t flags = desc->flags;
avail_flag = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
used_flag = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
/* To mark a desc as available, the driver sets the F_AVAIL bit in flags
* to match the internal avail wrap counter. It also sets the F_USED bit to
* match the inverse value but it's not mandatory.
*/
return (avail_flag != used_flag) && (avail_flag == avail_phase);
}
static inline bool
virtio_vring_packed_is_used(struct vring_packed_desc *desc, bool used_phase)
{
bool avail_flag, used_flag;
uint16_t flags = desc->flags;
avail_flag = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
used_flag = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
/* When the descriptor is used, two flags in descriptor
* avail flag and used flag are set to equal
* and used flag value == used_wrap_counter.
*/
return (used_flag == avail_flag) && (used_flag == used_phase);
}
static inline bool
virtio_req_iov_is_wr(struct vfu_virtio_req *req, uint32_t iov_num)
{
assert(iov_num <= VIRTIO_DEV_MAX_IOVS);
return req->desc_writeable[iov_num];
}
static inline struct vfu_virtio_req *
vfu_virtio_vq_alloc_req(struct vfu_virtio_endpoint *endpoint, struct vfu_virtio_vq *vq)
{
assert(endpoint->virtio_ops.alloc_req != NULL);
return endpoint->virtio_ops.alloc_req(endpoint, vq);
}
static inline void
vfu_virtio_vq_free_req(struct vfu_virtio_endpoint *endpoint, struct vfu_virtio_vq *vq,
struct vfu_virtio_req *req)
{
assert(endpoint->virtio_ops.free_req);
endpoint->virtio_ops.free_req(endpoint, vq, req);
}
void virtio_vq_used_ring_split_enqueue(struct vfu_virtio_vq *vq, uint16_t req_idx,
uint32_t used_len);
void virtio_vq_used_ring_packed_enqueue(struct vfu_virtio_vq *vq, uint16_t buffer_id,
uint32_t num_descs, uint32_t used_len);
struct vfu_virtio_req *virito_dev_packed_ring_get_next_avail_req(struct vfu_virtio_dev *dev,
struct vfu_virtio_vq *vq);
struct vfu_virtio_req *virito_dev_split_ring_get_next_avail_req(struct vfu_virtio_dev *dev,
struct vfu_virtio_vq *vq);
int vfu_virtio_quiesce_cb(struct spdk_vfu_endpoint *endpoint);
void vfu_virtio_dev_put_req(struct vfu_virtio_req *req);
void vfu_virtio_finish_req(struct vfu_virtio_req *req);
void vfu_virtio_vq_flush_irq(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq);
int vfu_virito_dev_process_packed_ring(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq);
int vfu_virito_dev_process_split_ring(struct vfu_virtio_dev *dev, struct vfu_virtio_vq *vq);
void vfu_virtio_notify_config(struct vfu_virtio_endpoint *virtio_endpoint);
int vfu_virtio_endpoint_setup(struct vfu_virtio_endpoint *virtio_endpoint,
struct spdk_vfu_endpoint *endpoint,
char *basename, const char *endpoint_name,
struct vfu_virtio_ops *ops);
int vfu_virtio_endpoint_destruct(struct vfu_virtio_endpoint *virtio_endpoint);
void vfu_virtio_get_device_info(struct vfu_virtio_endpoint *virtio_endpoint,
struct spdk_vfu_pci_device *device_info);
int vfu_virtio_attach_device(struct spdk_vfu_endpoint *endpoint);
int vfu_virtio_detach_device(struct spdk_vfu_endpoint *endpoint);
uint16_t vfu_virtio_get_vendor_capability(struct spdk_vfu_endpoint *endpoint, char *buf,
uint16_t buf_len, uint16_t idx);
int vfu_virtio_post_memory_add(struct spdk_vfu_endpoint *endpoint, void *map_start, void *map_end);
int vfu_virtio_pre_memory_remove(struct spdk_vfu_endpoint *endpoint, void *map_start,
void *map_end);
int vfu_virtio_pci_reset_cb(struct spdk_vfu_endpoint *endpoint);
int vfu_virtio_blk_add_bdev(const char *name, const char *bdev_name,
uint16_t num_queues, uint16_t qsize, bool packed_ring);
/* virtio_scsi */
int vfu_virtio_scsi_add_target(const char *name, uint8_t scsi_target_num,
const char *bdev_name);
int vfu_virtio_scsi_remove_target(const char *name, uint8_t scsi_target_num);
int vfu_virtio_scsi_set_options(const char *name, uint16_t num_io_queues, uint16_t qsize,
bool packed_ring);
#endif
Reference in New Issue View Git Blame Copy Permalink