ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vhost: upgrade SPDK vhost code to DPDK 17.05

Browse Source 
Also replace the internal DPDK v17.02-based rte_vhost
library with the patched DPDK v17.05-based version.

Change-Id: Ibec0b0746592a1a3911c31642a945ab65495e33e
Signed-off-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Signed-off-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
This commit is contained in:
Pawel Wodkowski 2017-05-08 09:43:16 -07:00 committed by Jim Harris
parent d391647bd0
commit 8d7acdaaef
21 changed files with 971 additions and 5209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/vhost/rte_vhost/Makefile
View File

@ -34,10 +34,11 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += -I.
CFLAGS += $(ENV_CFLAGS)
# These are the DPDK vhost files copied (for now) into SPDK
C_SRCS += fd_man.c socket.c vhost_user.c virtio_net.c vhost.c
C_SRCS += fd_man.c socket.c vhost_user.c vhost.c
LIBNAME = rte_vhost

227
lib/vhost/rte_vhost/fd_man.c
View File

@ -35,19 +35,65 @@
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <sys/select.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <string.h>
#include <rte_common.h>
#include <rte_log.h>
#include "fd_man.h"
#define FDPOLLERR (POLLERR | POLLHUP | POLLNVAL)
static int
get_last_valid_idx(struct fdset *pfdset, int last_valid_idx)
{
int i;
for (i = last_valid_idx; i >= 0 && pfdset->fd[i].fd == -1; i--)
;
return i;
}
static void
fdset_move(struct fdset *pfdset, int dst, int src)
{
pfdset->fd[dst] = pfdset->fd[src];
pfdset->rwfds[dst] = pfdset->rwfds[src];
}
static void
fdset_shrink_nolock(struct fdset *pfdset)
{
int i;
int last_valid_idx = get_last_valid_idx(pfdset, pfdset->num - 1);
for (i = 0; i < last_valid_idx; i++) {
if (pfdset->fd[i].fd != -1)
continue;
fdset_move(pfdset, i, last_valid_idx);
last_valid_idx = get_last_valid_idx(pfdset, last_valid_idx - 1);
}
pfdset->num = last_valid_idx + 1;
}
/*
* Find deleted fd entries and remove them
*/
static void
fdset_shrink(struct fdset *pfdset)
{
pthread_mutex_lock(&pfdset->fd_mutex);
fdset_shrink_nolock(pfdset);
pthread_mutex_unlock(&pfdset->fd_mutex);
}
/**
* Returns the index in the fdset for a given fd.
* If fd is -1, it means to search for a free entry.
* @return
* index for the fd, or -1 if fd isn't in the fdset.
*/
@ -56,72 +102,28 @@ fdset_find_fd(struct fdset *pfdset, int fd)
{
int i;
if (pfdset == NULL)
return -1;
for (i = 0; i < MAX_FDS && pfdset->fd[i].fd != fd; i++)
for (i = 0; i < pfdset->num && pfdset->fd[i].fd != fd; i++)
;
return i == MAX_FDS ? -1 : i;
return i == pfdset->num ? -1 : i;
}
static int
fdset_find_free_slot(struct fdset *pfdset)
{
return fdset_find_fd(pfdset, -1);
}
static int
fdset_add_fd(struct fdset *pfdset, int idx, int fd,
static void
fdset_add_fd(struct fdset *pfdset, int idx, int fd,
fd_cb rcb, fd_cb wcb, void *dat)
{
struct fdentry *pfdentry;
struct fdentry *pfdentry = &pfdset->fd[idx];
struct pollfd *pfd = &pfdset->rwfds[idx];
if (pfdset == NULL || idx >= MAX_FDS || fd >= FD_SETSIZE)
return -1;
pfdentry = &pfdset->fd[idx];
pfdentry->fd = fd;
pfdentry->fd = fd;
pfdentry->rcb = rcb;
pfdentry->wcb = wcb;
pfdentry->dat = dat;
return 0;
}
/**
* Fill the read/write fd_set with the fds in the fdset.
* @return
* the maximum fds filled in the read/write fd_set.
*/
static int
fdset_fill(fd_set *rfset, fd_set *wfset, struct fdset *pfdset)
{
struct fdentry *pfdentry;
int i, maxfds = -1;
int num = MAX_FDS;
if (pfdset == NULL)
return -1;
for (i = 0; i < num; i++) {
pfdentry = &pfdset->fd[i];
if (pfdentry->fd != -1) {
int added = 0;
if (pfdentry->rcb && rfset) {
FD_SET(pfdentry->fd, rfset);
added = 1;
}
if (pfdentry->wcb && wfset) {
FD_SET(pfdentry->fd, wfset);
added = 1;
}
if (added)
maxfds = pfdentry->fd < maxfds ?
maxfds : pfdentry->fd;
}
}
return maxfds;
pfd->fd = fd;
pfd->events = rcb ? POLLIN : 0;
pfd->events |= wcb ? POLLOUT : 0;
pfd->revents = 0;
}
void
@ -151,16 +153,17 @@ fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat)
return -1;
pthread_mutex_lock(&pfdset->fd_mutex);
/* Find a free slot in the list. */
i = fdset_find_free_slot(pfdset);
if (i == -1 || fdset_add_fd(pfdset, i, fd, rcb, wcb, dat) < 0) {
pthread_mutex_unlock(&pfdset->fd_mutex);
return -2;
i = pfdset->num < MAX_FDS ? pfdset->num++ : -1;
if (i == -1) {
fdset_shrink_nolock(pfdset);
i = pfdset->num < MAX_FDS ? pfdset->num++ : -1;
if (i == -1) {
pthread_mutex_unlock(&pfdset->fd_mutex);
return -2;
}
}
pfdset->num++;
fdset_add_fd(pfdset, i, fd, rcb, wcb, dat);
pthread_mutex_unlock(&pfdset->fd_mutex);
return 0;
@ -189,7 +192,6 @@ fdset_del(struct fdset *pfdset, int fd)
pfdset->fd[i].fd = -1;
pfdset->fd[i].rcb = pfdset->fd[i].wcb = NULL;
pfdset->fd[i].dat = NULL;
pfdset->num--;
i = -1;
}
pthread_mutex_unlock(&pfdset->fd_mutex);
@ -198,24 +200,6 @@ fdset_del(struct fdset *pfdset, int fd)
return dat;
}
/**
* Unregister the fd at the specified slot from the fdset.
*/
static void
fdset_del_slot(struct fdset *pfdset, int index)
{
if (pfdset == NULL || index < 0 || index >= MAX_FDS)
return;
pthread_mutex_lock(&pfdset->fd_mutex);
pfdset->fd[index].fd = -1;
pfdset->fd[index].rcb = pfdset->fd[index].wcb = NULL;
pfdset->fd[index].dat = NULL;
pfdset->num--;
pthread_mutex_unlock(&pfdset->fd_mutex);
}
/**
* This functions runs in infinite blocking loop until there is no fd in
@ -226,58 +210,68 @@ fdset_del_slot(struct fdset *pfdset, int index)
* will wait until the flag is reset to zero(which indicates the callback is
* finished), then it could free the context after fdset_del.
*/
void
fdset_event_dispatch(struct fdset *pfdset)
void *
fdset_event_dispatch(void *arg)
{
fd_set rfds, wfds;
int i, maxfds;
int i;
struct pollfd *pfd;
struct fdentry *pfdentry;
int num = MAX_FDS;
fd_cb rcb, wcb;
void *dat;
int fd;
int fd, numfds;
int remove1, remove2;
int ret;
int need_shrink;
struct fdset *pfdset = arg;
if (pfdset == NULL)
return;
return NULL;
while (1) {
struct timeval tv;
tv.tv_sec = 1;
tv.tv_usec = 0;
FD_ZERO(&rfds);
FD_ZERO(&wfds);
pthread_mutex_lock(&pfdset->fd_mutex);
maxfds = fdset_fill(&rfds, &wfds, pfdset);
pthread_mutex_unlock(&pfdset->fd_mutex);
/*
* When select is blocked, other threads might unregister
* When poll is blocked, other threads might unregister
* listenfds from and register new listenfds into fdset.
* When select returns, the entries for listenfds in the fdset
* When poll returns, the entries for listenfds in the fdset
* might have been updated. It is ok if there is unwanted call
* for new listenfds.
*/
ret = select(maxfds + 1, &rfds, &wfds, NULL, &tv);
if (ret <= 0)
continue;
pthread_mutex_lock(&pfdset->fd_mutex);
numfds = pfdset->num;
pthread_mutex_unlock(&pfdset->fd_mutex);
for (i = 0; i < num; i++) {
remove1 = remove2 = 0;
poll(pfdset->rwfds, numfds, 1000 /* millisecs */);
need_shrink = 0;
for (i = 0; i < numfds; i++) {
pthread_mutex_lock(&pfdset->fd_mutex);
pfdentry = &pfdset->fd[i];
fd = pfdentry->fd;
pfd = &pfdset->rwfds[i];
if (fd < 0) {
need_shrink = 1;
pthread_mutex_unlock(&pfdset->fd_mutex);
continue;
}
if (!pfd->revents) {
pthread_mutex_unlock(&pfdset->fd_mutex);
continue;
}
remove1 = remove2 = 0;
rcb = pfdentry->rcb;
wcb = pfdentry->wcb;
dat = pfdentry->dat;
pfdentry->busy = 1;
pthread_mutex_unlock(&pfdset->fd_mutex);
if (fd >= 0 && FD_ISSET(fd, &rfds) && rcb)
if (rcb && pfd->revents & (POLLIN | FDPOLLERR))
rcb(fd, dat, &remove1);
if (fd >= 0 && FD_ISSET(fd, &wfds) && wcb)
if (wcb && pfd->revents & (POLLOUT | FDPOLLERR))
wcb(fd, dat, &remove2);
pfdentry->busy = 0;
/*
@ -292,8 +286,15 @@ fdset_event_dispatch(struct fdset *pfdset)
* listen fd in another thread, we couldn't call
* fd_set_del.
*/
if (remove1 || remove2)
fdset_del_slot(pfdset, i);
if (remove1 || remove2) {
pfdentry->fd = -1;
need_shrink = 1;
}
}
if (need_shrink)
fdset_shrink(pfdset);
}
return NULL;
}

4
lib/vhost/rte_vhost/fd_man.h
View File

@ -35,6 +35,7 @@
#define _FD_MAN_H_
#include <stdint.h>
#include <pthread.h>
#include <poll.h>
#define MAX_FDS 1024
@ -49,6 +50,7 @@ struct fdentry {
};
struct fdset {
struct pollfd rwfds[MAX_FDS];
struct fdentry fd[MAX_FDS];
pthread_mutex_t fd_mutex;
int num; /* current fd number of this fdset */
@ -62,6 +64,6 @@ int fdset_add(struct fdset *pfdset, int fd,
void *fdset_del(struct fdset *pfdset, int fd);
void fdset_event_dispatch(struct fdset *pfdset);
void *fdset_event_dispatch(void *arg);
#endif

0
lib/vhost/rte_vhost_17_05/rte_vhost.h → lib/vhost/rte_vhost/rte_vhost.h
View File

193
lib/vhost/rte_vhost/rte_virtio_net.h
View File

@ -1,193 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _VIRTIO_NET_H_
#define _VIRTIO_NET_H_
/**
* @file
* Interface to vhost net
*/
#include <stdint.h>
#include <linux/vhost.h>
#include <linux/virtio_ring.h>
#include <linux/virtio_net.h>
#include <sys/eventfd.h>
#include <sys/socket.h>
#include <linux/if.h>
#include <rte_memory.h>
#include <rte_mempool.h>
#include <rte_ether.h>
#define RTE_VHOST_USER_CLIENT (1ULL << 0)
#define RTE_VHOST_USER_NO_RECONNECT (1ULL << 1)
#define RTE_VHOST_USER_DEQUEUE_ZERO_COPY (1ULL << 2)
/* Enum for virtqueue management. */
enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM};
/**
* Device and vring operations.
*/
struct virtio_net_device_ops {
int (*new_device)(int vid); /**< Add device. */
void (*destroy_device)(int vid); /**< Remove device. */
int (*vring_state_changed)(int vid, uint16_t queue_id, int enable); /**< triggered when a vring is enabled or disabled */
void *reserved[5]; /**< Reserved for future extension */
};
/**
* Disable features in feature_mask. Returns 0 on success.
*/
int rte_vhost_feature_disable(uint64_t feature_mask);
/**
* Enable features in feature_mask. Returns 0 on success.
*/
int rte_vhost_feature_enable(uint64_t feature_mask);
/* Returns currently supported vhost features */
uint64_t rte_vhost_feature_get(void);
int rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable);
/**
* Register vhost driver. path could be different for multiple
* instance support.
*/
int rte_vhost_driver_register(const char *path, uint64_t flags);
/* Unregister vhost driver. This is only meaningful to vhost user. */
int rte_vhost_driver_unregister(const char *path);
/* Register callbacks. */
int rte_vhost_driver_callback_register(struct virtio_net_device_ops const * const);
/* Start vhost driver session blocking loop. */
int rte_vhost_driver_session_start(void);
/**
* Get the numa node from which the virtio net device's memory
* is allocated.
*
* @param vid
* virtio-net device ID
*
* @return
* The numa node, -1 on failure
*/
int rte_vhost_get_numa_node(int vid);
/**
* Get the number of queues the device supports.
*
* @param vid
* virtio-net device ID
*
* @return
* The number of queues, 0 on failure
*/
uint32_t rte_vhost_get_queue_num(int vid);
/**
* Get the virtio net device's ifname, which is the vhost-user socket
* file path.
*
* @param vid
* virtio-net device ID
* @param buf
* The buffer to stored the queried ifname
* @param len
* The length of buf
*
* @return
* 0 on success, -1 on failure
*/
int rte_vhost_get_ifname(int vid, char *buf, size_t len);
/**
* Get how many avail entries are left in the queue
*
* @param vid
* virtio-net device ID
* @param queue_id
* virtio queue index
*
* @return
* num of avail entires left
*/
uint16_t rte_vhost_avail_entries(int vid, uint16_t queue_id);
/**
* This function adds buffers to the virtio devices RX virtqueue. Buffers can
* be received from the physical port or from another virtual device. A packet
* count is returned to indicate the number of packets that were succesfully
* added to the RX queue.
* @param vid
* virtio-net device ID
* @param queue_id
* virtio queue index in mq case
* @param pkts
* array to contain packets to be enqueued
* @param count
* packets num to be enqueued
* @return
* num of packets enqueued
*/
uint16_t rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
struct rte_mbuf **pkts, uint16_t count);
/**
* This function gets guest buffers from the virtio device TX virtqueue,
* construct host mbufs, copies guest buffer content to host mbufs and
* store them in pkts to be processed.
* @param vid
* virtio-net device
* @param queue_id
* virtio queue index in mq case
* @param mbuf_pool
* mbuf_pool where host mbuf is allocated.
* @param pkts
* array to contain packets to be dequeued
* @param count
* packets num to be dequeued
* @return
* num of packets dequeued
*/
uint16_t rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count);
#endif /* _VIRTIO_NET_H_ */

279
lib/vhost/rte_vhost/socket.c
View File

@ -52,22 +52,42 @@
#include "vhost.h"
#include "vhost_user.h"
TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
/*
* Every time rte_vhost_driver_register() is invoked, an associated
* vhost_user_socket struct will be created.
*/
struct vhost_user_socket {
struct vhost_user_connection_list conn_list;
pthread_mutex_t conn_mutex;
char *path;
int listenfd;
int connfd;
int socket_fd;
struct sockaddr_un un;
bool is_server;
bool reconnect;
bool dequeue_zero_copy;
/*
* The "supported_features" indicates the feature bits the
* vhost driver supports. The "features" indicates the feature
* bits after the rte_vhost_driver_features_disable/enable().
* It is also the final feature bits used for vhost-user
* features negotiation.
*/
uint64_t supported_features;
uint64_t features;
struct vhost_device_ops const *notify_ops;
};
struct vhost_user_connection {
struct vhost_user_socket *vsocket;
int connfd;
int vid;
TAILQ_ENTRY(vhost_user_connection) next;
};
#define MAX_VHOST_SOCKET 1024
@ -82,7 +102,8 @@ struct vhost_user {
static void vhost_user_server_new_connection(int fd, void *data, int *remove);
static void vhost_user_read_cb(int fd, void *dat, int *remove);
static int vhost_user_create_client(struct vhost_user_socket *vsocket);
static int create_unix_socket(struct vhost_user_socket *vsocket);
static int vhost_user_start_client(struct vhost_user_socket *vsocket);
static struct vhost_user vhost_user = {
.fdset = {
@ -160,7 +181,8 @@ send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num)
msgh.msg_controllen = sizeof(control);
cmsg = CMSG_FIRSTHDR(&msgh);
if (cmsg == NULL) {
RTE_LOG(ERR, VHOST_CONFIG, "null cmsg\n");
RTE_LOG(ERR, VHOST_CONFIG, "cmsg == NULL\n");
errno = EINVAL;
return -1;
}
cmsg->cmsg_len = CMSG_LEN(fdsize);
@ -213,19 +235,23 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
RTE_LOG(INFO, VHOST_CONFIG, "new device, handle is %d\n", vid);
vsocket->connfd = fd;
conn->connfd = fd;
conn->vsocket = vsocket;
conn->vid = vid;
ret = fdset_add(&vhost_user.fdset, fd, vhost_user_read_cb,
NULL, conn);
if (ret < 0) {
vsocket->connfd = -1;
conn->connfd = -1;
free(conn);
close(fd);
RTE_LOG(ERR, VHOST_CONFIG,
"failed to add fd %d into vhost server fdset\n",
fd);
}
pthread_mutex_lock(&vsocket->conn_mutex);
TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next);
pthread_mutex_unlock(&vsocket->conn_mutex);
}
/* call back when there is new vhost-user connection from client */
@ -251,29 +277,36 @@ vhost_user_read_cb(int connfd, void *dat, int *remove)
ret = vhost_user_msg_handler(conn->vid, connfd);
if (ret < 0) {
vsocket->connfd = -1;
close(connfd);
*remove = 1;
vhost_destroy_device(conn->vid);
pthread_mutex_lock(&vsocket->conn_mutex);
TAILQ_REMOVE(&vsocket->conn_list, conn, next);
pthread_mutex_unlock(&vsocket->conn_mutex);
free(conn);
if (vsocket->reconnect)
vhost_user_create_client(vsocket);
if (vsocket->reconnect) {
create_unix_socket(vsocket);
vhost_user_start_client(vsocket);
}
}
}
static int
create_unix_socket(const char *path, struct sockaddr_un *un, bool is_server)
create_unix_socket(struct vhost_user_socket *vsocket)
{
int fd;
struct sockaddr_un *un = &vsocket->un;
fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (fd < 0)
return -1;
RTE_LOG(INFO, VHOST_CONFIG, "vhost-user %s: socket created, fd: %d\n",
is_server ? "server" : "client", fd);
vsocket->is_server ? "server" : "client", fd);
if (!is_server && fcntl(fd, F_SETFL, O_NONBLOCK)) {
if (!vsocket->is_server && fcntl(fd, F_SETFL, O_NONBLOCK)) {
RTE_LOG(ERR, VHOST_CONFIG,
"vhost-user: can't set nonblocking mode for socket, fd: "
"%d (%s)\n", fd, strerror(errno));
@ -283,25 +316,21 @@ create_unix_socket(const char *path, struct sockaddr_un *un, bool is_server)
memset(un, 0, sizeof(*un));
un->sun_family = AF_UNIX;
strncpy(un->sun_path, path, sizeof(un->sun_path));
strncpy(un->sun_path, vsocket->path, sizeof(un->sun_path));
un->sun_path[sizeof(un->sun_path) - 1] = '\0';
return fd;
vsocket->socket_fd = fd;
return 0;
}
static int
vhost_user_create_server(struct vhost_user_socket *vsocket)
vhost_user_start_server(struct vhost_user_socket *vsocket)
{
int fd;
int ret;
struct sockaddr_un un;
int fd = vsocket->socket_fd;
const char *path = vsocket->path;
fd = create_unix_socket(path, &un, vsocket->is_server);
if (fd < 0)
return -1;
ret = bind(fd, (struct sockaddr *)&un, sizeof(un));
ret = bind(fd, (struct sockaddr *)&vsocket->un, sizeof(vsocket->un));
if (ret < 0) {
RTE_LOG(ERR, VHOST_CONFIG,
"failed to bind to %s: %s; remove it and try again\n",
@ -314,7 +343,6 @@ vhost_user_create_server(struct vhost_user_socket *vsocket)
if (ret < 0)
goto err;
vsocket->listenfd = fd;
ret = fdset_add(&vhost_user.fdset, fd, vhost_user_server_new_connection,
NULL, vsocket);
if (ret < 0) {
@ -433,26 +461,21 @@ vhost_user_reconnect_init(void)
}
static int
vhost_user_create_client(struct vhost_user_socket *vsocket)
vhost_user_start_client(struct vhost_user_socket *vsocket)
{
int fd;
int ret;
struct sockaddr_un un;
int fd = vsocket->socket_fd;
const char *path = vsocket->path;
struct vhost_user_reconnect *reconn;
fd = create_unix_socket(path, &un, vsocket->is_server);
if (fd < 0)
return -1;
ret = vhost_user_connect_nonblock(fd, (struct sockaddr *)&un,
sizeof(un));
ret = vhost_user_connect_nonblock(fd, (struct sockaddr *)&vsocket->un,
sizeof(vsocket->un));
if (ret == 0) {
vhost_user_add_connection(fd, vsocket);
return 0;
}
RTE_LOG(ERR, VHOST_CONFIG,
RTE_LOG(WARNING, VHOST_CONFIG,
"failed to connect to %s: %s\n",
path, strerror(errno));
@ -461,7 +484,7 @@ vhost_user_create_client(struct vhost_user_socket *vsocket)
return -1;
}
RTE_LOG(ERR, VHOST_CONFIG, "%s: reconnecting...\n", path);
RTE_LOG(INFO, VHOST_CONFIG, "%s: reconnecting...\n", path);
reconn = malloc(sizeof(*reconn));
if (reconn == NULL) {
RTE_LOG(ERR, VHOST_CONFIG,
@ -469,7 +492,7 @@ vhost_user_create_client(struct vhost_user_socket *vsocket)
close(fd);
return -1;
}
reconn->un = un;
reconn->un = vsocket->un;
reconn->fd = fd;
reconn->vsocket = vsocket;
pthread_mutex_lock(&reconn_list.mutex);
@ -479,6 +502,94 @@ vhost_user_create_client(struct vhost_user_socket *vsocket)
return 0;
}
static struct vhost_user_socket *
find_vhost_user_socket(const char *path)
{
int i;
for (i = 0; i < vhost_user.vsocket_cnt; i++) {
struct vhost_user_socket *vsocket = vhost_user.vsockets[i];
if (!strcmp(vsocket->path, path))
return vsocket;
}
return NULL;
}
int
rte_vhost_driver_disable_features(const char *path, uint64_t features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket)
vsocket->features &= ~features;
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
int
rte_vhost_driver_enable_features(const char *path, uint64_t features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket) {
if ((vsocket->supported_features & features) != features) {
/*
* trying to enable features the driver doesn't
* support.
*/
pthread_mutex_unlock(&vhost_user.mutex);
return -1;
}
vsocket->features |= features;
}
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
int
rte_vhost_driver_set_features(const char *path, uint64_t features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket) {
vsocket->supported_features = features;
vsocket->features = features;
}
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
int
rte_vhost_driver_get_features(const char *path, uint64_t *features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket)
*features = vsocket->features;
pthread_mutex_unlock(&vhost_user.mutex);
if (!vsocket) {
RTE_LOG(ERR, VHOST_CONFIG,
"socket file %s is not registered yet.\n", path);
return -1;
} else {
return 0;
}
}
/*
* Register a new vhost-user socket; here we could act as server
* (the default case), or client (when RTE_VHOST_USER_CLIENT) flag
@ -506,9 +617,25 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
goto out;
memset(vsocket, 0, sizeof(struct vhost_user_socket));
vsocket->path = strdup(path);
vsocket->connfd = -1;
TAILQ_INIT(&vsocket->conn_list);
pthread_mutex_init(&vsocket->conn_mutex, NULL);
vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
/*
* Set the supported features correctly for the builtin vhost-user
* net driver.
*
* Applications know nothing about features the builtin virtio net
* driver (virtio_net.c) supports, thus it's not possible for them
* to invoke rte_vhost_driver_set_features(). To workaround it, here
* we set it unconditionally. If the application want to implement
* another vhost-user driver (say SCSI), it should call the
* rte_vhost_driver_set_features(), which will overwrite following
* two values.
*/
vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
vsocket->features = VIRTIO_NET_SUPPORTED_FEATURES;
if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
if (vsocket->reconnect && reconn_tid == 0) {
@ -518,11 +645,10 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
goto out;
}
}
ret = vhost_user_create_client(vsocket);
} else {
vsocket->is_server = true;
ret = vhost_user_create_server(vsocket);
}
ret = create_unix_socket(vsocket);
if (ret < 0) {
free(vsocket->path);
free(vsocket);
@ -569,7 +695,7 @@ rte_vhost_driver_unregister(const char *path)
{
int i;
int count;
struct vhost_user_connection *conn;
struct vhost_user_connection *conn, *next;
pthread_mutex_lock(&vhost_user.mutex);
@ -578,22 +704,29 @@ rte_vhost_driver_unregister(const char *path)
if (!strcmp(vsocket->path, path)) {
if (vsocket->is_server) {
fdset_del(&vhost_user.fdset, vsocket->listenfd);
close(vsocket->listenfd);
fdset_del(&vhost_user.fdset, vsocket->socket_fd);
close(vsocket->socket_fd);
unlink(path);
} else if (vsocket->reconnect) {
vhost_user_remove_reconnect(vsocket);
}
conn = fdset_del(&vhost_user.fdset, vsocket->connfd);
if (conn) {
pthread_mutex_lock(&vsocket->conn_mutex);
for (conn = TAILQ_FIRST(&vsocket->conn_list);
conn != NULL;
conn = next) {
next = TAILQ_NEXT(conn, next);
fdset_del(&vhost_user.fdset, conn->connfd);
RTE_LOG(INFO, VHOST_CONFIG,
"free connfd = %d for device '%s'\n",
vsocket->connfd, path);
close(vsocket->connfd);
conn->connfd, path);
close(conn->connfd);
vhost_destroy_device(conn->vid);
TAILQ_REMOVE(&vsocket->conn_list, conn, next);
free(conn);
}
pthread_mutex_unlock(&vsocket->conn_mutex);
free(vsocket->path);
free(vsocket);
@ -611,9 +744,59 @@ rte_vhost_driver_unregister(const char *path)
return -1;
}
/*
* Register ops so that we can add/remove device to data core.
*/
int
rte_vhost_driver_session_start(void)
rte_vhost_driver_callback_register(const char *path,
struct vhost_device_ops const * const ops)
{
fdset_event_dispatch(&vhost_user.fdset);
return 0;
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket)
vsocket->notify_ops = ops;
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
struct vhost_device_ops const *
vhost_driver_callback_get(const char *path)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? vsocket->notify_ops : NULL;
}
int
rte_vhost_driver_start(const char *path)
{
struct vhost_user_socket *vsocket;
static pthread_t fdset_tid;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
pthread_mutex_unlock(&vhost_user.mutex);
if (!vsocket)
return -1;
if (fdset_tid == 0) {
int ret = pthread_create(&fdset_tid, NULL, fdset_event_dispatch,
&vhost_user.fdset);
if (ret < 0)
RTE_LOG(ERR, VHOST_CONFIG,
"failed to create fdset handling thread");
}
if (vsocket->is_server)
return vhost_user_start_server(vsocket);
else
return vhost_user_start_client(vsocket);
}

269
lib/vhost/rte_vhost/vhost.c
View File

@ -45,34 +45,12 @@
#include <rte_string_fns.h>
#include <rte_memory.h>
#include <rte_malloc.h>
#include <rte_vhost.h>
#include "vhost.h"
#define VHOST_USER_F_PROTOCOL_FEATURES 30
/* Features supported by this lib. */
#define VHOST_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \
(1ULL << VIRTIO_NET_F_CTRL_VQ) | \
(1ULL << VIRTIO_NET_F_CTRL_RX) | \
(1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
(VHOST_SUPPORTS_MQ) | \
(1ULL << VIRTIO_F_VERSION_1) | \
(1ULL << VHOST_F_LOG_ALL) | \
(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
(1ULL << VIRTIO_NET_F_HOST_TSO4) | \
(1ULL << VIRTIO_NET_F_HOST_TSO6) | \
(1ULL << VIRTIO_NET_F_CSUM) | \
(1ULL << VIRTIO_NET_F_GUEST_CSUM) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO6))
uint64_t VHOST_FEATURES = VHOST_SUPPORTED_FEATURES;
struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
/* device ops to add/remove device to/from data core. */
struct virtio_net_device_ops const *notify_ops;
struct virtio_net *
get_device(int vid)
{
@ -106,10 +84,8 @@ cleanup_device(struct virtio_net *dev, int destroy)
vhost_backend_cleanup(dev);
for (i = 0; i < dev->virt_qp_nb; i++) {
cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ], destroy);
cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ], destroy);
}
for (i = 0; i < dev->nr_vring; i++)
cleanup_vq(dev->virtqueue[i], destroy);
}
/*
@ -119,24 +95,21 @@ static void
free_device(struct virtio_net *dev)
{
uint32_t i;
struct vhost_virtqueue *rxq, *txq;
struct vhost_virtqueue *vq;
for (i = 0; i < dev->virt_qp_nb; i++) {
rxq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ];
txq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ];
for (i = 0; i < dev->nr_vring; i++) {
vq = dev->virtqueue[i];
rte_free(rxq->shadow_used_ring);
rte_free(txq->shadow_used_ring);
rte_free(vq->shadow_used_ring);
/* rxq and txq are allocated together as queue-pair */
rte_free(rxq);
rte_free(vq);
}
rte_free(dev);
}
static void
init_vring_queue(struct vhost_virtqueue *vq, int qp_idx)
init_vring_queue(struct vhost_virtqueue *vq)
{
memset(vq, 0, sizeof(struct vhost_virtqueue));
@ -146,69 +119,48 @@ init_vring_queue(struct vhost_virtqueue *vq, int qp_idx)
/* Backends are set to -1 indicating an inactive device. */
vq->backend = -1;
/* always set the default vq pair to enabled */
if (qp_idx == 0)
vq->enabled = 1;
/*
* always set the vq to enabled; this is to keep compatibility
* with the old QEMU, whereas there is no SET_VRING_ENABLE message.
*/
vq->enabled = 1;
TAILQ_INIT(&vq->zmbuf_list);
}
static void
init_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx)
{
uint32_t base_idx = qp_idx * VIRTIO_QNUM;
init_vring_queue(dev->virtqueue[base_idx + VIRTIO_RXQ], qp_idx);
init_vring_queue(dev->virtqueue[base_idx + VIRTIO_TXQ], qp_idx);
}
static void
reset_vring_queue(struct vhost_virtqueue *vq, int qp_idx)
reset_vring_queue(struct vhost_virtqueue *vq)
{
int callfd;
callfd = vq->callfd;
init_vring_queue(vq, qp_idx);
init_vring_queue(vq);
vq->callfd = callfd;
}
static void
reset_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx)
{
uint32_t base_idx = qp_idx * VIRTIO_QNUM;
reset_vring_queue(dev->virtqueue[base_idx + VIRTIO_RXQ], qp_idx);
reset_vring_queue(dev->virtqueue[base_idx + VIRTIO_TXQ], qp_idx);
}
int
alloc_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx)
alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
{
struct vhost_virtqueue *virtqueue = NULL;
uint32_t virt_rx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_RXQ;
uint32_t virt_tx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_TXQ;
struct vhost_virtqueue *vq;
virtqueue = rte_malloc(NULL,
sizeof(struct vhost_virtqueue) * VIRTIO_QNUM, 0);
if (virtqueue == NULL) {
vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
if (vq == NULL) {
RTE_LOG(ERR, VHOST_CONFIG,
"Failed to allocate memory for virt qp:%d.\n", qp_idx);
"Failed to allocate memory for vring:%u.\n", vring_idx);
return -1;
}
dev->virtqueue[virt_rx_q_idx] = virtqueue;
dev->virtqueue[virt_tx_q_idx] = virtqueue + VIRTIO_TXQ;
dev->virtqueue[vring_idx] = vq;
init_vring_queue(vq);
init_vring_queue_pair(dev, qp_idx);
dev->virt_qp_nb += 1;
dev->nr_vring += 1;
return 0;
}
/*
* Reset some variables in device structure, while keeping few
* others untouched, such as vid, ifname, virt_qp_nb: they
* others untouched, such as vid, ifname, nr_vring: they
* should be same unless the device is removed.
*/
void
@ -220,8 +172,8 @@ reset_device(struct virtio_net *dev)
dev->protocol_features = 0;
dev->flags = 0;
for (i = 0; i < dev->virt_qp_nb; i++)
reset_vring_queue_pair(dev, i);
for (i = 0; i < dev->nr_vring; i++)
reset_vring_queue(dev->virtqueue[i]);
}
/*
@ -248,6 +200,7 @@ vhost_new_device(void)
if (i == MAX_VHOST_DEVICE) {
RTE_LOG(ERR, VHOST_CONFIG,
"Failed to find a free slot for new device.\n");
rte_free(dev);
return -1;
}
@ -271,7 +224,7 @@ vhost_destroy_device(int vid)
if (dev->flags & VIRTIO_DEV_RUNNING) {
dev->flags &= ~VIRTIO_DEV_RUNNING;
notify_ops->destroy_device(vid);
dev->notify_ops->destroy_device(vid);
}
cleanup_device(dev, 1);
@ -308,6 +261,25 @@ vhost_enable_dequeue_zero_copy(int vid)
dev->dequeue_zero_copy = 1;
}
int
rte_vhost_get_mtu(int vid, uint16_t *mtu)
{
struct virtio_net *dev = get_device(vid);
if (!dev)
return -ENODEV;
if (!(dev->flags & VIRTIO_DEV_READY))
return -EAGAIN;
if (!(dev->features & VIRTIO_NET_F_MTU))
return -ENOTSUP;
*mtu = dev->mtu;
return 0;
}
int
rte_vhost_get_numa_node(int vid)
{
@ -342,7 +314,18 @@ rte_vhost_get_queue_num(int vid)
if (dev == NULL)
return 0;
return dev->virt_qp_nb;
return dev->nr_vring / 2;
}
uint16_t
rte_vhost_get_vring_num(int vid)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return 0;
return dev->nr_vring;
}
int
@ -361,6 +344,75 @@ rte_vhost_get_ifname(int vid, char *buf, size_t len)
return 0;
}
int
rte_vhost_get_negotiated_features(int vid, uint64_t *features)
{
struct virtio_net *dev;
dev = get_device(vid);
if (!dev)
return -1;
*features = dev->features;
return 0;
}
int
rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem)
{
struct virtio_net *dev;
struct rte_vhost_memory *m;
size_t size;
dev = get_device(vid);
if (!dev)
return -1;
size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region);
m = malloc(sizeof(struct rte_vhost_memory) + size);
if (!m)
return -1;
m->nregions = dev->mem->nregions;
memcpy(m->regions, dev->mem->regions, size);
*mem = m;
return 0;
}
int
rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
struct rte_vhost_vring *vring)
{
struct virtio_net *dev;
struct vhost_virtqueue *vq;
dev = get_device(vid);
if (!dev)
return -1;
if (vring_idx >= VHOST_MAX_VRING)
return -1;
vq = dev->virtqueue[vring_idx];
if (!vq)
return -1;
vring->desc = vq->desc;
vring->avail = vq->avail;
vring->used = vq->used;
vring->log_guest_addr = vq->log_guest_addr;
vring->callfd = vq->callfd;
vring->kickfd = vq->kickfd;
vring->size = vq->size;
vring->last_avail_idx = vq->last_avail_idx;
vring->last_used_idx = vq->last_used_idx;
return 0;
}
uint16_t
rte_vhost_avail_entries(int vid, uint16_t queue_id)
{
@ -396,33 +448,56 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
return 0;
}
uint64_t rte_vhost_feature_get(void)
void
rte_vhost_log_write(int vid, uint64_t addr, uint64_t len)
{
return VHOST_FEATURES;
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return;
vhost_log_write(dev, addr, len);
}
int rte_vhost_feature_disable(uint64_t feature_mask)
void
rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
uint64_t offset, uint64_t len)
{
VHOST_FEATURES = VHOST_FEATURES & ~feature_mask;
return 0;
struct virtio_net *dev;
struct vhost_virtqueue *vq;
dev = get_device(vid);
if (dev == NULL)
return;
if (vring_idx >= VHOST_MAX_VRING)
return;
vq = dev->virtqueue[vring_idx];
if (!vq)
return;
vhost_log_used_vring(dev, vq, offset, len);
}
int rte_vhost_feature_enable(uint64_t feature_mask)
{
if ((feature_mask & VHOST_SUPPORTED_FEATURES) == feature_mask) {
VHOST_FEATURES = VHOST_FEATURES | feature_mask;
return 0;
}
return -1;
}
/*
* Register ops so that we can add/remove device to data core.
*/
int
rte_vhost_driver_callback_register(struct virtio_net_device_ops const * const ops)
{
notify_ops = ops;
rte_vhost_set_vhost_vring_last_idx(int vid, uint16_t vring_idx,
uint16_t last_avail_idx, uint16_t last_used_idx) {
struct virtio_net *dev;
struct vhost_virtqueue *vq;
dev = get_device(vid);
if (!dev)
return -1;
if (vring_idx >= VHOST_MAX_VRING)
return -1;
vq = dev->virtqueue[vring_idx];
if (!vq)
return -1;
vq->last_avail_idx = last_avail_idx;
vq->last_used_idx = last_used_idx;
return 0;
}

143
lib/vhost/rte_vhost/vhost.h
View File

@ -39,14 +39,20 @@
#include <sys/queue.h>
#include <unistd.h>
#include <linux/vhost.h>
#include <linux/virtio_net.h>
#include <sys/socket.h>
#include <linux/if.h>
#include <rte_log.h>
#include <rte_ether.h>
#include "rte_virtio_net.h"
#include "rte_vhost.h"
#include "vhost_user.h"
/* Used to indicate that the device is running on a data core */
#define VIRTIO_DEV_RUNNING 1
/* Used to indicate that the device is ready to operate */
#define VIRTIO_DEV_READY 2
/* Backend value set by guest. */
#define VIRTIO_DEV_STOPPED -1
@ -111,24 +117,20 @@ struct vhost_virtqueue {
uint16_t shadow_used_idx;
} __rte_cache_aligned;
/* Old kernels have no such macro defined */
/* Old kernels have no such macros defined */
#ifndef VIRTIO_NET_F_GUEST_ANNOUNCE
#define VIRTIO_NET_F_GUEST_ANNOUNCE 21
#endif
#ifndef VIRTIO_NET_F_MQ
#define VIRTIO_NET_F_MQ 22
#endif
/*
* Make an extra wrapper for VIRTIO_NET_F_MQ and
* VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as they are
* introduced since kernel v3.8. This makes our
* code buildable for older kernel.
*/
#ifdef VIRTIO_NET_F_MQ
#define VHOST_MAX_QUEUE_PAIRS VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX
#define VHOST_SUPPORTS_MQ (1ULL << VIRTIO_NET_F_MQ)
#else
#define VHOST_MAX_QUEUE_PAIRS 1
#define VHOST_SUPPORTS_MQ 0
#define VHOST_MAX_VRING 0x100
#define VHOST_MAX_QUEUE_PAIRS 0x80
#ifndef VIRTIO_NET_F_MTU
#define VIRTIO_NET_F_MTU 3
#endif
/*
@ -138,6 +140,27 @@ struct vhost_virtqueue {
#define VIRTIO_F_VERSION_1 32
#endif
#define VHOST_USER_F_PROTOCOL_FEATURES 30
/* Features supported by this builtin vhost-user net driver. */
#define VIRTIO_NET_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \
(1ULL << VIRTIO_NET_F_CTRL_VQ) | \
(1ULL << VIRTIO_NET_F_CTRL_RX) | \
(1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
(1ULL << VIRTIO_NET_F_MQ) | \
(1ULL << VIRTIO_F_VERSION_1) | \
(1ULL << VHOST_F_LOG_ALL) | \
(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
(1ULL << VIRTIO_NET_F_HOST_TSO4) | \
(1ULL << VIRTIO_NET_F_HOST_TSO6) | \
(1ULL << VIRTIO_NET_F_CSUM) | \
(1ULL << VIRTIO_NET_F_GUEST_CSUM) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
(1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \
(1ULL << VIRTIO_NET_F_MTU))
struct guest_page {
uint64_t guest_phys_addr;
uint64_t host_phys_addr;
@ -150,7 +173,7 @@ struct guest_page {
*/
struct virtio_net {
/* Frontend (QEMU) memory and memory region information */
struct virtio_memory *mem;
struct rte_vhost_memory *mem;
uint64_t features;
uint64_t protocol_features;
int vid;
@ -158,8 +181,7 @@ struct virtio_net {
uint16_t vhost_hlen;
/* to tell if we need broadcast rarp packet */
rte_atomic16_t broadcast_rarp;
uint32_t virt_qp_nb;
uint32_t num_queues;
uint32_t nr_vring;
int dequeue_zero_copy;
struct vhost_virtqueue *virtqueue[VHOST_MAX_QUEUE_PAIRS * 2];
#define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
@ -168,38 +190,55 @@ struct virtio_net {
uint64_t log_base;
uint64_t log_addr;
struct ether_addr mac;
uint16_t mtu;
struct vhost_device_ops const *notify_ops;
uint32_t nr_guest_pages;
uint32_t max_guest_pages;
struct guest_page *guest_pages;
int has_new_mem_table;
struct VhostUserMemory mem_table;
int mem_table_fds[VHOST_MEMORY_MAX_NREGIONS];
int has_new_mem_table;
struct VhostUserMemory mem_table;
int mem_table_fds[VHOST_MEMORY_MAX_NREGIONS];
} __rte_cache_aligned;
/**
* Information relating to memory regions including offsets to
* addresses in QEMUs memory file.
*/
struct virtio_memory_region {
uint64_t guest_phys_addr;
uint64_t guest_user_addr;
uint64_t host_user_addr;
uint64_t size;
void *mmap_addr;
uint64_t mmap_size;
int fd;
};
#define VHOST_LOG_PAGE 4096
/**
* Memory structure includes region and mapping information.
*/
struct virtio_memory {
uint32_t nregions;
struct virtio_memory_region regions[0];
};
static inline void __attribute__((always_inline))
vhost_log_page(uint8_t *log_base, uint64_t page)
{
log_base[page / 8] |= 1 << (page % 8);
}
static inline void __attribute__((always_inline))
vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
{
uint64_t page;
if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) ||
!dev->log_base || !len))
return;
if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
return;
/* To make sure guest memory updates are committed before logging */
rte_smp_wmb();
page = addr / VHOST_LOG_PAGE;
while (page * VHOST_LOG_PAGE < addr + len) {
vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
page += 1;
}
}
static inline void __attribute__((always_inline))
vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq,
uint64_t offset, uint64_t len)
{
vhost_log_write(dev, vq->log_guest_addr + offset, len);
}
/* Macros for printing using RTE_LOG */
#define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1
@ -236,25 +275,6 @@ extern uint64_t VHOST_FEATURES;
#define MAX_VHOST_DEVICE 1024
extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
/* Convert guest physical Address to host virtual address */
static inline uint64_t __attribute__((always_inline))
gpa_to_vva(struct virtio_net *dev, uint64_t gpa)
{
struct virtio_memory_region *reg;
uint32_t i;
for (i = 0; i < dev->mem->nregions; i++) {
reg = &dev->mem->regions[i];
if (gpa >= reg->guest_phys_addr &&
gpa < reg->guest_phys_addr + reg->size) {
return gpa - reg->guest_phys_addr +
reg->host_user_addr;
}
}
return 0;
}
/* Convert guest physical address to host physical address */
static inline phys_addr_t __attribute__((always_inline))
gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
@ -275,7 +295,6 @@ gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
return 0;
}
extern struct virtio_net_device_ops const *notify_ops;
struct virtio_net *get_device(int vid);
int vhost_new_device(void);
@ -283,11 +302,13 @@ void cleanup_device(struct virtio_net *dev, int destroy);
void reset_device(struct virtio_net *dev);
void vhost_destroy_device(int);
int alloc_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx);
int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx);
void vhost_set_ifname(int, const char *if_name, unsigned int if_len);
void vhost_enable_dequeue_zero_copy(int vid);
struct vhost_device_ops const *vhost_driver_callback_get(const char *path);
/*
* Backend-specific cleanup.
*

251
lib/vhost/rte_vhost/vhost_user.c
View File

@ -51,6 +51,9 @@
#include "vhost.h"
#include "vhost_user.h"
#define VIRTIO_MIN_MTU 68
#define VIRTIO_MAX_MTU 65535
static const char *vhost_message_str[VHOST_USER_MAX] = {
[VHOST_USER_NONE] = "VHOST_USER_NONE",
[VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
@ -72,6 +75,7 @@ static const char *vhost_message_str[VHOST_USER_MAX] = {
[VHOST_USER_GET_QUEUE_NUM] = "VHOST_USER_GET_QUEUE_NUM",
[VHOST_USER_SET_VRING_ENABLE] = "VHOST_USER_SET_VRING_ENABLE",
[VHOST_USER_SEND_RARP] = "VHOST_USER_SEND_RARP",
[VHOST_USER_NET_SET_MTU] = "VHOST_USER_NET_SET_MTU",
};
static uint64_t
@ -88,7 +92,7 @@ static void
free_mem_region(struct virtio_net *dev)
{
uint32_t i;
struct virtio_memory_region *reg;
struct rte_vhost_mem_region *reg;
if (!dev || !dev->mem)
return;
@ -110,6 +114,10 @@ vhost_backend_cleanup(struct virtio_net *dev)
rte_free(dev->mem);
dev->mem = NULL;
}
free(dev->guest_pages);
dev->guest_pages = NULL;
if (dev->log_addr) {
munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
dev->log_addr = 0;
@ -131,7 +139,7 @@ vhost_user_reset_owner(struct virtio_net *dev)
{
if (dev->flags & VIRTIO_DEV_RUNNING) {
dev->flags &= ~VIRTIO_DEV_RUNNING;
notify_ops->destroy_device(dev->vid);
dev->notify_ops->destroy_device(dev->vid);
}
cleanup_device(dev, 0);
@ -143,9 +151,12 @@ vhost_user_reset_owner(struct virtio_net *dev)
* The features that we support are requested.
*/
static uint64_t
vhost_user_get_features(void)
vhost_user_get_features(struct virtio_net *dev)
{
return VHOST_FEATURES;
uint64_t features = 0;
rte_vhost_driver_get_features(dev->ifname, &features);
return features;
}
/*
@ -154,9 +165,17 @@ vhost_user_get_features(void)
static int
vhost_user_set_features(struct virtio_net *dev, uint64_t features)
{
if (features & ~VHOST_FEATURES)
uint64_t vhost_features = 0;
rte_vhost_driver_get_features(dev->ifname, &vhost_features);
if (features & ~vhost_features)
return -1;
if ((dev->flags & VIRTIO_DEV_RUNNING) && dev->features != features) {
if (dev->notify_ops->features_changed)
dev->notify_ops->features_changed(dev->vid, features);
}
dev->features = features;
if (dev->features &
((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
@ -223,13 +242,6 @@ numa_realloc(struct virtio_net *dev, int index)
struct vhost_virtqueue *old_vq, *vq;
int ret;
/*
* vq is allocated on pairs, we should try to do realloc
* on first queue of one queue pair only.
*/
if (index % VIRTIO_QNUM != 0)
return dev;
old_dev = dev;
vq = old_vq = dev->virtqueue[index];
@ -247,8 +259,7 @@ numa_realloc(struct virtio_net *dev, int index)
if (oldnode != newnode) {
RTE_LOG(INFO, VHOST_CONFIG,
"reallocate vq from %d to %d node\n", oldnode, newnode);
vq = rte_malloc_socket(NULL, sizeof(*vq) * VIRTIO_QNUM, 0,
newnode);
vq = rte_malloc_socket(NULL, sizeof(*vq), 0, newnode);
if (!vq)
return dev;
@ -280,7 +291,6 @@ numa_realloc(struct virtio_net *dev, int index)
out:
dev->virtqueue[index] = vq;
dev->virtqueue[index + 1] = vq + 1;
vhost_devices[dev->vid] = dev;
return dev;
@ -300,7 +310,7 @@ numa_realloc(struct virtio_net *dev, int index __rte_unused)
static uint64_t
qva_to_vva(struct virtio_net *dev, uint64_t qva)
{
struct virtio_memory_region *reg;
struct rte_vhost_mem_region *reg;
uint32_t i;
/* Find the region where the address lives. */
@ -401,6 +411,12 @@ static int
vhost_user_set_vring_base(struct virtio_net *dev,
VhostUserMsg *msg)
{
/* Remove from the data plane. */
if (dev->flags & VIRTIO_DEV_RUNNING) {
dev->flags &= ~VIRTIO_DEV_RUNNING;
dev->notify_ops->destroy_device(dev->vid);
}
dev->virtqueue[msg->payload.state.index]->last_used_idx = msg->payload.state.num;
dev->virtqueue[msg->payload.state.index]->last_avail_idx = msg->payload.state.num;
@ -413,15 +429,10 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
{
struct guest_page *page, *last_page;
if (dev->nr_guest_pages == dev->max_guest_pages &&
dev->nr_guest_pages > 0) {
dev->max_guest_pages *= 2;
if (dev->nr_guest_pages == dev->max_guest_pages) {
dev->max_guest_pages = RTE_MAX(8U, dev->max_guest_pages * 2);
dev->guest_pages = realloc(dev->guest_pages,
dev->max_guest_pages * sizeof(*page));
if (!dev->guest_pages) {
RTE_LOG(ERR, VHOST_CONFIG, "cannot realloc guest_pages\n");
abort();
}
}
if (dev->nr_guest_pages > 0) {
@ -441,7 +452,7 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
}
static void
add_guest_pages(struct virtio_net *dev, struct virtio_memory_region *reg,
add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
uint64_t page_size)
{
uint64_t reg_size = reg->size;
@ -460,14 +471,14 @@ add_guest_pages(struct virtio_net *dev, struct virtio_memory_region *reg,
reg_size -= size;
while (reg_size > 0) {
size = RTE_MIN(reg_size, page_size);
host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t)
host_user_addr);
add_one_guest_page(dev, guest_phys_addr, host_phys_addr,
page_size);
add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size);
host_user_addr += page_size;
guest_phys_addr += page_size;
reg_size -= page_size;
host_user_addr += size;
guest_phys_addr += size;
reg_size -= size;
}
}
@ -520,11 +531,11 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg)
return 0;
}
static int
static int
vhost_setup_mem_table(struct virtio_net *dev)
{
struct VhostUserMemory memory = dev->mem_table;
struct virtio_memory_region *reg;
struct rte_vhost_mem_region *reg;
void *mmap_addr;
uint64_t mmap_size;
uint64_t mmap_offset;
@ -545,8 +556,8 @@ vhost_setup_mem_table(struct virtio_net *dev)
sizeof(struct guest_page));
}
dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct virtio_memory) +
sizeof(struct virtio_memory_region) * memory.nregions, 0);
dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct rte_vhost_memory) +
sizeof(struct rte_vhost_mem_region) * memory.nregions, 0);
if (dev->mem == NULL) {
RTE_LOG(ERR, VHOST_CONFIG,
"(%d) failed to allocate memory for dev->mem\n",
@ -597,7 +608,8 @@ vhost_setup_mem_table(struct virtio_net *dev)
reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr +
mmap_offset;
add_guest_pages(dev, reg, alignment);
if (dev->dequeue_zero_copy)
add_guest_pages(dev, reg, alignment);
RTE_LOG(INFO, VHOST_CONFIG,
"guest memory region %u, size: 0x%" PRIx64 "\n"
@ -643,14 +655,14 @@ virtio_is_ready(struct virtio_net *dev)
struct vhost_virtqueue *vq;
uint32_t i;
for (i = 0; i < dev->num_queues; i++) {
if (dev->nr_vring == 0)
return 0;
for (i = 0; i < dev->nr_vring; i++) {
vq = dev->virtqueue[i];
if (!vq_is_ready(vq)) {
RTE_LOG(INFO, VHOST_CONFIG,
"virtio is not ready for processing.\n");
if (!vq_is_ready(vq))
return 0;
}
}
RTE_LOG(INFO, VHOST_CONFIG,
@ -663,7 +675,12 @@ vhost_user_set_vring_call(struct virtio_net *dev, struct VhostUserMsg *pmsg)
{
struct vhost_vring_file file;
struct vhost_virtqueue *vq;
uint32_t cur_qp_idx;
/* Remove from the data plane. */
if (dev->flags & VIRTIO_DEV_RUNNING) {
dev->flags &= ~VIRTIO_DEV_RUNNING;
dev->notify_ops->destroy_device(dev->vid);
}
file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
@ -673,43 +690,25 @@ vhost_user_set_vring_call(struct virtio_net *dev, struct VhostUserMsg *pmsg)
RTE_LOG(INFO, VHOST_CONFIG,
"vring call idx:%d file:%d\n", file.index, file.fd);
if (file.index + 1 > dev->num_queues) {
dev->num_queues = file.index + 1;
}
/*
* FIXME: VHOST_SET_VRING_CALL is the first per-vring message
* we get, so we do vring queue pair allocation here.
*/
cur_qp_idx = file.index / VIRTIO_QNUM;
if (cur_qp_idx + 1 > dev->virt_qp_nb) {
if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0)
return;
}
vq = dev->virtqueue[file.index];
assert(vq != NULL);
if (vq->callfd >= 0)
close(vq->callfd);
vq->callfd = file.fd;
if (virtio_is_ready(dev) && !(dev->flags & VIRTIO_DEV_RUNNING)) {
notify_ops->new_device(dev->vid);
}
}
/*
* In vhost-user, when we receive kick message, will test whether virtio
* device is ready for packet processing.
*/
static void
vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg)
{
struct vhost_vring_file file;
struct vhost_virtqueue *vq;
/* Remove from the data plane. */
if (dev->flags & VIRTIO_DEV_RUNNING) {
dev->flags &= ~VIRTIO_DEV_RUNNING;
dev->notify_ops->destroy_device(dev->vid);
}
file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
file.fd = VIRTIO_INVALID_EVENTFD;
@ -722,16 +721,6 @@ vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg)
if (vq->kickfd >= 0)
close(vq->kickfd);
vq->kickfd = file.fd;
if (virtio_is_ready(dev) && !(dev->flags & VIRTIO_DEV_RUNNING)) {
if (dev->dequeue_zero_copy) {
RTE_LOG(INFO, VHOST_CONFIG,
"dequeue zero copy is enabled\n");
}
if (notify_ops->new_device(dev->vid) == 0)
dev->flags |= VIRTIO_DEV_RUNNING;
}
}
static void
@ -762,9 +751,11 @@ vhost_user_get_vring_base(struct virtio_net *dev,
/* We have to stop the queue (virtio) if it is running. */
if (dev->flags & VIRTIO_DEV_RUNNING) {
dev->flags &= ~VIRTIO_DEV_RUNNING;
notify_ops->destroy_device(dev->vid);
dev->notify_ops->destroy_device(dev->vid);
}
dev->flags &= ~VIRTIO_DEV_READY;
/* Here we are safe to get the last used index */
msg->payload.state.num = vq->last_used_idx;
@ -779,6 +770,10 @@ vhost_user_get_vring_base(struct virtio_net *dev,
close(vq->kickfd);
vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
if (vq->callfd >= 0)
close(vq->callfd);
vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
if (dev->dequeue_zero_copy)
@ -803,8 +798,8 @@ vhost_user_set_vring_enable(struct virtio_net *dev,
"set queue enable: %d to qp idx: %d\n",
enable, msg->payload.state.index);
if (notify_ops->vring_state_changed)
notify_ops->vring_state_changed(dev->vid, msg->payload.state.index, enable);
if (dev->notify_ops->vring_state_changed)
dev->notify_ops->vring_state_changed(dev->vid, msg->payload.state.index, enable);
dev->virtqueue[msg->payload.state.index]->enabled = enable;
@ -902,6 +897,22 @@ vhost_user_send_rarp(struct virtio_net *dev, struct VhostUserMsg *msg)
return 0;
}
static int
vhost_user_net_set_mtu(struct virtio_net *dev, struct VhostUserMsg *msg)
{
if (msg->payload.u64 < VIRTIO_MIN_MTU ||
msg->payload.u64 > VIRTIO_MAX_MTU) {
RTE_LOG(ERR, VHOST_CONFIG, "Invalid MTU size (%"PRIu64")\n",
msg->payload.u64);
return -1;
}
dev->mtu = msg->payload.u64;
return 0;
}
/* return bytes# of read on success or negative val on failure. */
static int
read_vhost_message(int sockfd, struct VhostUserMsg *msg)
@ -941,6 +952,7 @@ send_vhost_message(int sockfd, struct VhostUserMsg *msg)
return 0;
msg->flags &= ~VHOST_USER_VERSION_MASK;
msg->flags &= ~VHOST_USER_NEED_REPLY;
msg->flags |= VHOST_USER_VERSION;
msg->flags |= VHOST_USER_REPLY_MASK;
@ -950,6 +962,44 @@ send_vhost_message(int sockfd, struct VhostUserMsg *msg)
return ret;
}
/*
* Allocate a queue pair if it hasn't been allocated yet
*/
static int
vhost_user_check_and_alloc_queue_pair(struct virtio_net *dev, VhostUserMsg *msg)
{
uint16_t vring_idx;
switch (msg->request) {
case VHOST_USER_SET_VRING_KICK:
case VHOST_USER_SET_VRING_CALL:
case VHOST_USER_SET_VRING_ERR:
vring_idx = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
break;
case VHOST_USER_SET_VRING_NUM:
case VHOST_USER_SET_VRING_BASE:
case VHOST_USER_SET_VRING_ENABLE:
vring_idx = msg->payload.state.index;
break;
case VHOST_USER_SET_VRING_ADDR:
vring_idx = msg->payload.addr.index;
break;
default:
return 0;
}
if (vring_idx >= VHOST_MAX_VRING) {
RTE_LOG(ERR, VHOST_CONFIG,
"invalid vring index: %u\n", vring_idx);
return -1;
}
if (dev->virtqueue[vring_idx])
return 0;
return alloc_vring_queue(dev, vring_idx);
}
int
vhost_user_msg_handler(int vid, int fd)
{
@ -961,6 +1011,16 @@ vhost_user_msg_handler(int vid, int fd)
if (dev == NULL)
return -1;
if (!dev->notify_ops) {
dev->notify_ops = vhost_driver_callback_get(dev->ifname);
if (!dev->notify_ops) {
RTE_LOG(ERR, VHOST_CONFIG,
"failed to get callback ops for driver %s\n",
dev->ifname);
return -1;
}
}
ret = read_vhost_message(fd, &msg);
if (ret <= 0 || msg.request >= VHOST_USER_MAX) {
if (ret < 0)
@ -978,9 +1038,17 @@ vhost_user_msg_handler(int vid, int fd)
RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n",
vhost_message_str[msg.request]);
ret = vhost_user_check_and_alloc_queue_pair(dev, &msg);
if (ret < 0) {
RTE_LOG(ERR, VHOST_CONFIG,
"failed to alloc queue\n");
return -1;
}
switch (msg.request) {
case VHOST_USER_GET_FEATURES:
msg.payload.u64 = vhost_user_get_features();
msg.payload.u64 = vhost_user_get_features(dev);
msg.size = sizeof(msg.payload.u64);
send_vhost_message(fd, &msg);
break;
@ -1005,7 +1073,7 @@ vhost_user_msg_handler(int vid, int fd)
break;
case VHOST_USER_SET_MEM_TABLE:
vhost_user_set_mem_table(dev, &msg);
ret = vhost_user_set_mem_table(dev, &msg);
break;
case VHOST_USER_SET_LOG_BASE:
@ -1062,10 +1130,35 @@ vhost_user_msg_handler(int vid, int fd)
vhost_user_send_rarp(dev, &msg);
break;
case VHOST_USER_NET_SET_MTU:
ret = vhost_user_net_set_mtu(dev, &msg);
break;
default:
ret = -1;
break;
}
if (msg.flags & VHOST_USER_NEED_REPLY) {
msg.payload.u64 = !!ret;
msg.size = sizeof(msg.payload.u64);
send_vhost_message(fd, &msg);
}
if (!(dev->flags & VIRTIO_DEV_RUNNING) && virtio_is_ready(dev)) {
dev->flags |= VIRTIO_DEV_READY;
if (!(dev->flags & VIRTIO_DEV_RUNNING)) {
if (dev->dequeue_zero_copy) {
RTE_LOG(INFO, VHOST_CONFIG,
"dequeue zero copy is enabled\n");
}
if (dev->notify_ops->new_device(dev->vid) == 0)
dev->flags |= VIRTIO_DEV_RUNNING;
}
}
return 0;
}

10
lib/vhost/rte_vhost/vhost_user.h
View File

@ -37,7 +37,7 @@
#include <stdint.h>
#include <linux/vhost.h>
#include "rte_virtio_net.h"
#include "rte_vhost.h"
/* refer to hw/virtio/vhost-user.c */
@ -46,10 +46,14 @@
#define VHOST_USER_PROTOCOL_F_MQ 0
#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1
#define VHOST_USER_PROTOCOL_F_RARP 2
#define VHOST_USER_PROTOCOL_F_REPLY_ACK 3
#define VHOST_USER_PROTOCOL_F_NET_MTU 4
#define VHOST_USER_PROTOCOL_FEATURES ((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
(1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |\
(1ULL << VHOST_USER_PROTOCOL_F_RARP))
(1ULL << VHOST_USER_PROTOCOL_F_RARP) | \
(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
(1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))
typedef enum VhostUserRequest {
VHOST_USER_NONE = 0,
@ -72,6 +76,7 @@ typedef enum VhostUserRequest {
VHOST_USER_GET_QUEUE_NUM = 17,
VHOST_USER_SET_VRING_ENABLE = 18,
VHOST_USER_SEND_RARP = 19,
VHOST_USER_NET_SET_MTU = 20,
VHOST_USER_MAX
} VhostUserRequest;
@ -98,6 +103,7 @@ typedef struct VhostUserMsg {
#define VHOST_USER_VERSION_MASK 0x3
#define VHOST_USER_REPLY_MASK (0x1 << 2)
#define VHOST_USER_NEED_REPLY (0x1 << 3)
uint32_t flags;
uint32_t size; /* the following payload size */
union {

1185
lib/vhost/rte_vhost/virtio_net.c
View File

File diff suppressed because it is too large Load Diff

45
lib/vhost/rte_vhost_17_05/Makefile
View File

@ -1,45 +0,0 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += -I.
CFLAGS += $(ENV_CFLAGS)
# These are the DPDK vhost files copied (for now) into SPDK
C_SRCS += fd_man.c socket.c vhost_user.c vhost.c
LIBNAME = rte_vhost
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

300
lib/vhost/rte_vhost_17_05/fd_man.c
View File

@ -1,300 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <string.h>
#include <rte_common.h>
#include <rte_log.h>
#include "fd_man.h"
#define FDPOLLERR (POLLERR | POLLHUP | POLLNVAL)
static int
get_last_valid_idx(struct fdset *pfdset, int last_valid_idx)
{
int i;
for (i = last_valid_idx; i >= 0 && pfdset->fd[i].fd == -1; i--)
;
return i;
}
static void
fdset_move(struct fdset *pfdset, int dst, int src)
{
pfdset->fd[dst] = pfdset->fd[src];
pfdset->rwfds[dst] = pfdset->rwfds[src];
}
static void
fdset_shrink_nolock(struct fdset *pfdset)
{
int i;
int last_valid_idx = get_last_valid_idx(pfdset, pfdset->num - 1);
for (i = 0; i < last_valid_idx; i++) {
if (pfdset->fd[i].fd != -1)
continue;
fdset_move(pfdset, i, last_valid_idx);
last_valid_idx = get_last_valid_idx(pfdset, last_valid_idx - 1);
}
pfdset->num = last_valid_idx + 1;
}
/*
* Find deleted fd entries and remove them
*/
static void
fdset_shrink(struct fdset *pfdset)
{
pthread_mutex_lock(&pfdset->fd_mutex);
fdset_shrink_nolock(pfdset);
pthread_mutex_unlock(&pfdset->fd_mutex);
}
/**
* Returns the index in the fdset for a given fd.
* @return
* index for the fd, or -1 if fd isn't in the fdset.
*/
static int
fdset_find_fd(struct fdset *pfdset, int fd)
{
int i;
for (i = 0; i < pfdset->num && pfdset->fd[i].fd != fd; i++)
;
return i == pfdset->num ? -1 : i;
}
static void
fdset_add_fd(struct fdset *pfdset, int idx, int fd,
fd_cb rcb, fd_cb wcb, void *dat)
{
struct fdentry *pfdentry = &pfdset->fd[idx];
struct pollfd *pfd = &pfdset->rwfds[idx];
pfdentry->fd = fd;
pfdentry->rcb = rcb;
pfdentry->wcb = wcb;
pfdentry->dat = dat;
pfd->fd = fd;
pfd->events = rcb ? POLLIN : 0;
pfd->events |= wcb ? POLLOUT : 0;
pfd->revents = 0;
}
void
fdset_init(struct fdset *pfdset)
{
int i;
if (pfdset == NULL)
return;
for (i = 0; i < MAX_FDS; i++) {
pfdset->fd[i].fd = -1;
pfdset->fd[i].dat = NULL;
}
pfdset->num = 0;
}
/**
* Register the fd in the fdset with read/write handler and context.
*/
int
fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat)
{
int i;
if (pfdset == NULL || fd == -1)
return -1;
pthread_mutex_lock(&pfdset->fd_mutex);
i = pfdset->num < MAX_FDS ? pfdset->num++ : -1;
if (i == -1) {
fdset_shrink_nolock(pfdset);
i = pfdset->num < MAX_FDS ? pfdset->num++ : -1;
if (i == -1) {
pthread_mutex_unlock(&pfdset->fd_mutex);
return -2;
}
}
fdset_add_fd(pfdset, i, fd, rcb, wcb, dat);
pthread_mutex_unlock(&pfdset->fd_mutex);
return 0;
}
/**
* Unregister the fd from the fdset.
* Returns context of a given fd or NULL.
*/
void *
fdset_del(struct fdset *pfdset, int fd)
{
int i;
void *dat = NULL;
if (pfdset == NULL || fd == -1)
return NULL;
do {
pthread_mutex_lock(&pfdset->fd_mutex);
i = fdset_find_fd(pfdset, fd);
if (i != -1 && pfdset->fd[i].busy == 0) {
/* busy indicates r/wcb is executing! */
dat = pfdset->fd[i].dat;
pfdset->fd[i].fd = -1;
pfdset->fd[i].rcb = pfdset->fd[i].wcb = NULL;
pfdset->fd[i].dat = NULL;
i = -1;
}
pthread_mutex_unlock(&pfdset->fd_mutex);
} while (i != -1);
return dat;
}
/**
* This functions runs in infinite blocking loop until there is no fd in
* pfdset. It calls corresponding r/w handler if there is event on the fd.
*
* Before the callback is called, we set the flag to busy status; If other
* thread(now rte_vhost_driver_unregister) calls fdset_del concurrently, it
* will wait until the flag is reset to zero(which indicates the callback is
* finished), then it could free the context after fdset_del.
*/
void *
fdset_event_dispatch(void *arg)
{
int i;
struct pollfd *pfd;
struct fdentry *pfdentry;
fd_cb rcb, wcb;
void *dat;
int fd, numfds;
int remove1, remove2;
int need_shrink;
struct fdset *pfdset = arg;
if (pfdset == NULL)
return NULL;
while (1) {
/*
* When poll is blocked, other threads might unregister
* listenfds from and register new listenfds into fdset.
* When poll returns, the entries for listenfds in the fdset
* might have been updated. It is ok if there is unwanted call
* for new listenfds.
*/
pthread_mutex_lock(&pfdset->fd_mutex);
numfds = pfdset->num;
pthread_mutex_unlock(&pfdset->fd_mutex);
poll(pfdset->rwfds, numfds, 1000 /* millisecs */);
need_shrink = 0;
for (i = 0; i < numfds; i++) {
pthread_mutex_lock(&pfdset->fd_mutex);
pfdentry = &pfdset->fd[i];
fd = pfdentry->fd;
pfd = &pfdset->rwfds[i];
if (fd < 0) {
need_shrink = 1;
pthread_mutex_unlock(&pfdset->fd_mutex);
continue;
}
if (!pfd->revents) {
pthread_mutex_unlock(&pfdset->fd_mutex);
continue;
}
remove1 = remove2 = 0;
rcb = pfdentry->rcb;
wcb = pfdentry->wcb;
dat = pfdentry->dat;
pfdentry->busy = 1;
pthread_mutex_unlock(&pfdset->fd_mutex);
if (rcb && pfd->revents & (POLLIN | FDPOLLERR))
rcb(fd, dat, &remove1);
if (wcb && pfd->revents & (POLLOUT | FDPOLLERR))
wcb(fd, dat, &remove2);
pfdentry->busy = 0;
/*
* fdset_del needs to check busy flag.
* We don't allow fdset_del to be called in callback
* directly.
*/
/*
* When we are to clean up the fd from fdset,
* because the fd is closed in the cb,
* the old fd val could be reused by when creates new
* listen fd in another thread, we couldn't call
* fd_set_del.
*/
if (remove1 || remove2) {
pfdentry->fd = -1;
need_shrink = 1;
}
}
if (need_shrink)
fdset_shrink(pfdset);
}
return NULL;
}

69
lib/vhost/rte_vhost_17_05/fd_man.h
View File

@ -1,69 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _FD_MAN_H_
#define _FD_MAN_H_
#include <stdint.h>
#include <pthread.h>
#include <poll.h>
#define MAX_FDS 1024
typedef void (*fd_cb)(int fd, void *dat, int *remove);
struct fdentry {
int fd; /* -1 indicates this entry is empty */
fd_cb rcb; /* callback when this fd is readable. */
fd_cb wcb; /* callback when this fd is writeable. */
void *dat; /* fd context */
int busy; /* whether this entry is being used in cb. */
};
struct fdset {
struct pollfd rwfds[MAX_FDS];
struct fdentry fd[MAX_FDS];
pthread_mutex_t fd_mutex;
int num; /* current fd number of this fdset */
};
void fdset_init(struct fdset *pfdset);
int fdset_add(struct fdset *pfdset, int fd,
fd_cb rcb, fd_cb wcb, void *dat);
void *fdset_del(struct fdset *pfdset, int fd);
void *fdset_event_dispatch(void *arg);
#endif

802
lib/vhost/rte_vhost_17_05/socket.c
View File

@ -1,802 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdint.h>
#include <stdio.h>
#include <stdbool.h>
#include <limits.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/queue.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <rte_log.h>
#include "fd_man.h"
#include "vhost.h"
#include "vhost_user.h"
TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
/*
* Every time rte_vhost_driver_register() is invoked, an associated
* vhost_user_socket struct will be created.
*/
struct vhost_user_socket {
struct vhost_user_connection_list conn_list;
pthread_mutex_t conn_mutex;
char *path;
int socket_fd;
struct sockaddr_un un;
bool is_server;
bool reconnect;
bool dequeue_zero_copy;
/*
* The "supported_features" indicates the feature bits the
* vhost driver supports. The "features" indicates the feature
* bits after the rte_vhost_driver_features_disable/enable().
* It is also the final feature bits used for vhost-user
* features negotiation.
*/
uint64_t supported_features;
uint64_t features;
struct vhost_device_ops const *notify_ops;
};
struct vhost_user_connection {
struct vhost_user_socket *vsocket;
int connfd;
int vid;
TAILQ_ENTRY(vhost_user_connection) next;
};
#define MAX_VHOST_SOCKET 1024
struct vhost_user {
struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET];
struct fdset fdset;
int vsocket_cnt;
pthread_mutex_t mutex;
};
#define MAX_VIRTIO_BACKLOG 128
static void vhost_user_server_new_connection(int fd, void *data, int *remove);
static void vhost_user_read_cb(int fd, void *dat, int *remove);
static int create_unix_socket(struct vhost_user_socket *vsocket);
static int vhost_user_start_client(struct vhost_user_socket *vsocket);
static struct vhost_user vhost_user = {
.fdset = {
.fd = { [0 ... MAX_FDS - 1] = {-1, NULL, NULL, NULL, 0} },
.fd_mutex = PTHREAD_MUTEX_INITIALIZER,
.num = 0
},
.vsocket_cnt = 0,
.mutex = PTHREAD_MUTEX_INITIALIZER,
};
/* return bytes# of read on success or negative val on failure. */
int
read_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num)
{
struct iovec iov;
struct msghdr msgh;
size_t fdsize = fd_num * sizeof(int);
char control[CMSG_SPACE(fdsize)];
struct cmsghdr *cmsg;
int ret;
memset(&msgh, 0, sizeof(msgh));
iov.iov_base = buf;
iov.iov_len = buflen;
msgh.msg_iov = &iov;
msgh.msg_iovlen = 1;
msgh.msg_control = control;
msgh.msg_controllen = sizeof(control);
ret = recvmsg(sockfd, &msgh, 0);
if (ret <= 0) {
RTE_LOG(ERR, VHOST_CONFIG, "recvmsg failed\n");
return ret;
}
if (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) {
RTE_LOG(ERR, VHOST_CONFIG, "truncted msg\n");
return -1;
}
for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL;
cmsg = CMSG_NXTHDR(&msgh, cmsg)) {
if ((cmsg->cmsg_level == SOL_SOCKET) &&
(cmsg->cmsg_type == SCM_RIGHTS)) {
memcpy(fds, CMSG_DATA(cmsg), fdsize);
break;
}
}
return ret;
}
int
send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num)
{
struct iovec iov;
struct msghdr msgh;
size_t fdsize = fd_num * sizeof(int);
char control[CMSG_SPACE(fdsize)];
struct cmsghdr *cmsg;
int ret;
memset(&msgh, 0, sizeof(msgh));
iov.iov_base = buf;
iov.iov_len = buflen;
msgh.msg_iov = &iov;
msgh.msg_iovlen = 1;
if (fds && fd_num > 0) {
msgh.msg_control = control;
msgh.msg_controllen = sizeof(control);
cmsg = CMSG_FIRSTHDR(&msgh);
if (cmsg == NULL) {
RTE_LOG(ERR, VHOST_CONFIG, "cmsg == NULL\n");
errno = EINVAL;
return -1;
}
cmsg->cmsg_len = CMSG_LEN(fdsize);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
memcpy(CMSG_DATA(cmsg), fds, fdsize);
} else {
msgh.msg_control = NULL;
msgh.msg_controllen = 0;
}
do {
ret = sendmsg(sockfd, &msgh, 0);
} while (ret < 0 && errno == EINTR);
if (ret < 0) {
RTE_LOG(ERR, VHOST_CONFIG, "sendmsg error\n");
return ret;
}
return ret;
}
static void
vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
{
int vid;
size_t size;
struct vhost_user_connection *conn;
int ret;
conn = malloc(sizeof(*conn));
if (conn == NULL) {
close(fd);
return;
}
vid = vhost_new_device();
if (vid == -1) {
close(fd);
free(conn);
return;
}
size = strnlen(vsocket->path, PATH_MAX);
vhost_set_ifname(vid, vsocket->path, size);
if (vsocket->dequeue_zero_copy)
vhost_enable_dequeue_zero_copy(vid);
RTE_LOG(INFO, VHOST_CONFIG, "new device, handle is %d\n", vid);
conn->connfd = fd;
conn->vsocket = vsocket;
conn->vid = vid;
ret = fdset_add(&vhost_user.fdset, fd, vhost_user_read_cb,
NULL, conn);
if (ret < 0) {
conn->connfd = -1;
free(conn);
close(fd);
RTE_LOG(ERR, VHOST_CONFIG,
"failed to add fd %d into vhost server fdset\n",
fd);
}
pthread_mutex_lock(&vsocket->conn_mutex);
TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next);
pthread_mutex_unlock(&vsocket->conn_mutex);
}
/* call back when there is new vhost-user connection from client */
static void
vhost_user_server_new_connection(int fd, void *dat, int *remove __rte_unused)
{
struct vhost_user_socket *vsocket = dat;
fd = accept(fd, NULL, NULL);
if (fd < 0)
return;
RTE_LOG(INFO, VHOST_CONFIG, "new vhost user connection is %d\n", fd);
vhost_user_add_connection(fd, vsocket);
}
static void
vhost_user_read_cb(int connfd, void *dat, int *remove)
{
struct vhost_user_connection *conn = dat;
struct vhost_user_socket *vsocket = conn->vsocket;
int ret;
ret = vhost_user_msg_handler(conn->vid, connfd);
if (ret < 0) {
close(connfd);
*remove = 1;
vhost_destroy_device(conn->vid);
pthread_mutex_lock(&vsocket->conn_mutex);
TAILQ_REMOVE(&vsocket->conn_list, conn, next);
pthread_mutex_unlock(&vsocket->conn_mutex);
free(conn);
if (vsocket->reconnect) {
create_unix_socket(vsocket);
vhost_user_start_client(vsocket);
}
}
}
static int
create_unix_socket(struct vhost_user_socket *vsocket)
{
int fd;
struct sockaddr_un *un = &vsocket->un;
fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (fd < 0)
return -1;
RTE_LOG(INFO, VHOST_CONFIG, "vhost-user %s: socket created, fd: %d\n",
vsocket->is_server ? "server" : "client", fd);
if (!vsocket->is_server && fcntl(fd, F_SETFL, O_NONBLOCK)) {
RTE_LOG(ERR, VHOST_CONFIG,
"vhost-user: can't set nonblocking mode for socket, fd: "
"%d (%s)\n", fd, strerror(errno));
close(fd);
return -1;
}
memset(un, 0, sizeof(*un));
un->sun_family = AF_UNIX;
strncpy(un->sun_path, vsocket->path, sizeof(un->sun_path));
un->sun_path[sizeof(un->sun_path) - 1] = '\0';
vsocket->socket_fd = fd;
return 0;
}
static int
vhost_user_start_server(struct vhost_user_socket *vsocket)
{
int ret;
int fd = vsocket->socket_fd;
const char *path = vsocket->path;
ret = bind(fd, (struct sockaddr *)&vsocket->un, sizeof(vsocket->un));
if (ret < 0) {
RTE_LOG(ERR, VHOST_CONFIG,
"failed to bind to %s: %s; remove it and try again\n",
path, strerror(errno));
goto err;
}
RTE_LOG(INFO, VHOST_CONFIG, "bind to %s\n", path);
ret = listen(fd, MAX_VIRTIO_BACKLOG);
if (ret < 0)
goto err;
ret = fdset_add(&vhost_user.fdset, fd, vhost_user_server_new_connection,
NULL, vsocket);
if (ret < 0) {
RTE_LOG(ERR, VHOST_CONFIG,
"failed to add listen fd %d to vhost server fdset\n",
fd);
goto err;
}
return 0;
err:
close(fd);
return -1;
}
struct vhost_user_reconnect {
struct sockaddr_un un;
int fd;
struct vhost_user_socket *vsocket;
TAILQ_ENTRY(vhost_user_reconnect) next;
};
TAILQ_HEAD(vhost_user_reconnect_tailq_list, vhost_user_reconnect);
struct vhost_user_reconnect_list {
struct vhost_user_reconnect_tailq_list head;
pthread_mutex_t mutex;
};
static struct vhost_user_reconnect_list reconn_list;
static pthread_t reconn_tid;
static int
vhost_user_connect_nonblock(int fd, struct sockaddr *un, size_t sz)
{
int ret, flags;
ret = connect(fd, un, sz);
if (ret < 0 && errno != EISCONN)
return -1;
flags = fcntl(fd, F_GETFL, 0);
if (flags < 0) {
RTE_LOG(ERR, VHOST_CONFIG,
"can't get flags for connfd %d\n", fd);
return -2;
}
if ((flags & O_NONBLOCK) && fcntl(fd, F_SETFL, flags & ~O_NONBLOCK)) {
RTE_LOG(ERR, VHOST_CONFIG,
"can't disable nonblocking on fd %d\n", fd);
return -2;
}
return 0;
}
static void *
vhost_user_client_reconnect(void *arg __rte_unused)
{
int ret;
struct vhost_user_reconnect *reconn, *next;
while (1) {
pthread_mutex_lock(&reconn_list.mutex);
/*
* An equal implementation of TAILQ_FOREACH_SAFE,
* which does not exist on all platforms.
*/
for (reconn = TAILQ_FIRST(&reconn_list.head);
reconn != NULL; reconn = next) {
next = TAILQ_NEXT(reconn, next);
ret = vhost_user_connect_nonblock(reconn->fd,
(struct sockaddr *)&reconn->un,
sizeof(reconn->un));
if (ret == -2) {
close(reconn->fd);
RTE_LOG(ERR, VHOST_CONFIG,
"reconnection for fd %d failed\n",
reconn->fd);
goto remove_fd;
}
if (ret == -1)
continue;
RTE_LOG(INFO, VHOST_CONFIG,
"%s: connected\n", reconn->vsocket->path);
vhost_user_add_connection(reconn->fd, reconn->vsocket);
remove_fd:
TAILQ_REMOVE(&reconn_list.head, reconn, next);
free(reconn);
}
pthread_mutex_unlock(&reconn_list.mutex);
sleep(1);
}
return NULL;
}
static int
vhost_user_reconnect_init(void)
{
int ret;
pthread_mutex_init(&reconn_list.mutex, NULL);
TAILQ_INIT(&reconn_list.head);
ret = pthread_create(&reconn_tid, NULL,
vhost_user_client_reconnect, NULL);
if (ret < 0)
RTE_LOG(ERR, VHOST_CONFIG, "failed to create reconnect thread");
return ret;
}
static int
vhost_user_start_client(struct vhost_user_socket *vsocket)
{
int ret;
int fd = vsocket->socket_fd;
const char *path = vsocket->path;
struct vhost_user_reconnect *reconn;
ret = vhost_user_connect_nonblock(fd, (struct sockaddr *)&vsocket->un,
sizeof(vsocket->un));
if (ret == 0) {
vhost_user_add_connection(fd, vsocket);
return 0;
}
RTE_LOG(WARNING, VHOST_CONFIG,
"failed to connect to %s: %s\n",
path, strerror(errno));
if (ret == -2 || !vsocket->reconnect) {
close(fd);
return -1;
}
RTE_LOG(INFO, VHOST_CONFIG, "%s: reconnecting...\n", path);
reconn = malloc(sizeof(*reconn));
if (reconn == NULL) {
RTE_LOG(ERR, VHOST_CONFIG,
"failed to allocate memory for reconnect\n");
close(fd);
return -1;
}
reconn->un = vsocket->un;
reconn->fd = fd;
reconn->vsocket = vsocket;
pthread_mutex_lock(&reconn_list.mutex);
TAILQ_INSERT_TAIL(&reconn_list.head, reconn, next);
pthread_mutex_unlock(&reconn_list.mutex);
return 0;
}
static struct vhost_user_socket *
find_vhost_user_socket(const char *path)
{
int i;
for (i = 0; i < vhost_user.vsocket_cnt; i++) {
struct vhost_user_socket *vsocket = vhost_user.vsockets[i];
if (!strcmp(vsocket->path, path))
return vsocket;
}
return NULL;
}
int
rte_vhost_driver_disable_features(const char *path, uint64_t features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket)
vsocket->features &= ~features;
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
int
rte_vhost_driver_enable_features(const char *path, uint64_t features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket) {
if ((vsocket->supported_features & features) != features) {
/*
* trying to enable features the driver doesn't
* support.
*/
pthread_mutex_unlock(&vhost_user.mutex);
return -1;
}
vsocket->features |= features;
}
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
int
rte_vhost_driver_set_features(const char *path, uint64_t features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket) {
vsocket->supported_features = features;
vsocket->features = features;
}
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
int
rte_vhost_driver_get_features(const char *path, uint64_t *features)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket)
*features = vsocket->features;
pthread_mutex_unlock(&vhost_user.mutex);
if (!vsocket) {
RTE_LOG(ERR, VHOST_CONFIG,
"socket file %s is not registered yet.\n", path);
return -1;
} else {
return 0;
}
}
/*
* Register a new vhost-user socket; here we could act as server
* (the default case), or client (when RTE_VHOST_USER_CLIENT) flag
* is set.
*/
int
rte_vhost_driver_register(const char *path, uint64_t flags)
{
int ret = -1;
struct vhost_user_socket *vsocket;
if (!path)
return -1;
pthread_mutex_lock(&vhost_user.mutex);
if (vhost_user.vsocket_cnt == MAX_VHOST_SOCKET) {
RTE_LOG(ERR, VHOST_CONFIG,
"error: the number of vhost sockets reaches maximum\n");
goto out;
}
vsocket = malloc(sizeof(struct vhost_user_socket));
if (!vsocket)
goto out;
memset(vsocket, 0, sizeof(struct vhost_user_socket));
vsocket->path = strdup(path);
TAILQ_INIT(&vsocket->conn_list);
pthread_mutex_init(&vsocket->conn_mutex, NULL);
vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
/*
* Set the supported features correctly for the builtin vhost-user
* net driver.
*
* Applications know nothing about features the builtin virtio net
* driver (virtio_net.c) supports, thus it's not possible for them
* to invoke rte_vhost_driver_set_features(). To workaround it, here
* we set it unconditionally. If the application want to implement
* another vhost-user driver (say SCSI), it should call the
* rte_vhost_driver_set_features(), which will overwrite following
* two values.
*/
vsocket->supported_features = VIRTIO_NET_SUPPORTED_FEATURES;
vsocket->features = VIRTIO_NET_SUPPORTED_FEATURES;
if ((flags & RTE_VHOST_USER_CLIENT) != 0) {
vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
if (vsocket->reconnect && reconn_tid == 0) {
if (vhost_user_reconnect_init() < 0) {
free(vsocket->path);
free(vsocket);
goto out;
}
}
} else {
vsocket->is_server = true;
}
ret = create_unix_socket(vsocket);
if (ret < 0) {
free(vsocket->path);
free(vsocket);
goto out;
}
vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket;
out:
pthread_mutex_unlock(&vhost_user.mutex);
return ret;
}
static bool
vhost_user_remove_reconnect(struct vhost_user_socket *vsocket)
{
int found = false;
struct vhost_user_reconnect *reconn, *next;
pthread_mutex_lock(&reconn_list.mutex);
for (reconn = TAILQ_FIRST(&reconn_list.head);
reconn != NULL; reconn = next) {
next = TAILQ_NEXT(reconn, next);
if (reconn->vsocket == vsocket) {
TAILQ_REMOVE(&reconn_list.head, reconn, next);
close(reconn->fd);
free(reconn);
found = true;
break;
}
}
pthread_mutex_unlock(&reconn_list.mutex);
return found;
}
/**
* Unregister the specified vhost socket
*/
int
rte_vhost_driver_unregister(const char *path)
{
int i;
int count;
struct vhost_user_connection *conn, *next;
pthread_mutex_lock(&vhost_user.mutex);
for (i = 0; i < vhost_user.vsocket_cnt; i++) {
struct vhost_user_socket *vsocket = vhost_user.vsockets[i];
if (!strcmp(vsocket->path, path)) {
if (vsocket->is_server) {
fdset_del(&vhost_user.fdset, vsocket->socket_fd);
close(vsocket->socket_fd);
unlink(path);
} else if (vsocket->reconnect) {
vhost_user_remove_reconnect(vsocket);
}
pthread_mutex_lock(&vsocket->conn_mutex);
for (conn = TAILQ_FIRST(&vsocket->conn_list);
conn != NULL;
conn = next) {
next = TAILQ_NEXT(conn, next);
fdset_del(&vhost_user.fdset, conn->connfd);
RTE_LOG(INFO, VHOST_CONFIG,
"free connfd = %d for device '%s'\n",
conn->connfd, path);
close(conn->connfd);
vhost_destroy_device(conn->vid);
TAILQ_REMOVE(&vsocket->conn_list, conn, next);
free(conn);
}
pthread_mutex_unlock(&vsocket->conn_mutex);
free(vsocket->path);
free(vsocket);
count = --vhost_user.vsocket_cnt;
vhost_user.vsockets[i] = vhost_user.vsockets[count];
vhost_user.vsockets[count] = NULL;
pthread_mutex_unlock(&vhost_user.mutex);
return 0;
}
}
pthread_mutex_unlock(&vhost_user.mutex);
return -1;
}
/*
* Register ops so that we can add/remove device to data core.
*/
int
rte_vhost_driver_callback_register(const char *path,
struct vhost_device_ops const * const ops)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
if (vsocket)
vsocket->notify_ops = ops;
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? 0 : -1;
}
struct vhost_device_ops const *
vhost_driver_callback_get(const char *path)
{
struct vhost_user_socket *vsocket;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
pthread_mutex_unlock(&vhost_user.mutex);
return vsocket ? vsocket->notify_ops : NULL;
}
int
rte_vhost_driver_start(const char *path)
{
struct vhost_user_socket *vsocket;
static pthread_t fdset_tid;
pthread_mutex_lock(&vhost_user.mutex);
vsocket = find_vhost_user_socket(path);
pthread_mutex_unlock(&vhost_user.mutex);
if (!vsocket)
return -1;
if (fdset_tid == 0) {
int ret = pthread_create(&fdset_tid, NULL, fdset_event_dispatch,
&vhost_user.fdset);
if (ret < 0)
RTE_LOG(ERR, VHOST_CONFIG,
"failed to create fdset handling thread");
}
if (vsocket->is_server)
return vhost_user_start_server(vsocket);
else
return vhost_user_start_client(vsocket);
}

503
lib/vhost/rte_vhost_17_05/vhost.c
View File

@ -1,503 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <linux/vhost.h>
#include <linux/virtio_net.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#ifdef RTE_LIBRTE_VHOST_NUMA
#include <numaif.h>
#endif
#include <rte_ethdev.h>
#include <rte_log.h>
#include <rte_string_fns.h>
#include <rte_memory.h>
#include <rte_malloc.h>
#include <rte_vhost.h>
#include "vhost.h"
struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
struct virtio_net *
get_device(int vid)
{
struct virtio_net *dev = vhost_devices[vid];
if (unlikely(!dev)) {
RTE_LOG(ERR, VHOST_CONFIG,
"(%d) device not found.\n", vid);
}
return dev;
}
static void
cleanup_vq(struct vhost_virtqueue *vq, int destroy)
{
if ((vq->callfd >= 0) && (destroy != 0))
close(vq->callfd);
if (vq->kickfd >= 0)
close(vq->kickfd);
}
/*
* Unmap any memory, close any file descriptors and
* free any memory owned by a device.
*/
void
cleanup_device(struct virtio_net *dev, int destroy)
{
uint32_t i;
vhost_backend_cleanup(dev);
for (i = 0; i < dev->nr_vring; i++)
cleanup_vq(dev->virtqueue[i], destroy);
}
/*
* Release virtqueues and device memory.
*/
static void
free_device(struct virtio_net *dev)
{
uint32_t i;
struct vhost_virtqueue *vq;
for (i = 0; i < dev->nr_vring; i++) {
vq = dev->virtqueue[i];
rte_free(vq->shadow_used_ring);
rte_free(vq);
}
rte_free(dev);
}
static void
init_vring_queue(struct vhost_virtqueue *vq)
{
memset(vq, 0, sizeof(struct vhost_virtqueue));
vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;
/* Backends are set to -1 indicating an inactive device. */
vq->backend = -1;
/*
* always set the vq to enabled; this is to keep compatibility
* with the old QEMU, whereas there is no SET_VRING_ENABLE message.
*/
vq->enabled = 1;
TAILQ_INIT(&vq->zmbuf_list);
}
static void
reset_vring_queue(struct vhost_virtqueue *vq)
{
int callfd;
callfd = vq->callfd;
init_vring_queue(vq);
vq->callfd = callfd;
}
int
alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx)
{
struct vhost_virtqueue *vq;
vq = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
if (vq == NULL) {
RTE_LOG(ERR, VHOST_CONFIG,
"Failed to allocate memory for vring:%u.\n", vring_idx);
return -1;
}
dev->virtqueue[vring_idx] = vq;
init_vring_queue(vq);
dev->nr_vring += 1;
return 0;
}
/*
* Reset some variables in device structure, while keeping few
* others untouched, such as vid, ifname, nr_vring: they
* should be same unless the device is removed.
*/
void
reset_device(struct virtio_net *dev)
{
uint32_t i;
dev->features = 0;
dev->protocol_features = 0;
dev->flags = 0;
for (i = 0; i < dev->nr_vring; i++)
reset_vring_queue(dev->virtqueue[i]);
}
/*
* Invoked when there is a new vhost-user connection established (when
* there is a new virtio device being attached).
*/
int
vhost_new_device(void)
{
struct virtio_net *dev;
int i;
dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0);
if (dev == NULL) {
RTE_LOG(ERR, VHOST_CONFIG,
"Failed to allocate memory for new dev.\n");
return -1;
}
for (i = 0; i < MAX_VHOST_DEVICE; i++) {
if (vhost_devices[i] == NULL)
break;
}
if (i == MAX_VHOST_DEVICE) {
RTE_LOG(ERR, VHOST_CONFIG,
"Failed to find a free slot for new device.\n");
rte_free(dev);
return -1;
}
vhost_devices[i] = dev;
dev->vid = i;
return i;
}
/*
* Invoked when there is the vhost-user connection is broken (when
* the virtio device is being detached).
*/
void
vhost_destroy_device(int vid)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return;
if (dev->flags & VIRTIO_DEV_RUNNING) {
dev->flags &= ~VIRTIO_DEV_RUNNING;
dev->notify_ops->destroy_device(vid);
}
cleanup_device(dev, 1);
free_device(dev);
vhost_devices[vid] = NULL;
}
void
vhost_set_ifname(int vid, const char *if_name, unsigned int if_len)
{
struct virtio_net *dev;
unsigned int len;
dev = get_device(vid);
if (dev == NULL)
return;
len = if_len > sizeof(dev->ifname) ?
sizeof(dev->ifname) : if_len;
strncpy(dev->ifname, if_name, len);
dev->ifname[sizeof(dev->ifname) - 1] = '\0';
}
void
vhost_enable_dequeue_zero_copy(int vid)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return;
dev->dequeue_zero_copy = 1;
}
int
rte_vhost_get_mtu(int vid, uint16_t *mtu)
{
struct virtio_net *dev = get_device(vid);
if (!dev)
return -ENODEV;
if (!(dev->flags & VIRTIO_DEV_READY))
return -EAGAIN;
if (!(dev->features & VIRTIO_NET_F_MTU))
return -ENOTSUP;
*mtu = dev->mtu;
return 0;
}
int
rte_vhost_get_numa_node(int vid)
{
#ifdef RTE_LIBRTE_VHOST_NUMA
struct virtio_net *dev = get_device(vid);
int numa_node;
int ret;
if (dev == NULL)
return -1;
ret = get_mempolicy(&numa_node, NULL, 0, dev,
MPOL_F_NODE | MPOL_F_ADDR);
if (ret < 0) {
RTE_LOG(ERR, VHOST_CONFIG,
"(%d) failed to query numa node: %d\n", vid, ret);
return -1;
}
return numa_node;
#else
RTE_SET_USED(vid);
return -1;
#endif
}
uint32_t
rte_vhost_get_queue_num(int vid)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return 0;
return dev->nr_vring / 2;
}
uint16_t
rte_vhost_get_vring_num(int vid)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return 0;
return dev->nr_vring;
}
int
rte_vhost_get_ifname(int vid, char *buf, size_t len)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return -1;
len = RTE_MIN(len, sizeof(dev->ifname));
strncpy(buf, dev->ifname, len);
buf[len - 1] = '\0';
return 0;
}
int
rte_vhost_get_negotiated_features(int vid, uint64_t *features)
{
struct virtio_net *dev;
dev = get_device(vid);
if (!dev)
return -1;
*features = dev->features;
return 0;
}
int
rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem)
{
struct virtio_net *dev;
struct rte_vhost_memory *m;
size_t size;
dev = get_device(vid);
if (!dev)
return -1;
size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region);
m = malloc(sizeof(struct rte_vhost_memory) + size);
if (!m)
return -1;
m->nregions = dev->mem->nregions;
memcpy(m->regions, dev->mem->regions, size);
*mem = m;
return 0;
}
int
rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
struct rte_vhost_vring *vring)
{
struct virtio_net *dev;
struct vhost_virtqueue *vq;
dev = get_device(vid);
if (!dev)
return -1;
if (vring_idx >= VHOST_MAX_VRING)
return -1;
vq = dev->virtqueue[vring_idx];
if (!vq)
return -1;
vring->desc = vq->desc;
vring->avail = vq->avail;
vring->used = vq->used;
vring->log_guest_addr = vq->log_guest_addr;
vring->callfd = vq->callfd;
vring->kickfd = vq->kickfd;
vring->size = vq->size;
vring->last_avail_idx = vq->last_avail_idx;
vring->last_used_idx = vq->last_used_idx;
return 0;
}
uint16_t
rte_vhost_avail_entries(int vid, uint16_t queue_id)
{
struct virtio_net *dev;
struct vhost_virtqueue *vq;
dev = get_device(vid);
if (!dev)
return 0;
vq = dev->virtqueue[queue_id];
if (!vq->enabled)
return 0;
return *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx;
}
int
rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return -1;
if (enable) {
RTE_LOG(ERR, VHOST_CONFIG,
"guest notification isn't supported.\n");
return -1;
}
dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
return 0;
}
void
rte_vhost_log_write(int vid, uint64_t addr, uint64_t len)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return;
vhost_log_write(dev, addr, len);
}
void
rte_vhost_log_used_vring(int vid, uint16_t vring_idx,
uint64_t offset, uint64_t len)
{
struct virtio_net *dev;
struct vhost_virtqueue *vq;
dev = get_device(vid);
if (dev == NULL)
return;
if (vring_idx >= VHOST_MAX_VRING)
return;
vq = dev->virtqueue[vring_idx];
if (!vq)
return;
vhost_log_used_vring(dev, vq, offset, len);
}
int
rte_vhost_set_vhost_vring_last_idx(int vid, uint16_t vring_idx,
uint16_t last_avail_idx, uint16_t last_used_idx) {
struct virtio_net *dev;
struct vhost_virtqueue *vq;
dev = get_device(vid);
if (!dev)
return -1;
if (vring_idx >= VHOST_MAX_VRING)
return -1;
vq = dev->virtqueue[vring_idx];
if (!vq)
return -1;
vq->last_avail_idx = last_avail_idx;
vq->last_used_idx = last_used_idx;
return 0;
}

319
lib/vhost/rte_vhost_17_05/vhost.h
View File

@ -1,319 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _VHOST_NET_CDEV_H_
#define _VHOST_NET_CDEV_H_
#include <stdint.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <unistd.h>
#include <linux/vhost.h>
#include <linux/virtio_net.h>
#include <sys/socket.h>
#include <linux/if.h>
#include <rte_log.h>
#include <rte_ether.h>
#include "rte_vhost.h"
#include "vhost_user.h"
/* Used to indicate that the device is running on a data core */
#define VIRTIO_DEV_RUNNING 1
/* Used to indicate that the device is ready to operate */
#define VIRTIO_DEV_READY 2
/* Backend value set by guest. */
#define VIRTIO_DEV_STOPPED -1
#define BUF_VECTOR_MAX 256
/**
* Structure contains buffer address, length and descriptor index
* from vring to do scatter RX.
*/
struct buf_vector {
uint64_t buf_addr;
uint32_t buf_len;
uint32_t desc_idx;
};
/*
* A structure to hold some fields needed in zero copy code path,
* mainly for associating an mbuf with the right desc_idx.
*/
struct zcopy_mbuf {
struct rte_mbuf *mbuf;
uint32_t desc_idx;
uint16_t in_use;
TAILQ_ENTRY(zcopy_mbuf) next;
};
TAILQ_HEAD(zcopy_mbuf_list, zcopy_mbuf);
/**
* Structure contains variables relevant to RX/TX virtqueues.
*/
struct vhost_virtqueue {
struct vring_desc *desc;
struct vring_avail *avail;
struct vring_used *used;
uint32_t size;
uint16_t last_avail_idx;
uint16_t last_used_idx;
#define VIRTIO_INVALID_EVENTFD (-1)
#define VIRTIO_UNINITIALIZED_EVENTFD (-2)
/* Backend value to determine if device should started/stopped */
int backend;
/* Used to notify the guest (trigger interrupt) */
int callfd;
/* Currently unused as polling mode is enabled */
int kickfd;
int enabled;
/* Physical address of used ring, for logging */
uint64_t log_guest_addr;
uint16_t nr_zmbuf;
uint16_t zmbuf_size;
uint16_t last_zmbuf_idx;
struct zcopy_mbuf *zmbufs;
struct zcopy_mbuf_list zmbuf_list;
struct vring_used_elem *shadow_used_ring;
uint16_t shadow_used_idx;
} __rte_cache_aligned;
/* Old kernels have no such macros defined */
#ifndef VIRTIO_NET_F_GUEST_ANNOUNCE
#define VIRTIO_NET_F_GUEST_ANNOUNCE 21
#endif
#ifndef VIRTIO_NET_F_MQ
#define VIRTIO_NET_F_MQ 22
#endif
#define VHOST_MAX_VRING 0x100
#define VHOST_MAX_QUEUE_PAIRS 0x80
#ifndef VIRTIO_NET_F_MTU
#define VIRTIO_NET_F_MTU 3
#endif
/*
* Define virtio 1.0 for older kernels
*/
#ifndef VIRTIO_F_VERSION_1
#define VIRTIO_F_VERSION_1 32
#endif
#define VHOST_USER_F_PROTOCOL_FEATURES 30
/* Features supported by this builtin vhost-user net driver. */
#define VIRTIO_NET_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \
(1ULL << VIRTIO_NET_F_CTRL_VQ) | \
(1ULL << VIRTIO_NET_F_CTRL_RX) | \
(1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \
(1ULL << VIRTIO_NET_F_MQ) | \
(1ULL << VIRTIO_F_VERSION_1) | \
(1ULL << VHOST_F_LOG_ALL) | \
(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
(1ULL << VIRTIO_NET_F_HOST_TSO4) | \
(1ULL << VIRTIO_NET_F_HOST_TSO6) | \
(1ULL << VIRTIO_NET_F_CSUM) | \
(1ULL << VIRTIO_NET_F_GUEST_CSUM) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
(1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \
(1ULL << VIRTIO_NET_F_MTU))
struct guest_page {
uint64_t guest_phys_addr;
uint64_t host_phys_addr;
uint64_t size;
};
/**
* Device structure contains all configuration information relating
* to the device.
*/
struct virtio_net {
/* Frontend (QEMU) memory and memory region information */
struct rte_vhost_memory *mem;
uint64_t features;
uint64_t protocol_features;
int vid;
uint32_t flags;
uint16_t vhost_hlen;
/* to tell if we need broadcast rarp packet */
rte_atomic16_t broadcast_rarp;
uint32_t nr_vring;
int dequeue_zero_copy;
struct vhost_virtqueue *virtqueue[VHOST_MAX_QUEUE_PAIRS * 2];
#define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
char ifname[IF_NAME_SZ];
uint64_t log_size;
uint64_t log_base;
uint64_t log_addr;
struct ether_addr mac;
uint16_t mtu;
struct vhost_device_ops const *notify_ops;
uint32_t nr_guest_pages;
uint32_t max_guest_pages;
struct guest_page *guest_pages;
int has_new_mem_table;
struct VhostUserMemory mem_table;
int mem_table_fds[VHOST_MEMORY_MAX_NREGIONS];
} __rte_cache_aligned;
#define VHOST_LOG_PAGE 4096
static inline void __attribute__((always_inline))
vhost_log_page(uint8_t *log_base, uint64_t page)
{
log_base[page / 8] |= 1 << (page % 8);
}
static inline void __attribute__((always_inline))
vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len)
{
uint64_t page;
if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) ||
!dev->log_base || !len))
return;
if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8)))
return;
/* To make sure guest memory updates are committed before logging */
rte_smp_wmb();
page = addr / VHOST_LOG_PAGE;
while (page * VHOST_LOG_PAGE < addr + len) {
vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
page += 1;
}
}
static inline void __attribute__((always_inline))
vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq,
uint64_t offset, uint64_t len)
{
vhost_log_write(dev, vq->log_guest_addr + offset, len);
}
/* Macros for printing using RTE_LOG */
#define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1
#define RTE_LOGTYPE_VHOST_DATA RTE_LOGTYPE_USER1
#ifdef RTE_LIBRTE_VHOST_DEBUG
#define VHOST_MAX_PRINT_BUFF 6072
#define LOG_LEVEL RTE_LOG_DEBUG
#define LOG_DEBUG(log_type, fmt, args...) RTE_LOG(DEBUG, log_type, fmt, ##args)
#define PRINT_PACKET(device, addr, size, header) do { \
char *pkt_addr = (char *)(addr); \
unsigned int index; \
char packet[VHOST_MAX_PRINT_BUFF]; \
\
if ((header)) \
snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Header size %d: ", (device->vid), (size)); \
else \
snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Packet size %d: ", (device->vid), (size)); \
for (index = 0; index < (size); index++) { \
snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), \
"%02hhx ", pkt_addr[index]); \
} \
snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), "\n"); \
\
LOG_DEBUG(VHOST_DATA, "%s", packet); \
} while (0)
#else
#define LOG_LEVEL RTE_LOG_INFO
#define LOG_DEBUG(log_type, fmt, args...) do {} while (0)
#define PRINT_PACKET(device, addr, size, header) do {} while (0)
#endif
extern uint64_t VHOST_FEATURES;
#define MAX_VHOST_DEVICE 1024
extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
/* Convert guest physical address to host physical address */
static inline phys_addr_t __attribute__((always_inline))
gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
{
uint32_t i;
struct guest_page *page;
for (i = 0; i < dev->nr_guest_pages; i++) {
page = &dev->guest_pages[i];
if (gpa >= page->guest_phys_addr &&
gpa + size < page->guest_phys_addr + page->size) {
return gpa - page->guest_phys_addr +
page->host_phys_addr;
}
}
return 0;
}
struct virtio_net *get_device(int vid);
int vhost_new_device(void);
void cleanup_device(struct virtio_net *dev, int destroy);
void reset_device(struct virtio_net *dev);
void vhost_destroy_device(int);
int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx);
void vhost_set_ifname(int, const char *if_name, unsigned int if_len);
void vhost_enable_dequeue_zero_copy(int vid);
struct vhost_device_ops const *vhost_driver_callback_get(const char *path);
/*
* Backend-specific cleanup.
*
* TODO: fix it; we have one backend now
*/
void vhost_backend_cleanup(struct virtio_net *dev);
#endif /* _VHOST_NET_CDEV_H_ */

1164
lib/vhost/rte_vhost_17_05/vhost_user.c
View File

File diff suppressed because it is too large Load Diff

134
lib/vhost/rte_vhost_17_05/vhost_user.h
View File

@ -1,134 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _VHOST_NET_USER_H
#define _VHOST_NET_USER_H
#include <stdint.h>
#include <linux/vhost.h>
#include "rte_vhost.h"
/* refer to hw/virtio/vhost-user.c */
#define VHOST_MEMORY_MAX_NREGIONS 8
#define VHOST_USER_PROTOCOL_F_MQ 0
#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1
#define VHOST_USER_PROTOCOL_F_RARP 2
#define VHOST_USER_PROTOCOL_F_REPLY_ACK 3
#define VHOST_USER_PROTOCOL_F_NET_MTU 4
#define VHOST_USER_PROTOCOL_FEATURES ((1ULL << VHOST_USER_PROTOCOL_F_MQ) | \
(1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |\
(1ULL << VHOST_USER_PROTOCOL_F_RARP) | \
(1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
(1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))
typedef enum VhostUserRequest {
VHOST_USER_NONE = 0,
VHOST_USER_GET_FEATURES = 1,
VHOST_USER_SET_FEATURES = 2,
VHOST_USER_SET_OWNER = 3,
VHOST_USER_RESET_OWNER = 4,
VHOST_USER_SET_MEM_TABLE = 5,
VHOST_USER_SET_LOG_BASE = 6,
VHOST_USER_SET_LOG_FD = 7,
VHOST_USER_SET_VRING_NUM = 8,
VHOST_USER_SET_VRING_ADDR = 9,
VHOST_USER_SET_VRING_BASE = 10,
VHOST_USER_GET_VRING_BASE = 11,
VHOST_USER_SET_VRING_KICK = 12,
VHOST_USER_SET_VRING_CALL = 13,
VHOST_USER_SET_VRING_ERR = 14,
VHOST_USER_GET_PROTOCOL_FEATURES = 15,
VHOST_USER_SET_PROTOCOL_FEATURES = 16,
VHOST_USER_GET_QUEUE_NUM = 17,
VHOST_USER_SET_VRING_ENABLE = 18,
VHOST_USER_SEND_RARP = 19,
VHOST_USER_NET_SET_MTU = 20,
VHOST_USER_MAX
} VhostUserRequest;
typedef struct VhostUserMemoryRegion {
uint64_t guest_phys_addr;
uint64_t memory_size;
uint64_t userspace_addr;
uint64_t mmap_offset;
} VhostUserMemoryRegion;
typedef struct VhostUserMemory {
uint32_t nregions;
uint32_t padding;
VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
} VhostUserMemory;
typedef struct VhostUserLog {
uint64_t mmap_size;
uint64_t mmap_offset;
} VhostUserLog;
typedef struct VhostUserMsg {
VhostUserRequest request;
#define VHOST_USER_VERSION_MASK 0x3
#define VHOST_USER_REPLY_MASK (0x1 << 2)
#define VHOST_USER_NEED_REPLY (0x1 << 3)
uint32_t flags;
uint32_t size; /* the following payload size */
union {
#define VHOST_USER_VRING_IDX_MASK 0xff
#define VHOST_USER_VRING_NOFD_MASK (0x1<<8)
uint64_t u64;
struct vhost_vring_state state;
struct vhost_vring_addr addr;
VhostUserMemory memory;
VhostUserLog log;
} payload;
int fds[VHOST_MEMORY_MAX_NREGIONS];
} __attribute((packed)) VhostUserMsg;
#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
/* The version of the protocol we support */
#define VHOST_USER_VERSION 0x1
/* vhost_user.c */
int vhost_user_msg_handler(int vid, int fd);
/* socket.c */
int read_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num);
int send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num);
#endif

2
lib/vhost/task.h
View File

@ -52,7 +52,7 @@ struct spdk_vhost_task {
int req_idx;
struct vhost_virtqueue *vq;
struct rte_vhost_vring *vq;
TAILQ_ENTRY(spdk_vhost_task) iovecs_link;
};

278
lib/vhost/vhost.c
View File

@ -36,11 +36,7 @@
#include <linux/virtio_ring.h>
#include <linux/virtio_scsi.h>
#include <rte_config.h>
#include <rte_malloc.h>
#include <rte_virtio_net.h>
#include <vhost.h>
#include <vhost_user.h>
#include <rte_vhost.h>
#include "spdk_internal/log.h"
#include "spdk/env.h"
@ -65,24 +61,110 @@ static uint32_t g_num_ctrlrs[RTE_MAX_LCORE];
/* Path to folder where character device will be created. Can be set by user. */
static char dev_dirname[PATH_MAX] = "";
struct spdk_vaddr_region {
void *vaddr;
uint64_t len;
#define SPDK_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE
uint64_t host_user_addr;
uint64_t host_user_size;
#define MAX_VHOST_DEVICE 1024
#define VHOST_USER_F_PROTOCOL_FEATURES 30
/* Features supported by SPDK VHOST lib. */
#define SPDK_VHOST_SCSI_FEATURES ((1ULL << VIRTIO_F_VERSION_1) | \
(1ULL << VHOST_F_LOG_ALL) | \
(1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \
(1ULL << VIRTIO_SCSI_F_INOUT) | \
(1ULL << VIRTIO_SCSI_F_HOTPLUG) | \
(1ULL << VIRTIO_SCSI_F_CHANGE ) | \
(1ULL << VIRTIO_SCSI_F_T10_PI ))
/* Features that are specified in VIRTIO SCSI but currently not supported:
* - Live migration not supported yet
* - Hotplug/hotremove
* - LUN params change
* - T10 PI
*/
#define SPDK_VHOST_SCSI_DISABLED_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \
(1ULL << VIRTIO_SCSI_F_HOTPLUG) | \
(1ULL << VIRTIO_SCSI_F_CHANGE ) | \
(1ULL << VIRTIO_SCSI_F_T10_PI ))
struct spdk_vhost_dev {
struct rte_vhost_memory *mem;
int vid;
uint16_t num_queues;
struct rte_vhost_vring virtqueue[0] __attribute((aligned(SPDK_CACHE_LINE_SIZE)));
};
/*
* Device linked list structure for data path.
*/
static void
spdk_vhost_dev_free(struct spdk_vhost_dev *dev)
{
free(dev->mem);
spdk_free(dev);
}
static void
spdk_vhost_dev_destruct(struct spdk_vhost_dev *dev)
{
struct rte_vhost_vring *q;
uint16_t i;
for (i = 0; i < dev->num_queues; i++) {
q = &dev->virtqueue[i];
rte_vhost_set_vhost_vring_last_idx(dev->vid, i, q->last_avail_idx, q->last_used_idx);
}
spdk_vhost_dev_free(dev);
}
static struct spdk_vhost_dev *
spdk_vhost_dev_create(int vid)
{
uint16_t num_queues = rte_vhost_get_vring_num(vid);
size_t size = sizeof(struct spdk_vhost_dev) + num_queues * sizeof(struct rte_vhost_vring);
struct spdk_vhost_dev *dev = spdk_zmalloc(size, SPDK_CACHE_LINE_SIZE, NULL);
uint16_t i;
if (dev == NULL) {
SPDK_ERRLOG("vhost device %d: Failed to allocate new vhost device with %"PRIu16" queues\n", vid,
num_queues);
return NULL;
}
for (i = 0; i < num_queues; i++) {
if (rte_vhost_get_vhost_vring(vid, i, &dev->virtqueue[i])) {
SPDK_ERRLOG("vhost device %d: Failed to get information of queue %"PRIu16"\n", vid, i);
goto err;
}
/* Disable notifications. */
if (rte_vhost_enable_guest_notification(vid, i, 0) != 0) {
SPDK_ERRLOG("vhost device %d: Failed to disable guest notification on queue %"PRIu16"\n", vid, i);
goto err;
}
}
dev->vid = vid;
dev->num_queues = num_queues;
if (rte_vhost_get_mem_table(vid, &dev->mem) != 0) {
SPDK_ERRLOG("vhost device %d: Failed to get guest memory table\n", vid);
goto err;
}
return dev;
err:
spdk_vhost_dev_free(dev);
return NULL;
}
static uint64_t
gpa_to_vva(struct spdk_vhost_dev *vdev, uint64_t addr)
{
return rte_vhost_gpa_to_vva(vdev->mem, addr);
}
struct spdk_vhost_scsi_ctrlr {
char *name;
/**< Pointer to device created by vhost lib. */
struct virtio_net *dev;
struct spdk_vaddr_region region[VHOST_MEMORY_MAX_NREGIONS];
uint32_t nregions;
struct spdk_vhost_dev *dev;
/**< TODO make this an array of spdk_scsi_devs. The vhost scsi
* request will tell us which scsi_dev to use.
@ -107,7 +189,7 @@ static struct spdk_vhost_scsi_ctrlr *dpdk_vid_mapping[MAX_VHOST_DEVICE];
* Get available requests from avail ring.
*/
static uint16_t
vq_avail_ring_get(struct vhost_virtqueue *vq, uint16_t *reqs, uint16_t reqs_len)
vq_avail_ring_get(struct rte_vhost_vring *vq, uint16_t *reqs, uint16_t reqs_len)
{
struct vring_avail *avail = vq->avail;
uint16_t size_mask = vq->size - 1;
@ -135,7 +217,7 @@ vq_avail_ring_get(struct vhost_virtqueue *vq, uint16_t *reqs, uint16_t reqs_len)
* Enqueue id and len to used ring.
*/
static void
vq_used_ring_enqueue(struct vhost_virtqueue *vq, uint16_t id, uint32_t len)
vq_used_ring_enqueue(struct rte_vhost_vring *vq, uint16_t id, uint32_t len)
{
struct vring_used *used = vq->used;
uint16_t size_mask = vq->size - 1;
@ -310,7 +392,7 @@ get_scsi_lun(struct spdk_scsi_dev *scsi_dev, const __u8 *lun)
}
static void
process_ctrl_request(struct spdk_vhost_scsi_ctrlr *vdev, struct vhost_virtqueue *controlq,
process_ctrl_request(struct spdk_vhost_scsi_ctrlr *vdev, struct rte_vhost_vring *controlq,
uint16_t req_idx)
{
struct spdk_vhost_task *task;
@ -323,9 +405,9 @@ process_ctrl_request(struct spdk_vhost_scsi_ctrlr *vdev, struct vhost_virtqueue
ctrl_req = (void *)gpa_to_vva(vdev->dev, desc->addr);
SPDK_TRACELOG(SPDK_TRACE_VHOST_QUEUE,
"Processing controlq descriptor: desc %d/%p, desc_addr %p, len %d, flags %d, last_used_idx %d; enabled %d; kickfd %d; size %d\n",
"Processing controlq descriptor: desc %d/%p, desc_addr %p, len %d, flags %d, last_used_idx %d; kickfd %d; size %d\n",
req_idx, desc, (void *)desc->addr, desc->len, desc->flags, controlq->last_used_idx,
controlq->enabled, controlq->kickfd, controlq->size);
controlq->kickfd, controlq->size);
SPDK_TRACEDUMP(SPDK_TRACE_VHOST_QUEUE, "Request desriptor", (uint8_t *)ctrl_req,
desc->len);
@ -393,8 +475,8 @@ static int
task_data_setup(struct spdk_vhost_task *task,
struct virtio_scsi_cmd_req **req)
{
struct vhost_virtqueue *vq = task->vq;
struct virtio_net *dev = task->vdev->dev;
struct rte_vhost_vring *vq = task->vq;
struct spdk_vhost_dev *dev = task->vdev->dev;
struct vring_desc *desc = &task->vq->desc[task->req_idx];
struct iovec *iovs = task->scsi.iovs;
uint16_t iovcnt = 0, iovcnt_max = task->scsi.iovcnt;
@ -550,7 +632,7 @@ process_request(struct spdk_vhost_task *task)
}
static void
process_controlq(struct spdk_vhost_scsi_ctrlr *vdev, struct vhost_virtqueue *vq)
process_controlq(struct spdk_vhost_scsi_ctrlr *vdev, struct rte_vhost_vring *vq)
{
uint16_t reqs[32];
uint16_t reqs_cnt, i;
@ -562,7 +644,7 @@ process_controlq(struct spdk_vhost_scsi_ctrlr *vdev, struct vhost_virtqueue *vq)
}
static void
process_requestq(struct spdk_vhost_scsi_ctrlr *vdev, struct vhost_virtqueue *vq)
process_requestq(struct spdk_vhost_scsi_ctrlr *vdev, struct rte_vhost_vring *vq)
{
uint16_t reqs[32];
uint16_t reqs_cnt, i;
@ -600,7 +682,7 @@ vdev_controlq_worker(void *arg)
{
struct spdk_vhost_scsi_ctrlr *vdev = arg;
process_controlq(vdev, vdev->dev->virtqueue[VIRTIO_SCSI_CONTROLQ]);
process_controlq(vdev, &vdev->dev->virtqueue[VIRTIO_SCSI_CONTROLQ]);
}
static void
@ -610,7 +692,7 @@ vdev_worker(void *arg)
uint32_t q_idx;
for (q_idx = VIRTIO_SCSI_REQUESTQ; q_idx < vdev->dev->num_queues; q_idx++) {
process_requestq(vdev, vdev->dev->virtqueue[q_idx]);
process_requestq(vdev, &vdev->dev->virtqueue[q_idx]);
}
}
@ -653,9 +735,7 @@ static void
add_vdev_cb(void *arg1, void *arg2)
{
struct spdk_vhost_scsi_ctrlr *vdev = arg1;
struct virtio_memory_region *region;
struct spdk_vaddr_region *vregion;
uint64_t start, end, len;
struct rte_vhost_mem_region *region;
uint32_t i;
for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
@ -665,22 +745,18 @@ add_vdev_cb(void *arg1, void *arg2)
spdk_scsi_dev_allocate_io_channels(vdev->scsi_dev[i]);
}
SPDK_NOTICELOG("Started poller for vhost controller %s on lcore %d\n", vdev->name, vdev->lcore);
vdev->nregions = vdev->dev->mem->nregions;
for (i = 0; i < vdev->nregions; i++) {
for (i = 0; i < vdev->dev->mem->nregions; i++) {
uint64_t start, end, len;
region = &vdev->dev->mem->regions[i];
start = FLOOR_2MB(region->mmap_addr);
end = CEIL_2MB(region->mmap_addr + region->mmap_size);
len = end - start;
vregion = &vdev->region[i];
vregion->vaddr = (void *)start;
vregion->len = len;
vregion->host_user_addr = region->host_user_addr;
vregion->host_user_size = region->size;
SPDK_NOTICELOG("Registering VM memory for vtophys translation - 0x%jx len:0x%jx\n",
start, len);
spdk_mem_register((void *)start, len);
spdk_iommu_mem_register(region->host_user_addr, region->size);
SPDK_NOTICELOG("Registering VM memory for vtophys translation - %p len:0x%jx\n",
vdev->region[i].vaddr, vdev->region[i].len);
spdk_mem_register(vregion->vaddr, vregion->len);
spdk_iommu_mem_register(vregion->host_user_addr, vregion->host_user_size);
}
spdk_poller_register(&vdev->requestq_poller, vdev_worker, vdev, vdev->lcore, 0);
@ -693,7 +769,7 @@ static void
remove_vdev_cb(void *arg1, void *arg2)
{
struct spdk_vhost_scsi_ctrlr *vdev = arg1;
struct spdk_vaddr_region *reg;
struct rte_vhost_mem_region *region;
uint32_t i;
for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
@ -704,25 +780,30 @@ remove_vdev_cb(void *arg1, void *arg2)
}
SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", vdev->name);
for (i = 0; i < vdev->nregions; i++) {
reg = &vdev->region[i];
spdk_iommu_mem_unregister(reg->host_user_addr, reg->host_user_size);
spdk_mem_unregister(reg->vaddr, reg->len);
for (i = 0; i < vdev->dev->mem->nregions; i++) {
uint64_t start, end, len;
region = &vdev->dev->mem->regions[i];
start = FLOOR_2MB(region->mmap_addr);
end = CEIL_2MB(region->mmap_addr + region->mmap_size);
len = end - start;
spdk_iommu_mem_unregister(region->host_user_addr, region->size);
spdk_mem_unregister((void *)start, len);
}
vdev->nregions = 0;
sem_post((sem_t *)arg2);
}
static void
destroy_device(int vid)
{
struct spdk_vhost_scsi_ctrlr *vdev = dpdk_vid_mapping[vid];
struct spdk_vhost_scsi_ctrlr *vdev;
struct spdk_event *event;
sem_t done_sem;
uint32_t i;
assert(vid < MAX_VHOST_DEVICE);
vdev = dpdk_vid_mapping[vid];
event = vhost_sem_event_alloc(vdev->lcore, vdev_event_done_cb, NULL, &done_sem);
spdk_poller_unregister(&vdev->requestq_poller, event);
if (vhost_sem_timedwait(&done_sem, 1))
@ -749,6 +830,8 @@ destroy_device(int vid)
g_num_ctrlrs[vdev->lcore]--;
vdev->lcore = -1;
spdk_vhost_dev_destruct(vdev->dev);
vdev->dev = NULL;
dpdk_vid_mapping[vid] = NULL;
}
@ -781,6 +864,18 @@ spdk_vhost_scsi_ctrlr_find(const char *ctrlr_name)
return NULL;
}
static int new_device(int vid);
static void destroy_device(int vid);
/*
* These callback allow devices to be added to the data core when configuration
* has been fully complete.
*/
static const struct vhost_device_ops spdk_vhost_scsi_device_ops = {
.new_device = new_device,
.destroy_device = destroy_device,
};
int
spdk_vhost_scsi_ctrlr_construct(const char *name, uint64_t cpumask)
{
@ -836,17 +931,36 @@ spdk_vhost_scsi_ctrlr_construct(const char *name, uint64_t cpumask)
SPDK_ERRLOG("Check if domain socket %s already exists\n", path);
return -EIO;
}
if (rte_vhost_driver_set_features(path, SPDK_VHOST_SCSI_FEATURES) ||
rte_vhost_driver_disable_features(path, SPDK_VHOST_SCSI_DISABLED_FEATURES)) {
SPDK_ERRLOG("Couldn't set vhost features for controller %s\n", name);
return -EINVAL;
}
vdev = rte_zmalloc(NULL, sizeof(*vdev), RTE_CACHE_LINE_SIZE);
if (rte_vhost_driver_callback_register(path, &spdk_vhost_scsi_device_ops) != 0) {
SPDK_ERRLOG("Couldn't register callbacks for controller %s\n", name);
return -ENOENT;
}
vdev = spdk_zmalloc(sizeof(*vdev), RTE_CACHE_LINE_SIZE, NULL);
if (vdev == NULL) {
SPDK_ERRLOG("Couldn't allocate memory for vhost dev\n");
return -ENOMEM;
}
spdk_vhost_ctrlrs[ctrlr_num] = vdev;
vdev->name = strdup(name);
vdev->cpumask = cpumask;
vdev->lcore = -1;
if (rte_vhost_driver_start(path) != 0) {
SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s", name, errno,
strerror(errno));
free(vdev->name);
spdk_free(vdev);
return -EIO;
}
spdk_vhost_ctrlrs[ctrlr_num] = vdev;
SPDK_NOTICELOG("Controller %s: new controller added\n", name);
return 0;
}
@ -1086,34 +1200,37 @@ spdk_vhost_scsi_allocate_reactor(uint64_t cpumask)
static int
new_device(int vid)
{
struct virtio_net *dev = vhost_devices[vid];
struct spdk_vhost_scsi_ctrlr *vdev = NULL;
struct spdk_event *event;
sem_t added;
uint32_t i;
vdev = spdk_vhost_scsi_ctrlr_find(dev->ifname);
char ifname[PATH_MAX];
sem_t added;
assert(vid < MAX_VHOST_DEVICE);
if (rte_vhost_get_ifname(vid, ifname, PATH_MAX) < 0) {
SPDK_ERRLOG("Couldn't get a valid ifname for device %d\n", vid);
return -1;
}
vdev = spdk_vhost_scsi_ctrlr_find(ifname);
if (vdev == NULL) {
SPDK_ERRLOG("Controller %s not found.\n", dev->ifname);
SPDK_ERRLOG("Controller %s not found.\n", ifname);
return -1;
}
if (vdev->lcore != -1) {
SPDK_ERRLOG("Controller %s already connected.\n", dev->ifname);
SPDK_ERRLOG("Controller %s already connected.\n", ifname);
return -1;
}
assert(vdev->dev == NULL);
vdev->dev = spdk_vhost_dev_create(vid);
if (vdev->dev == NULL) {
return -1;
}
dpdk_vid_mapping[vid] = vdev;
vdev->dev = dev;
/* Disable notifications. */
for (i = 0; i < dev->num_queues; i++) {
rte_vhost_enable_guest_notification(vid, i, 0);
}
dev->flags |= VIRTIO_DEV_RUNNING;
vdev->dev = dev;
vdev->lcore = spdk_vhost_scsi_allocate_reactor(vdev->cpumask);
event = vhost_sem_event_alloc(vdev->lcore, add_vdev_cb, vdev, &added);
@ -1123,27 +1240,10 @@ new_device(int vid)
return 0;
}
/*
* These callback allow devices to be added to the data core when configuration
* has been fully complete.
*/
static const struct virtio_net_device_ops virtio_net_device_ops = {
.new_device = new_device,
.destroy_device = destroy_device,
};
static void *
session_start(void *arg)
{
rte_vhost_driver_session_start();
return NULL;
}
void
spdk_vhost_startup(void *arg1, void *arg2)
{
int ret;
pthread_t tid;
const char *basename = arg1;
if (basename && strlen(basename) > 0) {
@ -1161,12 +1261,6 @@ spdk_vhost_startup(void *arg1, void *arg2)
ret = spdk_vhost_scsi_controller_construct();
if (ret != 0)
rte_exit(EXIT_FAILURE, "Cannot construct vhost controllers\n");
rte_vhost_driver_callback_register(&virtio_net_device_ops);
if (pthread_create(&tid, NULL, &session_start, NULL) < 0)
rte_panic("Failed to start session poller thread (%d): %s", errno, strerror(errno));
pthread_detach(tid);
}
static void *