ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vhost: change vdev global array to a linked list

Browse Source 
Drop max vhost initiators limitation in SPDK (64).
We're still limited by rte_vhost limits, but
they're set to 1024 at the moment.

Change-Id: Ia1ad25665d6e798bc22709cdd43b72d60f1f4cf0
Signed-off-by: Slawomir Mrozowicz <slawomirx.mrozowicz@intel.com>
Signed-off-by: Dariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/389811
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Dariusz Stojaczyk 2018-02-12 21:02:15 +01:00 committed by Jim Harris
parent 85b0c2b36f
commit 19725e9764
3 changed files with 46 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
lib/vhost/vhost.c
View File

@ -47,8 +47,6 @@ static uint32_t *g_num_ctrlrs;
/* Path to folder where character device will be created. Can be set by user. */ /* Path to folder where character device will be created. Can be set by user. */
static char dev_dirname[PATH_MAX] = ""; static char dev_dirname[PATH_MAX] = "";
#define MAX_VHOST_DEVICES 64
struct spdk_vhost_dev_event_ctx { struct spdk_vhost_dev_event_ctx {
/** Pointer to the controller obtained before enqueuing the event */ /** Pointer to the controller obtained before enqueuing the event */
struct spdk_vhost_dev *vdev; struct spdk_vhost_dev *vdev;
@ -83,7 +81,8 @@ const struct vhost_device_ops g_spdk_vhost_ops = {
.destroy_connection = destroy_connection, .destroy_connection = destroy_connection,
}; };
static struct spdk_vhost_dev *g_spdk_vhost_devices[MAX_VHOST_DEVICES]; static TAILQ_HEAD(, spdk_vhost_dev) g_spdk_vhost_devices = TAILQ_HEAD_INITIALIZER(
g_spdk_vhost_devices);
static pthread_mutex_t g_spdk_vhost_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t g_spdk_vhost_mutex = PTHREAD_MUTEX_INITIALIZER;
void *spdk_vhost_gpa_to_vva(struct spdk_vhost_dev *vdev, uint64_t addr) void *spdk_vhost_gpa_to_vva(struct spdk_vhost_dev *vdev, uint64_t addr)
@ -473,12 +472,10 @@ spdk_vhost_dev_has_feature(struct spdk_vhost_dev *vdev, unsigned feature_id)
static struct spdk_vhost_dev * static struct spdk_vhost_dev *
spdk_vhost_dev_find_by_id(unsigned id) spdk_vhost_dev_find_by_id(unsigned id)
{ {
unsigned i;
struct spdk_vhost_dev *vdev; struct spdk_vhost_dev *vdev;
for (i = 0; i < MAX_VHOST_DEVICES; i++) { TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) {
vdev = g_spdk_vhost_devices[i]; if (vdev->id == id) {
if (vdev && vdev->id == id) {
return vdev; return vdev;
} }
} }
@ -489,12 +486,10 @@ spdk_vhost_dev_find_by_id(unsigned id)
static struct spdk_vhost_dev * static struct spdk_vhost_dev *
spdk_vhost_dev_find_by_vid(int vid) spdk_vhost_dev_find_by_vid(int vid)
{ {
unsigned i;
struct spdk_vhost_dev *vdev; struct spdk_vhost_dev *vdev;
for (i = 0; i < MAX_VHOST_DEVICES; i++) { TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) {
vdev = g_spdk_vhost_devices[i]; if (vdev->vid == vid) {
if (vdev && vdev->vid == vid) {
return vdev; return vdev;
} }
} }
@ -562,20 +557,16 @@ spdk_vhost_free_reactor(uint32_t lcore)
struct spdk_vhost_dev * struct spdk_vhost_dev *
spdk_vhost_dev_find(const char *ctrlr_name) spdk_vhost_dev_find(const char *ctrlr_name)
{ {
unsigned i; struct spdk_vhost_dev *vdev;
size_t dev_dirname_len = strlen(dev_dirname); size_t dev_dirname_len = strlen(dev_dirname);
if (strncmp(ctrlr_name, dev_dirname, dev_dirname_len) == 0) { if (strncmp(ctrlr_name, dev_dirname, dev_dirname_len) == 0) {
ctrlr_name += dev_dirname_len; ctrlr_name += dev_dirname_len;
} }
for (i = 0; i < MAX_VHOST_DEVICES; i++) { TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) {
if (g_spdk_vhost_devices[i] == NULL) { if (strcmp(vdev->name, ctrlr_name) == 0) {
continue; return vdev;
}
if (strcmp(g_spdk_vhost_devices[i]->name, ctrlr_name) == 0) {
return g_spdk_vhost_devices[i];
} }
} }
@ -615,7 +606,7 @@ int
spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str, spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const char *mask_str,
const struct spdk_vhost_dev_backend *backend) const struct spdk_vhost_dev_backend *backend)
{ {
unsigned ctrlr_num; static unsigned ctrlr_num;
char path[PATH_MAX]; char path[PATH_MAX];
struct stat file_stat; struct stat file_stat;
struct spdk_cpuset *cpumask; struct spdk_cpuset *cpumask;
@ -623,6 +614,16 @@ spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const cha
assert(vdev); assert(vdev);
/* We expect devices inside g_spdk_vhost_devices to be sorted in ascending
* order in regard of vdev->id. For now we always set vdev->id = ctrlr_num++
* and append each vdev to the very end of g_spdk_vhost_devices list.
* This is required for foreach vhost events to work.
*/
if (ctrlr_num == UINT_MAX) {
assert(false);
return -EINVAL;
}
if (name == NULL) { if (name == NULL) {
SPDK_ERRLOG("Can't register controller with no name\n"); SPDK_ERRLOG("Can't register controller with no name\n");
return -EINVAL; return -EINVAL;
@ -647,18 +648,6 @@ spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const cha
goto out; goto out;
} }
for (ctrlr_num = 0; ctrlr_num < MAX_VHOST_DEVICES; ctrlr_num++) {
if (g_spdk_vhost_devices[ctrlr_num] == NULL) {
break;
}
}
if (ctrlr_num == MAX_VHOST_DEVICES) {
SPDK_ERRLOG("Max controllers reached (%d).\n", MAX_VHOST_DEVICES);
rc = -ENOSPC;
goto out;
}
if (snprintf(path, sizeof(path), "%s%s", dev_dirname, name) >= (int)sizeof(path)) { if (snprintf(path, sizeof(path), "%s%s", dev_dirname, name) >= (int)sizeof(path)) {
SPDK_ERRLOG("Resulting socket path for controller %s is too long: %s%s\n", name, dev_dirname, SPDK_ERRLOG("Resulting socket path for controller %s is too long: %s%s\n", name, dev_dirname,
name); name);
@ -707,7 +696,7 @@ spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const cha
vdev->name = strdup(name); vdev->name = strdup(name);
vdev->path = strdup(path); vdev->path = strdup(path);
vdev->id = ctrlr_num; vdev->id = ctrlr_num++;
vdev->vid = -1; vdev->vid = -1;
vdev->lcore = -1; vdev->lcore = -1;
vdev->cpumask = cpumask; vdev->cpumask = cpumask;
@ -720,7 +709,7 @@ spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const cha
vdev->stats_check_interval = SPDK_VHOST_DEV_STATS_CHECK_INTERVAL_MS * spdk_get_ticks_hz() / vdev->stats_check_interval = SPDK_VHOST_DEV_STATS_CHECK_INTERVAL_MS * spdk_get_ticks_hz() /
1000UL; 1000UL;
g_spdk_vhost_devices[ctrlr_num] = vdev; TAILQ_INSERT_TAIL(&g_spdk_vhost_devices, vdev, tailq);
if (rte_vhost_driver_start(path) != 0) { if (rte_vhost_driver_start(path) != 0) {
SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n", name, errno, SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n", name, errno,
@ -757,16 +746,18 @@ spdk_vhost_dev_unregister(struct spdk_vhost_dev *vdev)
free(vdev->name); free(vdev->name);
free(vdev->path); free(vdev->path);
spdk_cpuset_free(vdev->cpumask); spdk_cpuset_free(vdev->cpumask);
g_spdk_vhost_devices[vdev->id] = NULL; TAILQ_REMOVE(&g_spdk_vhost_devices, vdev, tailq);
return 0; return 0;
} }
static struct spdk_vhost_dev * static struct spdk_vhost_dev *
spdk_vhost_dev_next(int i) spdk_vhost_dev_next(unsigned i)
{ {
for (i++; i < MAX_VHOST_DEVICES; i++) { struct spdk_vhost_dev *vdev;
if (g_spdk_vhost_devices[i]) {
return g_spdk_vhost_devices[i]; TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) {
if (vdev->id > i) {
return vdev;
} }
} }
@ -854,7 +845,7 @@ spdk_vhost_event_async_fn(void *arg1, void *arg2)
free(ctx); free(ctx);
} }
static void spdk_vhost_external_event_foreach_continue(int vdev_id, static void spdk_vhost_external_event_foreach_continue(struct spdk_vhost_dev *vdev,
spdk_vhost_event_fn fn, void *arg); spdk_vhost_event_fn fn, void *arg);
static void static void
@ -874,9 +865,14 @@ spdk_vhost_event_async_foreach_fn(void *arg1, void *arg2)
vdev = spdk_vhost_dev_find_by_id(ctx->vdev_id); vdev = spdk_vhost_dev_find_by_id(ctx->vdev_id);
if (vdev == ctx->vdev) { if (vdev == ctx->vdev) {
ctx->cb_fn(vdev, arg2); ctx->cb_fn(vdev, arg2);
} else {
/* ctx->vdev is probably a dangling pointer at this point.
* It must have been removed in the meantime, so we just skip
* it in our foreach chain. */
} }
spdk_vhost_external_event_foreach_continue(ctx->vdev_id, ctx->cb_fn, arg2); vdev = spdk_vhost_dev_next(ctx->vdev_id);
spdk_vhost_external_event_foreach_continue(vdev, ctx->cb_fn, arg2);
pthread_mutex_unlock(&g_spdk_vhost_mutex); pthread_mutex_unlock(&g_spdk_vhost_mutex);
free(ctx); free(ctx);
@ -1155,14 +1151,8 @@ static void *
session_shutdown(void *arg) session_shutdown(void *arg)
{ {
struct spdk_vhost_dev *vdev = NULL; struct spdk_vhost_dev *vdev = NULL;
int i;
for (i = 0; i < MAX_VHOST_DEVICES; i++) {
vdev = g_spdk_vhost_devices[i];
if (vdev == NULL) {
continue;
}
TAILQ_FOREACH(vdev, &g_spdk_vhost_devices, tailq) {
rte_vhost_driver_unregister(vdev->path); rte_vhost_driver_unregister(vdev->path);
vdev->registered = false; vdev->registered = false;
} }
@ -1260,11 +1250,9 @@ spdk_vhost_call_external_event(const char *ctrlr_name, spdk_vhost_event_fn fn, v
} }
static void static void
spdk_vhost_external_event_foreach_continue(int vdev_id, spdk_vhost_event_fn fn, void *arg) spdk_vhost_external_event_foreach_continue(struct spdk_vhost_dev *vdev,
spdk_vhost_event_fn fn, void *arg)
{ {
struct spdk_vhost_dev *vdev;
vdev = spdk_vhost_dev_next(vdev_id);
if (vdev == NULL) { if (vdev == NULL) {
fn(NULL, arg); fn(NULL, arg);
return; return;
@ -1285,8 +1273,11 @@ spdk_vhost_external_event_foreach_continue(int vdev_id, spdk_vhost_event_fn fn,
void void
spdk_vhost_call_external_event_foreach(spdk_vhost_event_fn fn, void *arg) spdk_vhost_call_external_event_foreach(spdk_vhost_event_fn fn, void *arg)
{ {
struct spdk_vhost_dev *vdev;
pthread_mutex_lock(&g_spdk_vhost_mutex); pthread_mutex_lock(&g_spdk_vhost_mutex);
spdk_vhost_external_event_foreach_continue(-1, fn, arg); vdev = TAILQ_FIRST(&g_spdk_vhost_devices);
spdk_vhost_external_event_foreach_continue(vdev, fn, arg);
pthread_mutex_unlock(&g_spdk_vhost_mutex); pthread_mutex_unlock(&g_spdk_vhost_mutex);
} }

2
lib/vhost/vhost_internal.h
View File

@ -165,6 +165,8 @@ struct spdk_vhost_dev {
uint64_t negotiated_features; uint64_t negotiated_features;
struct spdk_vhost_virtqueue virtqueue[SPDK_VHOST_MAX_VQUEUES]; struct spdk_vhost_virtqueue virtqueue[SPDK_VHOST_MAX_VQUEUES];
TAILQ_ENTRY(spdk_vhost_dev) tailq;
}; };
struct spdk_vhost_dev *spdk_vhost_dev_find(const char *ctrlr_name); struct spdk_vhost_dev *spdk_vhost_dev_find(const char *ctrlr_name);

13
test/unit/lib/vhost/vhost.c/vhost_ut.c
View File

@ -228,7 +228,6 @@ create_controller_test(void)
{ {
struct spdk_vhost_dev *vdev, *vdev2; struct spdk_vhost_dev *vdev, *vdev2;
int ret; int ret;
unsigned ctrlr_num;
char long_name[PATH_MAX]; char long_name[PATH_MAX];
struct spdk_vhost_dev_backend backend; struct spdk_vhost_dev_backend backend;
@ -257,22 +256,10 @@ create_controller_test(void)
vdev2 = alloc_vdev(); vdev2 = alloc_vdev();
ret = spdk_vhost_dev_register(vdev2, "vdev_name_0", "0x1", &backend); ret = spdk_vhost_dev_register(vdev2, "vdev_name_0", "0x1", &backend);
CU_ASSERT(ret != 0); CU_ASSERT(ret != 0);
/* Create device when max number of devices is reached */
for (ctrlr_num = 0; ctrlr_num < MAX_VHOST_DEVICES; ctrlr_num ++) {
g_spdk_vhost_devices[ctrlr_num] = vdev;
}
ret = spdk_vhost_dev_register(vdev2, "vdev_name_1", "0x1", &backend);
CU_ASSERT(ret != 0);
free_vdev(vdev2); free_vdev(vdev2);
spdk_vhost_dev_unregister(vdev); spdk_vhost_dev_unregister(vdev);
free_vdev(vdev); free_vdev(vdev);
for (ctrlr_num = 0; ctrlr_num < MAX_VHOST_DEVICES; ctrlr_num++) {
g_spdk_vhost_devices[ctrlr_num] = NULL;
}
} }
static void static void