From dad4c43a8824d43fc921c80e304376ab32dd877e Mon Sep 17 00:00:00 2001 From: Darek Stojaczyk Date: Mon, 24 Jun 2019 09:59:37 +0200 Subject: [PATCH] vhost: add a single dpdk semaphore The semaphore was a part of struct spdk_vhost_session_fn_ctx so far, but since there's only one pthread waiting on that semaphore and hence only one event using it, we could just use a single global sem_t. Same thing with response code for those callbacks - there's only one needed. Going a step further, the function complete_session_event() was removed - it would only operate on global variables now, and its signature wouldn't make much sense after this refactor, so it's been inlined. This serves as cleanup. Change-Id: I63ef41d7e1564fff5e785de101d887bc1014aad9 Signed-off-by: Darek Stojaczyk Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/459160 Reviewed-by: Shuhei Matsumoto Reviewed-by: Ben Walker Tested-by: SPDK CI Jenkins --- lib/vhost/vhost.c | 55 ++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index d965a3748..2348aaca2 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -62,6 +62,16 @@ static struct spdk_thread *g_vhost_init_thread; static spdk_vhost_fini_cb g_fini_cpl_cb; +/** + * DPDK calls our callbacks synchronously but the work those callbacks + * perform needs to be async. Luckily, all DPDK callbacks are called on + * a DPDK-internal pthread, so we'll just wait on a semaphore in there. + */ +static sem_t g_dpdk_sem; + +/** Return code for the current DPDK callback */ +static int g_dpdk_response; + struct spdk_vhost_session_fn_ctx { /** Device pointer obtained before enqueuing the event */ struct spdk_vhost_dev *vdev; @@ -72,12 +82,6 @@ struct spdk_vhost_session_fn_ctx { /** User callback function to be executed on given thread. */ spdk_vhost_session_fn cb_fn; - /** Semaphore used to signal that event is done. */ - sem_t sem; - - /** Response to be written by enqueued event. */ - int response; - /** Custom user context */ void *user_ctx; }; @@ -891,15 +895,6 @@ spdk_vhost_put_poll_group(struct vhost_poll_group *pg) pg->ref--; } -static void -complete_session_event(struct spdk_vhost_session *vsession, int response) -{ - struct spdk_vhost_session_fn_ctx *ctx = vsession->event_ctx; - - ctx->response = response; - sem_post(&ctx->sem); -} - void spdk_vhost_session_start_done(struct spdk_vhost_session *vsession, int response) { @@ -908,7 +903,9 @@ spdk_vhost_session_start_done(struct spdk_vhost_session *vsession, int response) assert(vsession->vdev->active_session_num < UINT32_MAX); vsession->vdev->active_session_num++; } - complete_session_event(vsession, response); + + g_dpdk_response = response; + sem_post(&g_dpdk_sem); } void @@ -919,7 +916,9 @@ spdk_vhost_session_stop_done(struct spdk_vhost_session *vsession, int response) assert(vsession->vdev->active_session_num > 0); vsession->vdev->active_session_num--; } - complete_session_event(vsession, response); + + g_dpdk_response = response; + sem_post(&g_dpdk_sem); } static void @@ -1000,12 +999,6 @@ spdk_vhost_session_send_event(struct vhost_poll_group *pg, struct timespec timeout; int rc; - rc = sem_init(&ev_ctx.sem, 0, 0); - if (rc != 0) { - SPDK_ERRLOG("Failed to initialize semaphore for vhost timed event\n"); - return -errno; - } - ev_ctx.vdev = vsession->vdev; ev_ctx.vsession_id = vsession->id; ev_ctx.cb_fn = cb_fn; @@ -1018,16 +1011,15 @@ spdk_vhost_session_send_event(struct vhost_poll_group *pg, clock_gettime(CLOCK_REALTIME, &timeout); timeout.tv_sec += timeout_sec; - rc = sem_timedwait(&ev_ctx.sem, &timeout); + rc = sem_timedwait(&g_dpdk_sem, &timeout); if (rc != 0) { SPDK_ERRLOG("Timeout waiting for event: %s.\n", errmsg); - sem_wait(&ev_ctx.sem); + sem_wait(&g_dpdk_sem); } - sem_destroy(&ev_ctx.sem); pthread_mutex_lock(&g_spdk_vhost_mutex); vsession->event_ctx = NULL; - return ev_ctx.response; + return g_dpdk_response; } static int @@ -1514,6 +1506,14 @@ spdk_vhost_init(spdk_vhost_init_cb init_cb) goto err_out; } + ret = sem_init(&g_dpdk_sem, 0, 0); + if (ret != 0) { + SPDK_ERRLOG("Failed to initialize semaphore for rte_vhost pthread.\n"); + spdk_cpuset_free(g_tmp_cpuset); + ret = -1; + goto err_out; + } + spdk_for_each_thread(vhost_create_poll_group, init_cb, vhost_create_poll_group_done); @@ -1539,6 +1539,7 @@ _spdk_vhost_fini(void *arg1) spdk_vhost_unlock(); /* All devices are removed now. */ + sem_destroy(&g_dpdk_sem); spdk_cpuset_free(g_tmp_cpuset); TAILQ_FOREACH_SAFE(pg, &g_poll_groups, tailq, tpg) { TAILQ_REMOVE(&g_poll_groups, pg, tailq);