ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fio: Continue polling spdk_threads on the init thread after shutdown

Browse Source 
When fio wants to shut down a thread, it stops polling it immediately.
Some SPDK threads need to continue to be polled for a period of time
to allow the message passing to all complete, so this behavior in fio
could lead to a hang. Instead, when fio wants to shut down a thread,
place that thread onto a global list that's polled by the init thread
which lives for the duration of the program. The init thread will
continue polling the spdk_thread until it has exited.

Change-Id: I07292d5ba389167c45a14aa8ffbe72b49e88d17d
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3682
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: GangCao <gang.cao@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
This commit is contained in:
Ben Walker 2020-08-06 10:25:52 -07:00 committed by Tomasz Zawadzki
parent 4696163e44
commit 991a56f85d
1 changed files with 56 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
examples/bdev/fio_plugin/fio_plugin.c
View File

@ -87,6 +87,8 @@ struct spdk_fio_thread {
struct io_u **iocq; /* io completion queue */
unsigned int iocq_count; /* number of iocq entries filled by last getevents */
unsigned int iocq_size; /* number of iocq entries allocated */
TAILQ_ENTRY(spdk_fio_thread) link;
};
static bool g_spdk_env_initialized = false;
@ -96,6 +98,12 @@ static int spdk_fio_init(struct thread_data *td);
static void spdk_fio_cleanup(struct thread_data *td);
static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread);
static pthread_t g_init_thread_id = 0;
static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t g_init_cond;
static bool g_poll_loop = true;
static TAILQ_HEAD(, spdk_fio_thread) g_threads = TAILQ_HEAD_INITIALIZER(g_threads);
/* Default polling timeout (ns) */
#define SPDK_FIO_POLLING_TIMEOUT 1000000000ULL
@ -149,19 +157,9 @@ spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread)
{
spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_close_targets, fio_thread);
while (!spdk_thread_is_idle(fio_thread->thread)) {
spdk_fio_poll_thread(fio_thread);
}
spdk_set_thread(fio_thread->thread);
spdk_thread_exit(fio_thread->thread);
while (!spdk_thread_is_exited(fio_thread->thread)) {
spdk_thread_poll(fio_thread->thread, 0, 0);
}
spdk_thread_destroy(fio_thread->thread);
free(fio_thread->iocq);
free(fio_thread);
pthread_mutex_lock(&g_init_mtx);
TAILQ_INSERT_TAIL(&g_threads, fio_thread, link);
pthread_mutex_unlock(&g_init_mtx);
}
static void
@ -189,11 +187,6 @@ spdk_fio_calc_timeout(struct spdk_fio_thread *fio_thread, struct timespec *ts)
}
}
static pthread_t g_init_thread_id = 0;
static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t g_init_cond;
static bool g_poll_loop = true;
static void
spdk_fio_bdev_init_done(int rc, void *cb_arg)
{
@ -232,6 +225,7 @@ spdk_init_thread_poll(void *arg)
{
struct spdk_fio_options *eo = arg;
struct spdk_fio_thread *fio_thread;
struct spdk_fio_thread *thread, *tmp;
struct spdk_conf *config = NULL;
struct spdk_env_opts opts;
bool done;
@ -325,19 +319,37 @@ spdk_init_thread_poll(void *arg)
pthread_mutex_lock(&g_init_mtx);
pthread_cond_signal(&g_init_cond);
pthread_mutex_unlock(&g_init_mtx);
while (g_poll_loop) {
spdk_fio_poll_thread(fio_thread);
pthread_mutex_lock(&g_init_mtx);
if (!TAILQ_EMPTY(&g_threads)) {
TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {
spdk_fio_poll_thread(thread);
}
/* If there are exiting threads to poll, don't sleep. */
pthread_mutex_unlock(&g_init_mtx);
continue;
}
/* Figure out how long to sleep. */
clock_gettime(CLOCK_MONOTONIC, &ts);
spdk_fio_calc_timeout(fio_thread, &ts);
rc = pthread_cond_timedwait(&g_init_cond, &g_init_mtx, &ts);
pthread_mutex_unlock(&g_init_mtx);
if (rc != ETIMEDOUT) {
break;
}
}
pthread_mutex_unlock(&g_init_mtx);
spdk_fio_cleanup_thread(fio_thread);
/* Finalize the bdev layer */
done = false;
@ -345,9 +357,32 @@ spdk_init_thread_poll(void *arg)
do {
spdk_fio_poll_thread(fio_thread);
} while (!done && !spdk_thread_is_idle(fio_thread->thread));
spdk_fio_cleanup_thread(fio_thread);
TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {
spdk_fio_poll_thread(thread);
}
} while (!done);
/* Now exit all the threads */
TAILQ_FOREACH(thread, &g_threads, link) {
spdk_set_thread(thread->thread);
spdk_thread_exit(thread->thread);
spdk_set_thread(NULL);
}
/* And wait for them to gracefully exit */
while (!TAILQ_EMPTY(&g_threads)) {
TAILQ_FOREACH_SAFE(thread, &g_threads, link, tmp) {
if (spdk_thread_is_exited(thread->thread)) {
TAILQ_REMOVE(&g_threads, thread, link);
spdk_thread_destroy(thread->thread);
free(thread->iocq);
free(thread);
} else {
spdk_thread_poll(thread->thread, 0, 0);
}
}
}
pthread_exit(NULL);