From 89c1e5bfc057e355b12f2e5bf00fe32b1d0483ac Mon Sep 17 00:00:00 2001 From: Apokleos Date: Fri, 22 Jul 2022 03:07:36 +0000 Subject: [PATCH] SPDK Interrupt Mode: Improve processing of reactor interrupt mode. spdk in interrupt, reactor dosen't correctly handle exited threads, causing vhost threads still in reactor's lw_threads list. The fix will do cleanup thread when it's state becomes EXITED. Though it's exposed in v22.05.x, but the master branch also has the problem. We will do this as below: (1) When thread's state becomes SPDK_THREAD_STATE_EXITED, reactor process thread exits first. (2) Then reactor do remove lw_thread and destroy it. Fix issue: #2574 Signed-off-by: Apokleos Change-Id: I3ac2681d70480563db3a0aee4aff61c2f272b140 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13706 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Changpeng Liu Reviewed-by: Xiaodong Liu Reviewed-by: Ben Walker --- lib/event/reactor.c | 14 +++++++------- lib/thread/thread.c | 8 ++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/event/reactor.c b/lib/event/reactor.c index 3565431b7..79bba543d 100644 --- a/lib/event/reactor.c +++ b/lib/event/reactor.c @@ -846,13 +846,6 @@ reactor_post_process_lw_thread(struct spdk_reactor *reactor, struct spdk_lw_thre { struct spdk_thread *thread = spdk_thread_get_from_ctx(lw_thread); - if (spdk_unlikely(lw_thread->resched)) { - lw_thread->resched = false; - _reactor_remove_lw_thread(reactor, lw_thread); - _reactor_schedule_thread(thread); - return true; - } - if (spdk_unlikely(spdk_thread_is_exited(thread) && spdk_thread_is_idle(thread))) { _reactor_remove_lw_thread(reactor, lw_thread); @@ -860,6 +853,13 @@ reactor_post_process_lw_thread(struct spdk_reactor *reactor, struct spdk_lw_thre return true; } + if (spdk_unlikely(lw_thread->resched)) { + lw_thread->resched = false; + _reactor_remove_lw_thread(reactor, lw_thread); + _reactor_schedule_thread(thread); + return true; + } + return false; } diff --git a/lib/thread/thread.c b/lib/thread/thread.c index 9e4897763..5ccd28d96 100644 --- a/lib/thread/thread.c +++ b/lib/thread/thread.c @@ -543,6 +543,9 @@ thread_exit(struct spdk_thread *thread, uint64_t now) exited: thread->state = SPDK_THREAD_STATE_EXITED; + if (spdk_unlikely(thread->in_interrupt)) { + g_thread_op_fn(thread, SPDK_THREAD_OP_RESCHED); + } } int @@ -1813,6 +1816,11 @@ _on_thread(void *ctx) pthread_mutex_lock(&g_devlist_mutex); ct->cur_thread = TAILQ_NEXT(ct->cur_thread, tailq); + while (ct->cur_thread && ct->cur_thread->state != SPDK_THREAD_STATE_RUNNING) { + SPDK_DEBUGLOG(thread, "thread %s is not running but still not destroyed.\n", + ct->cur_thread->name); + ct->cur_thread = TAILQ_NEXT(ct->cur_thread, tailq); + } pthread_mutex_unlock(&g_devlist_mutex); if (!ct->cur_thread) {