From ea863bb0b33148972774791431ac49191381fd0c Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Sun, 8 Mar 2020 20:57:32 -0400 Subject: [PATCH] lib/event: Count number of threads per reactor Add thread_count to struct spdk_reactor to count number of threads per reactor. This number will be used in the next patch to know if all threads are idle or not for each reactor to support CPU power saving. Signed-off-by: Shuhei Matsumoto Change-Id: I4f7cc5a6b78d85e9f8d0b539c60058c13e282759 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1169 Tested-by: SPDK CI Jenkins Reviewed-by: Aleksey Marchuk Reviewed-by: Jim Harris Reviewed-by: Paul Luse --- include/spdk_internal/event.h | 1 + lib/event/reactor.c | 9 +++++++++ test/unit/lib/event/reactor.c/reactor_ut.c | 2 ++ 3 files changed, 12 insertions(+) diff --git a/include/spdk_internal/event.h b/include/spdk_internal/event.h index 97d625b76..282147d2a 100644 --- a/include/spdk_internal/event.h +++ b/include/spdk_internal/event.h @@ -68,6 +68,7 @@ struct spdk_lw_thread { struct spdk_reactor { /* Lightweight threads running on this reactor */ TAILQ_HEAD(, spdk_lw_thread) threads; + uint32_t thread_count; /* Logical core number for this reactor. */ uint32_t lcore; diff --git a/lib/event/reactor.c b/lib/event/reactor.c index 77dd4fc8b..e0814c8b9 100644 --- a/lib/event/reactor.c +++ b/lib/event/reactor.c @@ -68,6 +68,7 @@ spdk_reactor_construct(struct spdk_reactor *reactor, uint32_t lcore) reactor->flags.is_valid = true; TAILQ_INIT(&reactor->threads); + reactor->thread_count = 0; reactor->events = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 65536, SPDK_ENV_SOCKET_ID_ANY); assert(reactor->events != NULL); @@ -153,6 +154,7 @@ spdk_reactors_fini(void) SPDK_ENV_FOREACH_CORE(i) { reactor = spdk_reactor_get(i); + assert(reactor->thread_count == 0); if (spdk_likely(reactor != NULL) && reactor->events != NULL) { spdk_ring_free(reactor->events); } @@ -328,6 +330,8 @@ reactor_run(struct spdk_reactor *reactor) if (spdk_unlikely(lw_thread->resched)) { lw_thread->resched = false; TAILQ_REMOVE(&reactor->threads, lw_thread, link); + assert(reactor->thread_count > 0); + reactor->thread_count--; _reactor_schedule_thread(thread); continue; } @@ -335,6 +339,8 @@ reactor_run(struct spdk_reactor *reactor) if (spdk_unlikely(spdk_thread_is_exited(thread) && spdk_thread_is_idle(thread))) { TAILQ_REMOVE(&reactor->threads, lw_thread, link); + assert(reactor->thread_count > 0); + reactor->thread_count--; spdk_thread_destroy(thread); continue; } @@ -376,6 +382,8 @@ _spdk_reactor_run(void *arg) TAILQ_FOREACH_SAFE(lw_thread, &reactor->threads, link, tmp) { thread = spdk_thread_get_from_ctx(lw_thread); TAILQ_REMOVE(&reactor->threads, lw_thread, link); + assert(reactor->thread_count > 0); + reactor->thread_count--; spdk_set_thread(thread); if (!spdk_thread_is_exited(thread)) { rc = spdk_thread_exit(thread); @@ -489,6 +497,7 @@ _schedule_thread(void *arg1, void *arg2) assert(reactor != NULL); TAILQ_INSERT_TAIL(&reactor->threads, lw_thread, link); + reactor->thread_count++; } static int diff --git a/test/unit/lib/event/reactor.c/reactor_ut.c b/test/unit/lib/event/reactor.c/reactor_ut.c index ec198f950..495ac686f 100644 --- a/test/unit/lib/event/reactor.c/reactor_ut.c +++ b/test/unit/lib/event/reactor.c/reactor_ut.c @@ -145,6 +145,7 @@ test_schedule_thread(void) CU_ASSERT(spdk_thread_get_from_ctx(lw_thread) == thread); TAILQ_REMOVE(&reactor->threads, lw_thread, link); + reactor->thread_count--; spdk_set_thread(thread); spdk_thread_exit(thread); spdk_thread_destroy(thread); @@ -224,6 +225,7 @@ test_reschedule_thread(void) MOCK_CLEAR(spdk_env_get_current_core); TAILQ_REMOVE(&reactor->threads, lw_thread, link); + reactor->thread_count--; spdk_set_thread(thread); spdk_thread_exit(thread); spdk_thread_destroy(thread);