thread: Use TLS to accelerate thread look up

Change-Id: I8136db265c9cb8d61de4845ce6eaff2351b5b597
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/435939
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

examples/bdev/fio_plugin/fio_plugin.c

@@ -43,6 +43,8 @@
 #include "spdk/queue.h"
 #include "spdk/util.h"
 
+#include "spdk_internal/thread.h"
+
 #include "config-host.h"
 #include "fio.h"
 #include "optgroup.h"
@@ -142,6 +144,8 @@ spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread)
 
 	while (spdk_fio_poll_thread(fio_thread) > 0) {}
 
+	spdk_set_thread(fio_thread->thread);
+
 	spdk_free_thread();
 	free(fio_thread->iocq);
 	free(fio_thread);

include/spdk_internal/thread.h

@@ -0,0 +1,47 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_THREAD_INTERNAL_H_
+#define SPDK_THREAD_INTERNAL_H_
+
+#include "spdk/stdinc.h"
+
+struct spdk_thread;
+
+/**
+ * Force the current system thread to act as if executing the given SPDK thread.
+ * This is only used for unit testing.
+ */
+void spdk_set_thread(struct spdk_thread *thread);
+
+#endif /* SPDK_THREAD_INTERNAL_H_ */

lib/event/reactor.c

@@ -36,6 +36,7 @@
 
 #include "spdk_internal/event.h"
 #include "spdk_internal/log.h"
+#include "spdk_internal/thread.h"
 
 #include "spdk/log.h"
 #include "spdk/thread.h"
@@ -134,7 +135,7 @@ spdk_event_call(struct spdk_event *event)
 }
 
 static inline uint32_t
-_spdk_event_queue_run_batch(struct spdk_reactor *reactor)
+_spdk_event_queue_run_batch(struct spdk_reactor *reactor, struct spdk_thread *thread)
 {
 	unsigned count, i;
 	void *events[SPDK_EVENT_BATCH_SIZE];
@@ -153,6 +154,8 @@ _spdk_event_queue_run_batch(struct spdk_reactor *reactor)
 		return 0;
 	}
 
+	spdk_set_thread(thread);
+
 	for (i = 0; i < count; i++) {
 		struct spdk_event *event = events[i];
 
@@ -160,6 +163,8 @@ _spdk_event_queue_run_batch(struct spdk_reactor *reactor)
 		event->fn(event->arg1, event->arg2);
 	}
 
+	spdk_set_thread(NULL);
+
 	spdk_mempool_put_bulk(g_spdk_event_mempool, events, count);
 
 	return count;
@@ -294,7 +299,9 @@ _spdk_reactor_run(void *arg)
 
 	sleep_cycles = reactor->max_delay_us * spdk_get_ticks_hz() / SPDK_SEC_TO_USEC;
 	if (g_context_switch_monitor_enabled) {
+		spdk_set_thread(thread);
 		_spdk_reactor_context_switch_monitor_start(reactor, NULL);
+		spdk_set_thread(NULL);
 	}
 	now = spdk_get_ticks();
 	reactor->tsc_last = now;
@@ -302,7 +309,7 @@ _spdk_reactor_run(void *arg)
 	while (1) {
 		bool took_action = false;
 
-		event_count = _spdk_event_queue_run_batch(reactor);
+		event_count = _spdk_event_queue_run_batch(reactor, thread);
 		if (event_count > 0) {
 			rc = 1;
 			now = spdk_get_ticks();
@@ -346,6 +353,7 @@ _spdk_reactor_run(void *arg)
 		}
 	}
 
+	spdk_set_thread(thread);
 	_spdk_reactor_context_switch_monitor_stop(reactor, NULL);
 	spdk_free_thread();
 	return 0;

lib/rocksdb/env_spdk.cc

@@ -45,6 +45,8 @@ extern "C" {
 #include "spdk/log.h"
 #include "spdk/thread.h"
 #include "spdk/bdev.h"
+
+#include "spdk_internal/thread.h"
 }
 
 namespace rocksdb
@@ -579,8 +581,11 @@ public:
 
 void SpdkInitializeThread(void)
 {
+	struct spdk_thread *thread;
+
 	if (g_fs != NULL) {
-		spdk_allocate_thread(NULL, NULL, NULL, NULL, "spdk_rocksdb");
+		thread = spdk_allocate_thread(NULL, NULL, NULL, NULL, "spdk_rocksdb");
+		spdk_set_thread(thread);
 		g_sync_args.channel = spdk_fs_alloc_io_channel_sync(g_fs);
 	}
 }

lib/thread/thread.c

@@ -40,6 +40,7 @@
 #include "spdk/util.h"
 
 #include "spdk_internal/log.h"
+#include "spdk_internal/thread.h"
 
 #ifdef __linux__
 #include <sys/prctl.h>
@@ -130,22 +131,12 @@ struct spdk_thread {
 static TAILQ_HEAD(, spdk_thread) g_threads = TAILQ_HEAD_INITIALIZER(g_threads);
 static uint32_t g_thread_count = 0;
 
-static struct spdk_thread *
+static __thread struct spdk_thread *tls_thread = NULL;
+
+static inline struct spdk_thread *
 _get_thread(void)
 {
-	pthread_t thread_id;
+	return tls_thread;
-	struct spdk_thread *thread;
-
-	thread_id = pthread_self();
-
-	thread = NULL;
-	TAILQ_FOREACH(thread, &g_threads, tailq) {
-		if (thread->thread_id == thread_id) {
-			return thread;
-		}
-	}
-
-	return NULL;
 }
 
 static void
@@ -257,6 +248,12 @@ spdk_allocate_thread(spdk_thread_pass_msg msg_fn,
 	return thread;
 }
 
+void
+spdk_set_thread(struct spdk_thread *thread)
+{
+	tls_thread = thread;
+}
+
 void
 spdk_free_thread(void)
 {
@@ -290,6 +287,8 @@ spdk_free_thread(void)
 
 	free(thread);
 
+	tls_thread = NULL;
+
 	pthread_mutex_unlock(&g_devlist_mutex);
 }
 
@@ -360,6 +359,10 @@ spdk_thread_poll(struct spdk_thread *thread, uint32_t max_msgs)
 	struct spdk_poller *poller;
 	int rc = 0;
 
+	assert(_get_thread() == NULL);
+
+	tls_thread = thread;
+
 	msg_count = _spdk_msg_queue_run_batch(thread, max_msgs);
 	if (msg_count) {
 		rc = 1;
@@ -420,6 +423,8 @@ spdk_thread_poll(struct spdk_thread *thread, uint32_t max_msgs)
 		}
 	}
 
+	tls_thread = NULL;
+
 	return rc;
 }
 

test/common/lib/ut_multithread.c

@@ -34,6 +34,7 @@
 #include "spdk_cunit.h"
 #include "spdk/thread.h"
 #include "spdk_internal/mock.h"
+#include "spdk_internal/thread.h"
 
 #include "common/lib/test_env.c"
 
@@ -67,9 +68,12 @@ set_thread(uintptr_t thread_id)
 	g_thread_id = thread_id;
 	if (thread_id == INVALID_THREAD) {
 		MOCK_CLEAR(pthread_self);
+		spdk_set_thread(NULL);
 	} else {
 		MOCK_SET(pthread_self, (pthread_t)thread_id);
+		spdk_set_thread(g_ut_threads[thread_id].thread);
 	}
+
 }
 
 int
@@ -120,7 +124,7 @@ poll_thread(uintptr_t thread_id)
 	CU_ASSERT(thread_id < g_ut_num_threads);
 
 	original_thread_id = g_thread_id;
-	set_thread(thread_id);
+	set_thread(INVALID_THREAD);
 
 	while (spdk_thread_poll(thread->thread, 0) > 0) {
 		busy = true;

test/unit/lib/bdev/bdev.c/bdev_ut.c

@@ -40,6 +40,8 @@
 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */
 #undef SPDK_CONFIG_VTUNE
 
+#include "spdk_internal/thread.h"
+
 #include "bdev/bdev.c"
 
 DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *, (struct spdk_conf *cp,
@@ -1402,6 +1404,7 @@ bdev_io_alignment(void)
 int
 main(int argc, char **argv)
 {
+	struct spdk_thread	*thread;
 	CU_pSuite		suite = NULL;
 	unsigned int		num_failures;
 
@@ -1432,7 +1435,8 @@ main(int argc, char **argv)
 		return CU_get_error();
 	}
 
-	spdk_allocate_thread(_bdev_send_msg, NULL, NULL, NULL, "thread0");
+	thread = spdk_allocate_thread(_bdev_send_msg, NULL, NULL, NULL, "thread0");
+	spdk_set_thread(thread);
 	CU_basic_set_mode(CU_BRM_VERBOSE);
 	CU_basic_run_tests();
 	num_failures = CU_get_number_of_failures();

test/unit/lib/bdev/part.c/part_ut.c

@@ -40,6 +40,8 @@
 /* HACK: disable VTune integration so the unit test doesn't need VTune headers and libs to build */
 #undef SPDK_CONFIG_VTUNE
 
+#include "spdk_internal/thread.h"
+
 #include "bdev/bdev.c"
 #include "bdev/part.c"
 
@@ -149,6 +151,7 @@ part_test(void)
 int
 main(int argc, char **argv)
 {
+	struct spdk_thread	*thread;
 	CU_pSuite		suite = NULL;
 	unsigned int		num_failures;
 
@@ -169,7 +172,8 @@ main(int argc, char **argv)
 		return CU_get_error();
 	}
 
-	spdk_allocate_thread(_part_send_msg, NULL, NULL, NULL, "thread0");
+	thread = spdk_allocate_thread(_part_send_msg, NULL, NULL, NULL, "thread0");
+	spdk_set_thread(thread);
 	CU_basic_set_mode(CU_BRM_VERBOSE);
 	CU_basic_run_tests();
 	num_failures = CU_get_number_of_failures();

test/unit/lib/bdev/pmem/bdev_pmem_ut.c

@@ -36,6 +36,8 @@
 #include "common/lib/test_env.c"
 #include "unit/lib/json_mock.c"
 
+#include "spdk_internal/thread.h"
+
 #include "bdev/pmem/bdev_pmem.c"
 
 DEFINE_STUB(spdk_conf_find_section, struct spdk_conf_section *,
@@ -377,9 +379,12 @@ ut_pmem_blk_clean(void)
 static int
 ut_pmem_blk_init(void)
 {
+	struct spdk_thread *thread;
+
 	errno = 0;
 
-	spdk_allocate_thread(_pmem_send_msg, NULL, NULL, NULL, NULL);
+	thread = spdk_allocate_thread(_pmem_send_msg, NULL, NULL, NULL, NULL);
+	spdk_set_thread(thread);
 
 	g_pool_ok.buffer = calloc(g_pool_ok.nblock, g_pool_ok.bsize);
 	if (g_pool_ok.buffer == NULL) {

test/unit/lib/blob/blob.c/blob_ut.c

@@ -36,6 +36,7 @@
 #include "spdk_cunit.h"
 #include "spdk/blob.h"
 #include "spdk/string.h"
+#include "spdk_internal/thread.h"
 
 #include "common/lib/test_env.c"
 #include "../bs_dev_common.c"
@@ -5859,6 +5860,7 @@ blob_io_unit_compatiblity(void)
 
 int main(int argc, char **argv)
 {
+	struct spdk_thread *thread;
 	CU_pSuite	suite = NULL;
 	unsigned int	num_failures;
 
@@ -5931,7 +5933,8 @@ int main(int argc, char **argv)
 	}
 
 	g_dev_buffer = calloc(1, DEV_BUFFER_SIZE);
-	spdk_allocate_thread(_bs_send_msg, NULL, NULL, NULL, "thread0");
+	thread = spdk_allocate_thread(_bs_send_msg, NULL, NULL, NULL, "thread0");
+	spdk_set_thread(thread);
 	CU_basic_set_mode(CU_BRM_VERBOSE);
 	CU_basic_run_tests();
 	num_failures = CU_get_number_of_failures();

test/unit/lib/blobfs/blobfs_async_ut/blobfs_async_ut.c

@@ -41,6 +41,8 @@
 #include "blobfs/blobfs.c"
 #include "blobfs/tree.c"
 
+#include "spdk_internal/thread.h"
+
 #include "unit/lib/blob/bs_dev_common.c"
 
 struct spdk_filesystem *g_fs;
@@ -464,6 +466,7 @@ channel_ops_sync(void)
 
 int main(int argc, char **argv)
 {
+	struct spdk_thread *thread;
 	CU_pSuite	suite = NULL;
 	unsigned int	num_failures;
 
@@ -491,7 +494,8 @@ int main(int argc, char **argv)
 		return CU_get_error();
 	}
 
-	spdk_allocate_thread(_fs_send_msg, NULL, NULL, NULL, "thread0");
+	thread = spdk_allocate_thread(_fs_send_msg, NULL, NULL, NULL, "thread0");
+	spdk_set_thread(thread);
 
 	g_dev_buffer = calloc(1, DEV_BUFFER_SIZE);
 	CU_basic_set_mode(CU_BRM_VERBOSE);

test/unit/lib/blobfs/blobfs_sync_ut/blobfs_sync_ut.c

@@ -38,6 +38,7 @@
 #include "spdk/log.h"
 #include "spdk/thread.h"
 #include "spdk/barrier.h"
+#include "spdk_internal/thread.h"
 
 #include "spdk_cunit.h"
 #include "unit/lib/blob/bs_dev_common.c"
@@ -339,9 +340,11 @@ terminate_spdk_thread(void *arg)
 static void *
 spdk_thread(void *arg)
 {
+	struct spdk_thread *thread;
 	struct ut_request *req;
 
-	spdk_allocate_thread(_fs_send_msg, NULL, NULL, NULL, "thread1");
+	thread = spdk_allocate_thread(_fs_send_msg, NULL, NULL, NULL, "thread1");
+	spdk_set_thread(thread);
 
 	while (1) {
 		pthread_mutex_lock(&g_mutex);
@@ -364,6 +367,7 @@ spdk_thread(void *arg)
 
 int main(int argc, char **argv)
 {
+	struct spdk_thread *thread;
 	CU_pSuite	suite = NULL;
 	pthread_t	spdk_tid;
 	unsigned int	num_failures;
@@ -389,7 +393,8 @@ int main(int argc, char **argv)
 		return CU_get_error();
 	}
 
-	spdk_allocate_thread(_fs_send_msg, NULL, NULL, NULL, "thread0");
+	thread = spdk_allocate_thread(_fs_send_msg, NULL, NULL, NULL, "thread0");
+	spdk_set_thread(thread);
 
 	pthread_create(&spdk_tid, NULL, spdk_thread, NULL);
 	g_dev_buffer = calloc(1, DEV_BUFFER_SIZE);

test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c

@@ -35,6 +35,7 @@
 
 #include "spdk_cunit.h"
 #include "spdk_internal/mock.h"
+#include "spdk_internal/thread.h"
 
 #include "common/lib/test_env.c"
 #include "nvmf/ctrlr.c"
@@ -765,6 +766,7 @@ test_identify_ns(void)
 
 int main(int argc, char **argv)
 {
+	struct spdk_thread *thread;
 	CU_pSuite	suite = NULL;
 	unsigned int	num_failures;
 
@@ -789,7 +791,8 @@ int main(int argc, char **argv)
 		return CU_get_error();
 	}
 
-	spdk_allocate_thread(ctrlr_ut_pass_msg, NULL, NULL, NULL, "ctrlr_ut");
+	thread = spdk_allocate_thread(ctrlr_ut_pass_msg, NULL, NULL, NULL, "ctrlr_ut");
+	spdk_set_thread(thread);
 
 	CU_basic_set_mode(CU_BRM_VERBOSE);
 	CU_basic_run_tests();

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

@@ -36,6 +36,7 @@
 #include "common/lib/test_env.c"
 #include "spdk_cunit.h"
 #include "spdk_internal/mock.h"
+#include "spdk_internal/thread.h"
 
 #include "nvmf/subsystem.c"
 
@@ -448,6 +449,7 @@ test_spdk_nvmf_subsystem_set_sn(void)
 
 int main(int argc, char **argv)
 {
+	struct spdk_thread *thread;
 	CU_pSuite	suite = NULL;
 	unsigned int	num_failures;
 
@@ -469,11 +471,14 @@ int main(int argc, char **argv)
 		return CU_get_error();
 	}
 
-	spdk_allocate_thread(_subsystem_send_msg, NULL, NULL, NULL, "thread0");
+	thread = spdk_allocate_thread(_subsystem_send_msg, NULL, NULL, NULL, "thread0");
+	spdk_set_thread(thread);
+
 	CU_basic_set_mode(CU_BRM_VERBOSE);
 	CU_basic_run_tests();
 	num_failures = CU_get_number_of_failures();
 	CU_cleanup_registry();
+
 	spdk_free_thread();
 
 	return num_failures;

test/unit/lib/nvmf/tcp.c/tcp_ut.c

@@ -35,6 +35,7 @@
 
 #include "spdk_cunit.h"
 #include "spdk_internal/mock.h"
+#include "spdk_internal/thread.h"
 
 #include "common/lib/test_env.c"
 #include "nvmf/ctrlr.c"
@@ -205,6 +206,7 @@ test_nvmf_tcp_create(void)
 
 	thread = spdk_allocate_thread(NULL, NULL, NULL, NULL, NULL);
 	SPDK_CU_ASSERT_FATAL(thread != NULL);
+	spdk_set_thread(thread);
 
 	/* case 1 */
 	memset(&opts, 0, sizeof(opts));
@@ -273,6 +275,7 @@ test_nvmf_tcp_destroy(void)
 
 	thread = spdk_allocate_thread(NULL, NULL, NULL, NULL, NULL);
 	SPDK_CU_ASSERT_FATAL(thread != NULL);
+	spdk_set_thread(thread);
 
 	/* case 1 */
 	memset(&opts, 0, sizeof(opts));

test/unit/lib/thread/thread.c/thread_ut.c

@@ -35,6 +35,8 @@
 
 #include "spdk_cunit.h"
 
+#include "spdk_internal/thread.h"
+
 #include "thread/thread.c"
 #include "common/lib/ut_multithread.c"
 
@@ -343,7 +345,8 @@ thread_name(void)
 	const char *name;
 
 	/* Create thread with no name, which automatically generates one */
-	spdk_allocate_thread(NULL, NULL, NULL, NULL, NULL);
+	thread = spdk_allocate_thread(NULL, NULL, NULL, NULL, NULL);
+	spdk_set_thread(thread);
 	thread = spdk_get_thread();
 	SPDK_CU_ASSERT_FATAL(thread != NULL);
 	name = spdk_thread_get_name(thread);
@@ -351,7 +354,8 @@ thread_name(void)
 	spdk_free_thread();
 
 	/* Create thread named "test_thread" */
-	spdk_allocate_thread(NULL, NULL, NULL, NULL, "test_thread");
+	thread = spdk_allocate_thread(NULL, NULL, NULL, NULL, "test_thread");
+	spdk_set_thread(thread);
 	thread = spdk_get_thread();
 	SPDK_CU_ASSERT_FATAL(thread != NULL);
 	name = spdk_thread_get_name(thread);