thread: Allow for an extra region of memory allocated on each thread

Schedulers can use this region to store required information.

Change-Id: I93efb44f1a534596f6285bbe014579311fe011e7
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/c/444454
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

examples/bdev/fio_plugin/fio_plugin.c

@@ -281,7 +281,7 @@ spdk_init_thread_poll(void *arg)
 	}
 	spdk_unaffinitize_thread();
 
-	spdk_thread_lib_init(NULL);
+	spdk_thread_lib_init(NULL, 0);
 
 	/* Create an SPDK thread temporarily */
 	rc = spdk_fio_init_thread(&td);

include/spdk/thread.h

@@ -177,10 +177,13 @@ struct spdk_io_channel {
  *
  * \param new_thread_fn Called each time a new SPDK thread is created. The implementor
  * is expected to frequently call spdk_thread_poll() on the provided thread.
+ * \param ctx_sz For each thread allocated, an additional region of memory of
+ * size ctx_size will also be allocated, for use by the thread scheduler. A pointer
+ * to this region may be obtained by calling spdk_thread_get_ctx().
  *
  * \return 0 on success. Negated errno on failure.
  */
-int spdk_thread_lib_init(spdk_new_thread_fn new_thread_fn);
+int spdk_thread_lib_init(spdk_new_thread_fn new_thread_fn, size_t ctx_sz);
 
 /**
  * Release all resources associated with this library.
@@ -209,6 +212,16 @@ struct spdk_thread *spdk_thread_create(const char *name);
  */
 void spdk_thread_exit(struct spdk_thread *thread);
 
+/**
+ * Return a pointer to this thread's context.
+ *
+ * \param thread The thread on which to get the context.
+ *
+ * \return a pointer to the per-thread context, or NULL if there is
+ * no per-thread context.
+ */
+void *spdk_thread_get_ctx(struct spdk_thread *thread);
+
 /**
  * Perform one iteration worth of processing on the thread. This includes
  * both expired and continuous pollers as well as messages.

lib/event/reactor.c

@@ -343,7 +343,7 @@ spdk_reactors_init(void)
 
 	memset(g_reactors, 0, (last_core + 1) * sizeof(struct spdk_reactor));
 
-	spdk_thread_lib_init(NULL);
+	spdk_thread_lib_init(NULL, 0);
 
 	SPDK_ENV_FOREACH_CORE(i) {
 		reactor = spdk_reactor_get(i);

lib/thread/thread.c

@@ -55,6 +55,7 @@
 static pthread_mutex_t g_devlist_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 static spdk_new_thread_fn g_new_thread_fn = NULL;
+static size_t g_ctx_sz = 0;
 
 struct io_device {
 	void				*io_device;
@@ -132,6 +133,9 @@ struct spdk_thread {
 
 	SLIST_HEAD(, spdk_msg)		msg_cache;
 	size_t				msg_cache_count;
+
+	/* User context allocated at the end */
+	uint8_t				ctx[0];
 };
 
 static TAILQ_HEAD(, spdk_thread) g_threads = TAILQ_HEAD_INITIALIZER(g_threads);
@@ -158,13 +162,15 @@ _set_thread_name(const char *thread_name)
 }
 
 int
-spdk_thread_lib_init(spdk_new_thread_fn new_thread_fn)
+spdk_thread_lib_init(spdk_new_thread_fn new_thread_fn, size_t ctx_sz)
 {
 	char mempool_name[SPDK_MAX_MEMZONE_NAME_LEN];
 
 	assert(g_new_thread_fn == NULL);
 	g_new_thread_fn = new_thread_fn;
 
+	g_ctx_sz = ctx_sz;
+
 	snprintf(mempool_name, sizeof(mempool_name), "msgpool_%d", getpid());
 	g_spdk_msg_mempool = spdk_mempool_create(mempool_name,
 			     262144 - 1, /* Power of 2 minus 1 is optimal for memory consumption */
@@ -200,7 +206,7 @@ spdk_thread_create(const char *name)
 	struct spdk_msg *msgs[SPDK_MSG_MEMPOOL_CACHE_SIZE];
 	int rc, i;
 
-	thread = calloc(1, sizeof(*thread));
+	thread = calloc(1, sizeof(*thread) + g_ctx_sz);
 	if (!thread) {
 		SPDK_ERRLOG("Unable to allocate memory for thread\n");
 		return NULL;
@@ -326,6 +332,16 @@ spdk_thread_exit(struct spdk_thread *thread)
 	free(thread);
 }
 
+void *
+spdk_thread_get_ctx(struct spdk_thread *thread)
+{
+	if (g_ctx_sz > 0) {
+		return thread->ctx;
+	}
+
+	return NULL;
+}
+
 static inline uint32_t
 _spdk_msg_queue_run_batch(struct spdk_thread *thread, uint32_t max_msgs)
 {