lib/thread: Wait until thread_is_exited() is true by thread_poll() after thread_exit()

Following patches will require thread termination so that thread_exit()
will move thread to exiting, thread_poll() will move thread from
exiting to exited, thread_is_exited() will detect the threadis exited,
and then call thread_destroy().

So change all places as a preparation.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I6b2e8aee5ed7cd160a88b4c9aaed7d90bd9dac07
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1640
Community-CI: Broadcom CI
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

examples/bdev/fio_plugin/fio_plugin.c

@@ -156,6 +156,9 @@ spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread)
 	spdk_set_thread(fio_thread->thread);
 
 	spdk_thread_exit(fio_thread->thread);
+	while (!spdk_thread_is_exited(fio_thread->thread)) {
+		spdk_thread_poll(fio_thread->thread, 0, 0);
+	}
 	spdk_thread_destroy(fio_thread->thread);
 	free(fio_thread->iocq);
 	free(fio_thread);

examples/nvmf/nvmf/nvmf.c

@@ -204,6 +204,9 @@ nvmf_reactor_run(void *arg)
 		TAILQ_REMOVE(&nvmf_reactor->threads, lw_thread, link);
 		spdk_set_thread(thread);
 		spdk_thread_exit(thread);
+		while (!spdk_thread_is_exited(thread)) {
+			spdk_thread_poll(thread, 0, 0);
+		}
 		spdk_thread_destroy(thread);
 	}
 	pthread_mutex_unlock(&nvmf_reactor->mutex);

lib/event/reactor.c

@@ -394,6 +394,9 @@ _spdk_reactor_run(void *arg)
 		spdk_set_thread(thread);
 		rc = spdk_thread_exit(thread);
 		assert(rc == 0);
+		while (!spdk_thread_is_exited(thread)) {
+			spdk_thread_poll(thread, 0, 0);
+		}
 		spdk_thread_destroy(thread);
 	}
 

test/common/lib/ut_multithread.c

@@ -103,13 +103,18 @@ void
 free_threads(void)
 {
 	uint32_t i;
+	struct spdk_thread *thread;
 	int rc __attribute__((unused));
 
 	for (i = 0; i < g_ut_num_threads; i++) {
 		set_thread(i);
-		rc = spdk_thread_exit(g_ut_threads[i].thread);
+		thread = g_ut_threads[i].thread;
+		rc = spdk_thread_exit(thread);
 		assert(rc == 0);
-		spdk_thread_destroy(g_ut_threads[i].thread);
+		while (!spdk_thread_is_exited(thread)) {
+			spdk_thread_poll(thread, 0, 0);
+		}
+		spdk_thread_destroy(thread);
 		g_ut_threads[i].thread = NULL;
 	}
 

test/unit/lib/bdev/bdev_ocssd.c/bdev_ocssd_ut.c

@@ -1181,6 +1181,9 @@ main(int argc, const char **argv)
 	num_failures = CU_get_number_of_failures();
 
 	spdk_thread_exit(g_thread);
+	while (!spdk_thread_is_exited(g_thread)) {
+		spdk_thread_poll(g_thread, 0, 0);
+	}
 	spdk_thread_destroy(g_thread);
 
 	CU_cleanup_registry();

test/unit/lib/bdev/compress.c/compress_ut.c

@@ -620,6 +620,9 @@ test_cleanup(void)
 
 	thread = spdk_get_thread();
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 
 	spdk_thread_lib_fini();

test/unit/lib/bdev/vbdev_zone_block.c/vbdev_zone_block_ut.c

@@ -1491,6 +1491,9 @@ int main(int argc, char **argv)
 	num_failures = CU_get_number_of_failures();
 
 	spdk_thread_exit(g_thread);
+	while (!spdk_thread_is_exited(g_thread)) {
+		spdk_thread_poll(g_thread, 0, 0);
+	}
 	spdk_thread_destroy(g_thread);
 
 	CU_cleanup_registry();

test/unit/lib/blobfs/blobfs_sync_ut/blobfs_sync_ut.c

@@ -669,10 +669,16 @@ int main(int argc, char **argv)
 
 	spdk_set_thread(thread);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 
 	spdk_set_thread(g_dispatch_thread);
 	spdk_thread_exit(g_dispatch_thread);
+	while (!spdk_thread_is_exited(g_dispatch_thread)) {
+		spdk_thread_poll(g_dispatch_thread, 0, 0);
+	}
 	spdk_thread_destroy(g_dispatch_thread);
 
 	spdk_thread_lib_fini();

test/unit/lib/event/reactor.c/reactor_ut.c

@@ -148,6 +148,9 @@ test_schedule_thread(void)
 	reactor->thread_count--;
 	spdk_set_thread(thread);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 	spdk_set_thread(NULL);
 
@@ -228,6 +231,9 @@ test_reschedule_thread(void)
 	reactor->thread_count--;
 	spdk_set_thread(thread);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 	spdk_set_thread(NULL);
 

test/unit/lib/ftl/common/utils.c

@@ -134,11 +134,19 @@ test_init_ftl_band(struct spdk_ftl_dev *dev, size_t id, size_t zone_size)
 void
 test_free_ftl_dev(struct spdk_ftl_dev *dev)
 {
+	struct spdk_thread *thread;
+
 	SPDK_CU_ASSERT_FATAL(dev != NULL);
 	free(dev->ioch);
-	spdk_set_thread(dev->core_thread);
+
-	spdk_thread_exit(dev->core_thread);
+	thread = dev->core_thread;
-	spdk_thread_destroy(dev->core_thread);
+
+	spdk_set_thread(thread);
+	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
+	spdk_thread_destroy(thread);
 	spdk_mempool_free(dev->lba_pool);
 	free(dev->bands);
 	free(dev);

test/unit/lib/nvmf/tcp.c/tcp_ut.c

@@ -396,6 +396,9 @@ test_nvmf_tcp_create(void)
 	CU_ASSERT_PTR_NULL(transport);
 
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 }
 
@@ -426,6 +429,9 @@ test_nvmf_tcp_destroy(void)
 	CU_ASSERT(spdk_nvmf_tcp_destroy(transport) == 0);
 
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 }
 
@@ -462,6 +468,9 @@ test_nvmf_tcp_poll_group_create(void)
 	spdk_nvmf_tcp_destroy(transport);
 
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 }
 
@@ -520,6 +529,9 @@ test_nvmf_tcp_send_c2h_data(void)
 	CU_ASSERT(pdu.data_iov[2].iov_len == 99);
 
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 }
 

test/unit/lib/thread/thread.c/thread_ut.c

@@ -81,6 +81,9 @@ thread_alloc(void)
 	SPDK_CU_ASSERT_FATAL(thread != NULL);
 	spdk_set_thread(thread);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 	spdk_thread_lib_fini();
 
@@ -93,6 +96,9 @@ thread_alloc(void)
 	SPDK_CU_ASSERT_FATAL(thread != NULL);
 	spdk_set_thread(thread);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 
 	/* Scheduling fails */
@@ -111,6 +117,9 @@ thread_alloc(void)
 	SPDK_CU_ASSERT_FATAL(thread != NULL);
 	spdk_set_thread(thread);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 
 	/* Scheduling fails */
@@ -611,6 +620,9 @@ thread_name(void)
 	name = spdk_thread_get_name(thread);
 	CU_ASSERT(name != NULL);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 
 	/* Create thread named "test_thread" */
@@ -622,6 +634,9 @@ thread_name(void)
 	SPDK_CU_ASSERT_FATAL(name != NULL);
 	CU_ASSERT(strcmp(name, "test_thread") == 0);
 	spdk_thread_exit(thread);
+	while (!spdk_thread_is_exited(thread)) {
+		spdk_thread_poll(thread, 0, 0);
+	}
 	spdk_thread_destroy(thread);
 
 	spdk_thread_lib_fini();