event: Move scheduler set call and scheduling reactor set call

This fixes issue where scheduler is not being
initialized on first set.

Signed-off-by: Maciej Szwed <maciej.szwed@intel.com>
Change-Id: I881438955f08852fb72187a2b92c7acfb12cd662
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5029
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

lib/event/reactor.c

@@ -110,23 +110,20 @@ _spdk_scheduler_set(char *name)
 		return -ENOENT;
 	}
 
-	if (g_reactors == NULL || g_scheduling_reactor == NULL) {
-		g_new_scheduler = scheduler;
-		g_scheduler = scheduler;
-		return 0;
-	}
-
 	if (g_scheduling_reactor->flags.is_scheduling) {
-		g_new_scheduler = scheduler;
+		if (g_scheduler != g_new_scheduler) {
+			/* Scheduler already changed, cannot defer multiple deinits */
+			return -EBUSY;
+		}
 	} else {
-		if (g_scheduler->deinit != NULL) {
+		if (g_scheduler != NULL && g_scheduler->deinit != NULL) {
 			g_scheduler->deinit(&g_governor);
 		}
-
-		g_new_scheduler = scheduler;
 		g_scheduler = scheduler;
 	}
 
+	g_new_scheduler = scheduler;
+
 	if (scheduler->init != NULL) {
 		scheduler->init(&g_governor);
 	}
@@ -197,16 +194,11 @@ static bool reactor_thread_op_supported(enum spdk_thread_op op);
 int
 spdk_reactors_init(void)
 {
+	struct spdk_reactor *reactor;
 	int rc;
-	uint32_t i, last_core;
+	uint32_t i, last_core, current_core;
 	char mempool_name[32];
 
-	rc = _spdk_scheduler_set("static");
-	if (rc != 0) {
-		SPDK_ERRLOG("Failed setting up scheduler\n");
-		return rc;
-	}
-
 	snprintf(mempool_name, sizeof(mempool_name), "evtpool_%d", getpid());
 	g_spdk_event_mempool = spdk_mempool_create(mempool_name,
 			       262144 - 1, /* Power of 2 minus 1 is optimal for memory consumption */
@@ -247,6 +239,14 @@ spdk_reactors_init(void)
 		reactor_construct(&g_reactors[i], i);
 	}
 
+	current_core = spdk_env_get_current_core();
+	reactor = spdk_reactor_get(current_core);
+	assert(reactor != NULL);
+	g_scheduling_reactor = reactor;
+
+	rc = _spdk_scheduler_set("static");
+	assert(rc == 0);
+
 	g_reactor_state = SPDK_REACTOR_STATE_INITIALIZED;
 
 	return 0;
@@ -827,7 +827,6 @@ spdk_reactors_start(void)
 	/* Start the main reactor */
 	reactor = spdk_reactor_get(current_core);
 	assert(reactor != NULL);
-	g_scheduling_reactor = reactor;
 	reactor_run(reactor);
 
 	spdk_env_thread_wait_all();

test/unit/lib/event/reactor.c/reactor_ut.c

@@ -59,6 +59,8 @@ test_init_reactors(void)
 {
 	uint32_t core;
 
+	MOCK_SET(spdk_env_get_current_core, 0);
+
 	allocate_cores(3);
 
 	CU_ASSERT(spdk_reactors_init() == 0);
@@ -71,6 +73,8 @@ test_init_reactors(void)
 	spdk_reactors_fini();
 
 	free_cores();
+
+	MOCK_CLEAR(spdk_env_get_current_core);
 }
 
 static void
@@ -90,6 +94,8 @@ test_event_call(void)
 	struct spdk_event *evt;
 	struct spdk_reactor *reactor;
 
+	MOCK_SET(spdk_env_get_current_core, 0);
+
 	allocate_cores(1);
 
 	CU_ASSERT(spdk_reactors_init() == 0);
@@ -109,6 +115,8 @@ test_event_call(void)
 	spdk_reactors_fini();
 
 	free_cores();
+
+	MOCK_CLEAR(spdk_env_get_current_core);
 }
 
 static void
@@ -119,6 +127,8 @@ test_schedule_thread(void)
 	struct spdk_reactor *reactor;
 	struct spdk_lw_thread *lw_thread;
 
+	MOCK_SET(spdk_env_get_current_core, 0);
+
 	allocate_cores(5);
 
 	CU_ASSERT(spdk_reactors_init() == 0);
@@ -169,6 +179,8 @@ test_reschedule_thread(void)
 	struct spdk_reactor *reactor;
 	struct spdk_lw_thread *lw_thread;
 
+	MOCK_SET(spdk_env_get_current_core, 0);
+
 	allocate_cores(3);
 
 	CU_ASSERT(spdk_reactors_init() == 0);
@@ -269,12 +281,12 @@ test_for_each_reactor(void)
 	bool done = false;
 	struct spdk_reactor *reactor;
 
+	MOCK_SET(spdk_env_get_current_core, 0);
+
 	allocate_cores(5);
 
 	CU_ASSERT(spdk_reactors_init() == 0);
 
-	MOCK_SET(spdk_env_get_current_core, 0);
-
 	spdk_for_each_reactor(for_each_reactor_cb, &count, &done, for_each_reactor_done);
 
 	MOCK_CLEAR(spdk_env_get_current_core);
@@ -350,13 +362,14 @@ test_reactor_stats(void)
 	 * - idle TSC of reactor should be 500 (= 200 + 300).
 	 */
 
+	MOCK_SET(spdk_env_get_current_core, 0);
+
 	allocate_cores(1);
 
 	CU_ASSERT(spdk_reactors_init() == 0);
 
 	spdk_cpuset_set_cpu(&cpuset, 0, true);
 
-	MOCK_SET(spdk_env_get_current_core, 0);
 	MOCK_SET(spdk_get_ticks, 100);
 
 	thread1 = spdk_thread_create(NULL, &cpuset);
@@ -427,6 +440,8 @@ test_reactor_stats(void)
 	spdk_reactors_fini();
 
 	free_cores();
+
+	MOCK_CLEAR(spdk_env_get_current_core);
 }
 
 int