event: remove spdk_app_init

In this patch, spdk_app_init will be merged into
spdk_app_start.
Due to this change, we need to change event_perf
to use spdk_app_start, since we cannot use spdk_app_init
anymore. So the related changes for event_perf is to
make it work with reactor framework.

Change-Id: Id67edf209fd628cca361a499068c93aeedfe6167
Signed-off-by: Ziye Yang <optimistyzy@gmail.com>
Reviewed-on: https://review.gerrithub.io/364153
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

app/iscsi_tgt/iscsi_tgt.c

@@ -177,12 +177,11 @@ main(int argc, char **argv)
 
 	opts.shutdown_cb = spdk_iscsi_shutdown;
 	opts.usr1_handler = spdk_sigusr1;
-	spdk_app_init(&opts);
 
 	printf("Total cores available: %u\n", spdk_env_get_core_count());
 	printf("Using net framework %s\n", spdk_net_framework_get_name());
 	/* Blocks until the application is exiting */
-	app_rc = spdk_app_start(spdk_startup, NULL, NULL);
+	app_rc = spdk_app_start(&opts, spdk_startup, NULL, NULL);
 
 	rc = spdk_app_fini();
 

app/nvmf_tgt/nvmf_tgt.c

@@ -307,11 +307,10 @@ spdk_nvmf_tgt_start(struct spdk_app_opts *opts)
 	int rc;
 
 	opts->shutdown_cb = spdk_nvmf_shutdown_cb;
-	spdk_app_init(opts);
 
 	printf("Total cores available: %d\n", spdk_env_get_core_count());
 	/* Blocks until the application is exiting */
-	rc = spdk_app_start(spdk_nvmf_startup, NULL, NULL);
+	rc = spdk_app_start(opts, spdk_nvmf_startup, NULL, NULL);
 
 	spdk_app_fini();
 

app/vhost/vhost.c

@@ -150,10 +150,9 @@ main(int argc, char *argv[])
 	}
 
 	opts.shutdown_cb = spdk_vhost_shutdown_cb;
-	spdk_app_init(&opts);
 
 	/* Blocks until the application is exiting */
-	rc = spdk_app_start(spdk_vhost_startup, (void *)socket_path, NULL);
+	rc = spdk_app_start(&opts, spdk_vhost_startup, (void *)socket_path, NULL);
 
 	spdk_app_fini();
 

include/spdk/event.h

@@ -97,22 +97,17 @@ struct spdk_app_opts {
 void spdk_app_opts_init(struct spdk_app_opts *opts);
 
 /**
- * \brief Initialize an application to use the event framework. This must be called prior to using
+ * \brief Start the framework. Once started, the framework will call start_fn on the master
- * any other functions in this library.
+ * core with the arguments provided. This call will block until \ref spdk_app_stop is called.
  */
-void spdk_app_init(struct spdk_app_opts *opts);
+int spdk_app_start(struct spdk_app_opts *opts, spdk_event_fn start_fn,
+		   void *arg1, void *arg2);
 
 /**
  * \brief Perform final shutdown operations on an application using the event framework.
  */
 int spdk_app_fini(void);
 
-/**
- * \brief Start the framework. Once started, the framework will call start_fn on the master
- * core with the arguments provided. This call will block until \ref spdk_app_stop is called.
- */
-int spdk_app_start(spdk_event_fn start_fn, void *arg1, void *arg2);
-
 /**
  * \brief Start shutting down the framework.  Typically this function is not called directly, and
  * the shutdown process is started implicitly by a process signal.  But in applications that are

include/spdk_internal/event.h

@@ -51,8 +51,6 @@ int spdk_reactors_fini(void);
 void spdk_reactors_start(void);
 void spdk_reactors_stop(void);
 
-uint32_t spdk_event_queue_run_batch(uint32_t lcore);
-
 struct spdk_subsystem {
 	const char *name;
 	/* User must call spdk_subsystem_init_next() when they are done with their initialization. */

lib/event/app.c

@@ -216,8 +216,9 @@ spdk_app_opts_init(struct spdk_app_opts *opts)
 	opts->max_delay_us = 0;
 }
 
-void
+int
-spdk_app_init(struct spdk_app_opts *opts)
+spdk_app_start(struct spdk_app_opts *opts, spdk_event_fn start_fn,
+	       void *arg1, void *arg2)
 {
 	struct spdk_conf		*config;
 	struct spdk_conf_section	*sp;
@@ -228,6 +229,12 @@ spdk_app_init(struct spdk_app_opts *opts)
 	uint64_t		tpoint_group_mask;
 	char			*end;
 	struct spdk_env_opts env_opts = {};
+	struct spdk_event *app_start_event;
+
+	if (!opts) {
+		SPDK_ERRLOG("opts should not be NULL\n");
+		exit(EXIT_FAILURE);
+	}
 
 	if (opts->enable_coredump) {
 		struct rlimit core_limits;
@@ -409,6 +416,18 @@ spdk_app_init(struct spdk_app_opts *opts)
 			spdk_trace_set_tpoint_group_mask(tpoint_group_mask);
 		}
 	}
+
+	g_spdk_app.rc = 0;
+	app_start_event = spdk_event_allocate(spdk_env_get_current_core(), start_fn,
+					      arg1, arg2);
+
+	spdk_event_call(spdk_event_allocate(spdk_env_get_current_core(), spdk_subsystem_init,
+					    app_start_event, NULL));
+
+	/* This blocks until spdk_app_stop is called */
+	spdk_reactors_start();
+
+	return g_spdk_app.rc;
 }
 
 int
@@ -425,25 +444,6 @@ spdk_app_fini(void)
 	return rc;
 }
 
-int
-spdk_app_start(spdk_event_fn start_fn, void *arg1, void *arg2)
-{
-	struct spdk_event *app_start_event;
-
-	g_spdk_app.rc = 0;
-
-	app_start_event = spdk_event_allocate(spdk_env_get_current_core(), start_fn,
-					      arg1, arg2);
-
-	spdk_event_call(spdk_event_allocate(spdk_env_get_current_core(), spdk_subsystem_init,
-					    app_start_event, NULL));
-
-	/* This blocks until spdk_app_stop is called */
-	spdk_reactors_start();
-
-	return g_spdk_app.rc;
-}
-
 void
 spdk_app_stop(int rc)
 {

lib/event/reactor.c

@@ -200,12 +200,6 @@ _spdk_event_queue_run_batch(struct spdk_reactor *reactor)
 	return count;
 }
 
-uint32_t
-spdk_event_queue_run_batch(uint32_t lcore)
-{
-	return _spdk_event_queue_run_batch(spdk_reactor_get(lcore));
-}
-
 /**
  *
  * \brief Set current reactor thread name to "reactor <cpu #>".

lib/rocksdb/env_spdk.cc

@@ -510,9 +510,7 @@ initialize_spdk(void *arg)
 {
 	struct spdk_app_opts *opts = (struct spdk_app_opts *)arg;
 
-	spdk_app_init(opts);
+	spdk_app_start(opts, spdk_rocksdb_run, NULL, NULL);
-
-	spdk_app_start(spdk_rocksdb_run, NULL, NULL);
 	spdk_app_fini();
 
 	delete opts;

test/lib/bdev/bdevio/bdevio.c

@@ -764,15 +764,16 @@ main(int argc, char **argv)
 {
 	const char		*config_file;
 	int			num_failures;
+	struct spdk_app_opts	opts = {};
 
 	if (argc == 1) {
 		config_file = "/usr/local/etc/spdk/iscsi.conf";
 	} else {
 		config_file = argv[1];
 	}
-	bdevtest_init(config_file, "0x3");
+	bdevtest_init(config_file, "0x3", &opts);
 
-	num_failures = spdk_app_start(test_main, NULL, NULL);
+	num_failures = spdk_app_start(&opts, test_main, NULL, NULL);
 	spdk_app_fini();
 
 	return num_failures;

test/lib/bdev/bdevperf/bdevperf.c

@@ -532,6 +532,11 @@ bdevperf_run(void *arg1, void *arg2)
 	struct io_target *target;
 	struct spdk_event *event;
 
+	task_pool = rte_mempool_create("task_pool", 4096 * spdk_env_get_core_count(),
+				       sizeof(struct bdevperf_task),
+				       64, 0, NULL, NULL, task_ctor, NULL,
+				       SOCKET_ID_ANY, 0);
+
 	bdevperf_construct_targets();
 
 	printf("Running I/O for %d seconds...\n", g_time_in_sec);
@@ -562,6 +567,7 @@ main(int argc, char **argv)
 	const char *workload_type;
 	int op;
 	bool mix_specified;
+	struct spdk_app_opts opts = {};
 
 	/* default value */
 	config_file = NULL;
@@ -716,14 +722,9 @@ main(int argc, char **argv)
 
 	blockdev_heads_init();
 
-	bdevtest_init(config_file, core_mask);
+	bdevtest_init(config_file, core_mask, &opts);
 
-	task_pool = rte_mempool_create("task_pool", 4096 * spdk_env_get_core_count(),
+	spdk_app_start(&opts, bdevperf_run, NULL, NULL);
-				       sizeof(struct bdevperf_task),
-				       64, 0, NULL, NULL, task_ctor, NULL,
-				       SOCKET_ID_ANY, 0);
-
-	spdk_app_start(bdevperf_run, NULL, NULL);
 
 	performance_dump(g_time_in_sec);
 	spdk_app_fini();

test/lib/bdev/common.c

@@ -37,13 +37,13 @@
 #include "spdk/event.h"
 
 static void
-bdevtest_init(const char *config_file, const char *cpumask)
+bdevtest_init(const char *config_file, const char *cpumask,
+	      struct spdk_app_opts *opts)
 {
-	struct spdk_app_opts opts;
+	assert(opts != NULL);
 
-	spdk_app_opts_init(&opts);
+	spdk_app_opts_init(opts);
-	opts.name = "bdevtest";
+	opts->name = "bdevtest";
-	opts.config_file = config_file;
+	opts->config_file = config_file;
-	opts.reactor_mask = cpumask;
+	opts->reactor_mask = cpumask;
-	spdk_app_init(&opts);
 }

test/lib/blobfs/fuse/fuse.c

@@ -335,14 +335,13 @@ int main(int argc, char **argv)
 	opts.reactor_mask = "0x3";
 	opts.dpdk_mem_size = 6144;
 	opts.shutdown_cb = spdk_fuse_shutdown;
-	spdk_app_init(&opts);
 
 	g_bdev_name = argv[2];
 	g_mountpoint = argv[3];
 	g_fuse_argc = argc - 2;
 	g_fuse_argv = &argv[2];
 
-	spdk_app_start(spdk_fuse_run, NULL, NULL);
+	spdk_app_start(&opts, spdk_fuse_run, NULL, NULL);
 	spdk_app_fini();
 
 	return 0;

test/lib/blobfs/mkfs/mkfs.c

@@ -106,12 +106,11 @@ int main(int argc, char **argv)
 	opts.reactor_mask = "0x3";
 	opts.dpdk_mem_size = 1024;
 	opts.shutdown_cb = NULL;
-	spdk_app_init(&opts);
 
 	spdk_fs_set_cache_size(512);
 
 	g_bdev_name = argv[2];
-	spdk_app_start(spdk_mkfs_run, NULL, NULL);
+	spdk_app_start(&opts, spdk_mkfs_run, NULL, NULL);
 	spdk_app_fini();
 
 	return 0;

test/lib/event/event_perf/event_perf.c

@@ -43,10 +43,10 @@
 
 static uint64_t g_tsc_rate;
 static uint64_t g_tsc_us_rate;
+static uint64_t g_tsc_end;
 
 static int g_time_in_sec;
 
-static __thread uint64_t __call_count = 0;
 static uint64_t call_count[RTE_MAX_LCORE];
 
 static void
@@ -55,39 +55,47 @@ submit_new_event(void *arg1, void *arg2)
 	struct spdk_event *event;
 	static __thread uint32_t next_lcore = RTE_MAX_LCORE;
 
+	if (spdk_get_ticks() > g_tsc_end) {
+		spdk_app_stop(0);
+		return;
+	}
+
 	if (next_lcore == RTE_MAX_LCORE) {
 		next_lcore = rte_get_next_lcore(rte_lcore_id(), 0, 1);
 	}
 
-	++__call_count;
+	call_count[next_lcore]++;
 	event = spdk_event_allocate(next_lcore, submit_new_event, NULL, NULL);
 	spdk_event_call(event);
 }
 
-static int
+static void
-event_work_fn(void *arg)
+event_work_fn(void *arg1, void *arg2)
 {
-	uint64_t tsc_end;
-
-	tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
 
 	submit_new_event(NULL, NULL);
 	submit_new_event(NULL, NULL);
 	submit_new_event(NULL, NULL);
 	submit_new_event(NULL, NULL);
+}
 
-	while (1) {
+static void
+event_perf_start(void *arg1, void *arg2)
+{
+	uint32_t i;
 
-		spdk_event_queue_run_batch(rte_lcore_id());
+	g_tsc_rate = spdk_get_ticks_hz();
+	g_tsc_us_rate = g_tsc_rate / (1000 * 1000);
+	g_tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate;
 
-		if (spdk_get_ticks() > tsc_end) {
+	printf("Running I/O for %d seconds...", g_time_in_sec);
-			break;
+	fflush(stdout);
-		}
+
+	SPDK_ENV_FOREACH_CORE(i) {
+		spdk_event_call(spdk_event_allocate(i, event_work_fn,
+						    NULL, NULL));
 	}
 
-	call_count[rte_lcore_id()] = __call_count;
-
-	return 0;
 }
 
 static void
@@ -115,11 +123,9 @@ performance_dump(int io_time)
 int
 main(int argc, char **argv)
 {
-	struct spdk_app_opts opts;
+	struct spdk_app_opts opts = {};
 	int op;
-	uint32_t i, current_core;
 
-	spdk_app_opts_init(&opts);
 	opts.name = "event_perf";
 
 	g_time_in_sec = 0;
@@ -143,27 +149,12 @@ main(int argc, char **argv)
 		exit(1);
 	}
 
-	spdk_app_init(&opts);
-
-	g_tsc_rate = spdk_get_ticks_hz();
-	g_tsc_us_rate = g_tsc_rate / (1000 * 1000);
-
 	printf("Running I/O for %d seconds...", g_time_in_sec);
 	fflush(stdout);
 
-	/* call event_work_fn on each slave lcore */
+	spdk_app_start(&opts, event_perf_start, NULL, NULL);
-	current_core = spdk_env_get_current_core();
-	SPDK_ENV_FOREACH_CORE(i) {
-		if (i != current_core) {
-			rte_eal_remote_launch(event_work_fn, NULL, i);
-		}
-	}
-
-	/* call event_work_fn on lcore0 */
-	event_work_fn(NULL);
-
-	rte_eal_mp_wait_lcore();
 
+	spdk_app_fini();
 	performance_dump(g_time_in_sec);
 
 	printf("done.\n");

test/lib/event/reactor/reactor.c

@@ -136,9 +136,7 @@ main(int argc, char **argv)
 
 	opts.shutdown_cb = test_cleanup;
 
-	spdk_app_init(&opts);
+	spdk_app_start(&opts, test_start, NULL, NULL);
-
-	spdk_app_start(test_start, NULL, NULL);
 
 	test_cleanup();
 

test/lib/event/reactor_perf/reactor_perf.c

@@ -132,9 +132,7 @@ main(int argc, char **argv)
 
 	opts.shutdown_cb = test_cleanup;
 
-	spdk_app_init(&opts);
+	spdk_app_start(&opts, test_start, NULL, NULL);
-
-	spdk_app_start(test_start, NULL, NULL);
 
 	spdk_app_fini();