rpc: Add start_subsystems_init RPC to start subsystems when -w option is used

RPC state of the RPC server, g_rpc_state is set by spdk_rpc_set_state(state).

If the command line option '-w' is specified, g_rpc_state is initialized to
SPDK_RPC_STARTUP. Otherwise, g_rpc_state is initialized to SPDK_RPC_RUNTIME.

When g_rpc_state is initialized to SPDK_RPC_STARTUP, SPDK start RPC server
and wait for start_subsystems_init RPC.

When SPDK receive start_subsystems_init RPC, SPDK start initialization of
subsystems. The RPC waits for completion of initialization of subsystems.

When SPDK completes initialization of subsystems, SPDK change
g_rpc_state to SPDK_RPC_RUNTIME and then start application.

Upcoming new RPCs to initialize options of NVMf-tgt and iSCSI-tgt will be
able to be allowed before start_subsystems_init RPC is called.

Change-Id: I4fa9c22b64e2fdbc9b9fdb2c47f0018f73f84f7e
Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/406919
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

include/spdk/event.h

@@ -100,6 +100,11 @@ struct spdk_app_opts {
 	 * specified in microseconds.
 	 */
 	uint64_t		max_delay_us;
+
+	/* Wait for the associated RPC before initializing subsystems
+	 * when this flag is enabled.
+	 */
+	bool			delay_subsystem_init;
 };
 
 /**
@@ -189,7 +194,7 @@ int spdk_app_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask);
  */
 struct spdk_cpuset *spdk_app_get_core_mask(void);
 
-#define SPDK_APP_GETOPT_STRING "c:de:ghi:m:n:p:qr:s:t:uB:W:"
+#define SPDK_APP_GETOPT_STRING "c:de:ghi:m:n:p:qr:s:t:uwB:W:"
 
 enum spdk_app_parse_args_rvals {
 	SPDK_APP_PARSE_ARGS_HELP = 0,

lib/event/app.c

@@ -58,14 +58,12 @@ struct spdk_app {
 };
 
 static struct spdk_app g_spdk_app;
+static struct spdk_event *g_app_start_event = NULL;
 static struct spdk_event *g_shutdown_event = NULL;
 static int g_init_lcore;
+static bool g_delay_subsystem_init = false;
 static bool g_shutdown_sig_received = false;
 
-static spdk_event_fn g_app_start_fn;
-static void *g_app_start_arg1;
-static void *g_app_start_arg2;
-
 int
 spdk_app_get_shm_id(void)
 {
@@ -207,6 +205,7 @@ spdk_app_opts_init(struct spdk_app_opts *opts)
 	opts->max_delay_us = 0;
 	opts->print_level = SPDK_APP_DEFAULT_LOG_PRINT_LEVEL;
 	opts->rpc_addr = SPDK_DEFAULT_RPC_ADDR;
+	opts->delay_subsystem_init = false;
 }
 
 static int
@@ -270,7 +269,7 @@ static void
 spdk_app_start_application(void)
 {
 	spdk_rpc_set_state(SPDK_RPC_RUNTIME);
-	g_app_start_fn(g_app_start_arg1, g_app_start_arg2);
+	spdk_event_call(g_app_start_event);
 }
 
 static void
@@ -279,7 +278,9 @@ spdk_app_start_rpc(void *arg1, void *arg2)
 	const char *rpc_addr = arg1;
 
 	spdk_rpc_initialize(rpc_addr);
-	spdk_app_start_application();
+	if (!g_delay_subsystem_init) {
+		spdk_app_start_application();
+	}
 }
 
 static struct spdk_conf *
@@ -550,13 +551,17 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_event_fn start_fn,
 	g_spdk_app.shutdown_cb = opts->shutdown_cb;
 	g_spdk_app.rc = 0;
 	g_init_lcore = spdk_env_get_current_core();
-	g_app_start_fn = start_fn;
+	g_delay_subsystem_init = opts->delay_subsystem_init;
-	g_app_start_arg1 = arg1;
+	g_app_start_event = spdk_event_allocate(g_init_lcore, start_fn, arg1, arg2);
-	g_app_start_arg2 = arg2;
+
 	rpc_start_event = spdk_event_allocate(g_init_lcore, spdk_app_start_rpc,
 					      (void *)opts->rpc_addr, NULL);
 
-	spdk_subsystem_init(rpc_start_event);
+	if (!g_delay_subsystem_init) {
+		spdk_subsystem_init(rpc_start_event);
+	} else {
+		spdk_event_call(rpc_start_event);
+	}
 
 	/* This blocks until spdk_app_stop is called */
 	spdk_reactors_start();
@@ -630,6 +635,7 @@ usage(char *executable_name, struct spdk_app_opts *default_opts, void (*app_usag
 		printf("all hugepage memory)\n");
 	}
 	printf(" -u         disable PCI access.\n");
+	printf(" -w         wait for RPCs to initialize subsystems\n");
 	printf(" -B addr    pci addr to blacklist\n");
 	printf(" -W addr    pci addr to whitelist (-B and -W cannot be used at the same time)\n");
 	spdk_tracelog_usage(stdout, "-t");
@@ -757,6 +763,9 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts,
 		case 'u':
 			opts->no_pci = true;
 			break;
+		case 'w':
+			opts->delay_subsystem_init = true;
+			break;
 		case 'B':
 			if (opts->pci_whitelist) {
 				free(opts->pci_whitelist);
@@ -803,9 +812,51 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts,
 		}
 	}
 
+	/* TBD: Replace warning by failure when RPCs for startup are prepared. */
+	if (opts->config_file && opts->delay_subsystem_init) {
+		fprintf(stderr,
+			"WARNING: -w and config file are used at the same time. "
+			"- Please be careful one options might overwrite others.\n");
+	}
+
 parse_done:
 	free(getopt_str);
 
 parse_early_fail:
 	return rval;
 }
+
+static void
+spdk_rpc_start_subsystem_init_cpl(void *arg1, void *arg2)
+{
+	struct spdk_jsonrpc_request *request = arg1;
+	struct spdk_json_write_ctx *w;
+
+	spdk_app_start_application();
+
+	w = spdk_jsonrpc_begin_result(request);
+	if (w == NULL) {
+		return;
+	}
+
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+}
+
+static void
+spdk_rpc_start_subsystem_init(struct spdk_jsonrpc_request *request,
+			      const struct spdk_json_val *params)
+{
+	struct spdk_event *cb_event;
+
+	if (params != NULL) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "start_subsystem_init requires no parameters");
+		return;
+	}
+
+	cb_event = spdk_event_allocate(g_init_lcore, spdk_rpc_start_subsystem_init_cpl,
+				       request, NULL);
+	spdk_subsystem_init(cb_event);
+}
+SPDK_RPC_REGISTER("start_subsystem_init", spdk_rpc_start_subsystem_init, SPDK_RPC_STARTUP)

scripts/rpc.py

@@ -40,6 +40,13 @@ if __name__ == "__main__":
                         help='Verbose mode', action='store_true')
     subparsers = parser.add_subparsers(help='RPC methods')
 
+    @call_cmd
+    def start_subsystem_init(args):
+        rpc.start_subsystem_init(args.client)
+
+    p = subparsers.add_parser('start_subsystem_init', help='Start initialization of subsystems')
+    p.set_defaults(func=start_subsystem_init)
+
     @call_cmd
     def get_rpc_methods(args):
         print_dict(rpc.get_rpc_methods(args.client))

scripts/rpc/__init__.py

@@ -14,6 +14,10 @@ from . import subsystem
 from . import vhost
 
 
+def start_subsystem_init(client):
+    return client.call('start_subsystem_init')
+
+
 def get_rpc_methods(client):
     return client.call('get_rpc_methods')