From 28589dbbe864bd035916b8b7e52c20e25de91d31 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Wed, 2 May 2018 13:50:39 +0900 Subject: [PATCH] rpc: Add start_subsystems_init RPC to start subsystems when -w option is used RPC state of the RPC server, g_rpc_state is set by spdk_rpc_set_state(state). If the command line option '-w' is specified, g_rpc_state is initialized to SPDK_RPC_STARTUP. Otherwise, g_rpc_state is initialized to SPDK_RPC_RUNTIME. When g_rpc_state is initialized to SPDK_RPC_STARTUP, SPDK start RPC server and wait for start_subsystems_init RPC. When SPDK receive start_subsystems_init RPC, SPDK start initialization of subsystems. The RPC waits for completion of initialization of subsystems. When SPDK completes initialization of subsystems, SPDK change g_rpc_state to SPDK_RPC_RUNTIME and then start application. Upcoming new RPCs to initialize options of NVMf-tgt and iSCSI-tgt will be able to be allowed before start_subsystems_init RPC is called. Change-Id: I4fa9c22b64e2fdbc9b9fdb2c47f0018f73f84f7e Signed-off-by: Shuhei Matsumoto Reviewed-on: https://review.gerrithub.io/406919 Tested-by: SPDK Automated Test System Reviewed-by: Jim Harris Reviewed-by: Pawel Wodkowski Reviewed-by: Daniel Verkamp Reviewed-by: Ben Walker --- include/spdk/event.h | 7 +++- lib/event/app.c | 71 +++++++++++++++++++++++++++++++++++------ scripts/rpc.py | 7 ++++ scripts/rpc/__init__.py | 4 +++ 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/include/spdk/event.h b/include/spdk/event.h index 0688efc72..ed8a8b686 100644 --- a/include/spdk/event.h +++ b/include/spdk/event.h @@ -100,6 +100,11 @@ struct spdk_app_opts { * specified in microseconds. */ uint64_t max_delay_us; + + /* Wait for the associated RPC before initializing subsystems + * when this flag is enabled. + */ + bool delay_subsystem_init; }; /** @@ -189,7 +194,7 @@ int spdk_app_parse_core_mask(const char *mask, struct spdk_cpuset *cpumask); */ struct spdk_cpuset *spdk_app_get_core_mask(void); -#define SPDK_APP_GETOPT_STRING "c:de:ghi:m:n:p:qr:s:t:uB:W:" +#define SPDK_APP_GETOPT_STRING "c:de:ghi:m:n:p:qr:s:t:uwB:W:" enum spdk_app_parse_args_rvals { SPDK_APP_PARSE_ARGS_HELP = 0, diff --git a/lib/event/app.c b/lib/event/app.c index 3d8d058fa..0b39033c5 100644 --- a/lib/event/app.c +++ b/lib/event/app.c @@ -58,14 +58,12 @@ struct spdk_app { }; static struct spdk_app g_spdk_app; +static struct spdk_event *g_app_start_event = NULL; static struct spdk_event *g_shutdown_event = NULL; static int g_init_lcore; +static bool g_delay_subsystem_init = false; static bool g_shutdown_sig_received = false; -static spdk_event_fn g_app_start_fn; -static void *g_app_start_arg1; -static void *g_app_start_arg2; - int spdk_app_get_shm_id(void) { @@ -207,6 +205,7 @@ spdk_app_opts_init(struct spdk_app_opts *opts) opts->max_delay_us = 0; opts->print_level = SPDK_APP_DEFAULT_LOG_PRINT_LEVEL; opts->rpc_addr = SPDK_DEFAULT_RPC_ADDR; + opts->delay_subsystem_init = false; } static int @@ -270,7 +269,7 @@ static void spdk_app_start_application(void) { spdk_rpc_set_state(SPDK_RPC_RUNTIME); - g_app_start_fn(g_app_start_arg1, g_app_start_arg2); + spdk_event_call(g_app_start_event); } static void @@ -279,7 +278,9 @@ spdk_app_start_rpc(void *arg1, void *arg2) const char *rpc_addr = arg1; spdk_rpc_initialize(rpc_addr); - spdk_app_start_application(); + if (!g_delay_subsystem_init) { + spdk_app_start_application(); + } } static struct spdk_conf * @@ -550,13 +551,17 @@ spdk_app_start(struct spdk_app_opts *opts, spdk_event_fn start_fn, g_spdk_app.shutdown_cb = opts->shutdown_cb; g_spdk_app.rc = 0; g_init_lcore = spdk_env_get_current_core(); - g_app_start_fn = start_fn; - g_app_start_arg1 = arg1; - g_app_start_arg2 = arg2; + g_delay_subsystem_init = opts->delay_subsystem_init; + g_app_start_event = spdk_event_allocate(g_init_lcore, start_fn, arg1, arg2); + rpc_start_event = spdk_event_allocate(g_init_lcore, spdk_app_start_rpc, (void *)opts->rpc_addr, NULL); - spdk_subsystem_init(rpc_start_event); + if (!g_delay_subsystem_init) { + spdk_subsystem_init(rpc_start_event); + } else { + spdk_event_call(rpc_start_event); + } /* This blocks until spdk_app_stop is called */ spdk_reactors_start(); @@ -630,6 +635,7 @@ usage(char *executable_name, struct spdk_app_opts *default_opts, void (*app_usag printf("all hugepage memory)\n"); } printf(" -u disable PCI access.\n"); + printf(" -w wait for RPCs to initialize subsystems\n"); printf(" -B addr pci addr to blacklist\n"); printf(" -W addr pci addr to whitelist (-B and -W cannot be used at the same time)\n"); spdk_tracelog_usage(stdout, "-t"); @@ -757,6 +763,9 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, case 'u': opts->no_pci = true; break; + case 'w': + opts->delay_subsystem_init = true; + break; case 'B': if (opts->pci_whitelist) { free(opts->pci_whitelist); @@ -803,9 +812,51 @@ spdk_app_parse_args(int argc, char **argv, struct spdk_app_opts *opts, } } + /* TBD: Replace warning by failure when RPCs for startup are prepared. */ + if (opts->config_file && opts->delay_subsystem_init) { + fprintf(stderr, + "WARNING: -w and config file are used at the same time. " + "- Please be careful one options might overwrite others.\n"); + } + parse_done: free(getopt_str); parse_early_fail: return rval; } + +static void +spdk_rpc_start_subsystem_init_cpl(void *arg1, void *arg2) +{ + struct spdk_jsonrpc_request *request = arg1; + struct spdk_json_write_ctx *w; + + spdk_app_start_application(); + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} + +static void +spdk_rpc_start_subsystem_init(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct spdk_event *cb_event; + + if (params != NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "start_subsystem_init requires no parameters"); + return; + } + + cb_event = spdk_event_allocate(g_init_lcore, spdk_rpc_start_subsystem_init_cpl, + request, NULL); + spdk_subsystem_init(cb_event); +} +SPDK_RPC_REGISTER("start_subsystem_init", spdk_rpc_start_subsystem_init, SPDK_RPC_STARTUP) diff --git a/scripts/rpc.py b/scripts/rpc.py index 9f7e4927e..5118da92b 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -40,6 +40,13 @@ if __name__ == "__main__": help='Verbose mode', action='store_true') subparsers = parser.add_subparsers(help='RPC methods') + @call_cmd + def start_subsystem_init(args): + rpc.start_subsystem_init(args.client) + + p = subparsers.add_parser('start_subsystem_init', help='Start initialization of subsystems') + p.set_defaults(func=start_subsystem_init) + @call_cmd def get_rpc_methods(args): print_dict(rpc.get_rpc_methods(args.client)) diff --git a/scripts/rpc/__init__.py b/scripts/rpc/__init__.py index 75f371117..2e6a6a4e3 100755 --- a/scripts/rpc/__init__.py +++ b/scripts/rpc/__init__.py @@ -14,6 +14,10 @@ from . import subsystem from . import vhost +def start_subsystem_init(client): + return client.call('start_subsystem_init') + + def get_rpc_methods(client): return client.call('get_rpc_methods')