From 46a3d50baa482641a33d5dd9de91e932c73da454 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Fri, 8 Jun 2018 08:00:26 +0900 Subject: [PATCH] subsystem/nvmf: Add set_nvmf_target_options/config RPCs Two new RPCs allocate and initialize options and conf dynamically. Initialization of NVMf subsystem skips initialization of them if they are already allocated. To use and test them, add python script too. Change-Id: I3be83c33edf9960cb4340d52abb47b7331a6fb92 Signed-off-by: Shuhei Matsumoto Reviewed-on: https://review.gerrithub.io/406932 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Jim Harris --- lib/event/subsystems/nvmf/conf.c | 91 +++++++++++++++++---- lib/event/subsystems/nvmf/event_nvmf.h | 5 +- lib/event/subsystems/nvmf/nvmf_rpc.c | 107 +++++++++++++++++++++++++ lib/event/subsystems/nvmf/nvmf_tgt.c | 3 +- scripts/rpc.py | 21 +++++ scripts/rpc/nvmf.py | 26 ++++++ 6 files changed, 235 insertions(+), 18 deletions(-) diff --git a/lib/event/subsystems/nvmf/conf.c b/lib/event/subsystems/nvmf/conf.c index 8f6dea431..ec26094fb 100644 --- a/lib/event/subsystems/nvmf/conf.c +++ b/lib/event/subsystems/nvmf/conf.c @@ -41,11 +41,10 @@ #include "spdk/string.h" #include "spdk/util.h" -#define ACCEPT_TIMEOUT_US 10000 /* 10ms */ - #define SPDK_NVMF_MAX_NAMESPACES (1 << 14) -struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf; +struct spdk_nvmf_tgt_opts *g_spdk_nvmf_tgt_opts = NULL; +struct spdk_nvmf_tgt_conf *g_spdk_nvmf_tgt_conf = NULL; static int spdk_add_nvmf_discovery_subsystem(void) @@ -65,15 +64,14 @@ spdk_add_nvmf_discovery_subsystem(void) } static void -spdk_nvmf_read_config_file_params(struct spdk_conf_section *sp, - struct spdk_nvmf_tgt_opts *opts) +spdk_nvmf_read_config_file_tgt_opts(struct spdk_conf_section *sp, + struct spdk_nvmf_tgt_opts *opts) { int max_queue_depth; int max_queues_per_sess; int in_capsule_data_size; int max_io_size; int io_unit_size; - int acceptor_poll_rate; max_queue_depth = spdk_conf_section_get_intval(sp, "MaxQueueDepth"); if (max_queue_depth >= 0) { @@ -99,29 +97,90 @@ spdk_nvmf_read_config_file_params(struct spdk_conf_section *sp, if (io_unit_size >= 0) { opts->io_unit_size = io_unit_size; } +} + +static void +spdk_nvmf_read_config_file_tgt_conf(struct spdk_conf_section *sp, + struct spdk_nvmf_tgt_conf *conf) +{ + int acceptor_poll_rate; acceptor_poll_rate = spdk_conf_section_get_intval(sp, "AcceptorPollRate"); if (acceptor_poll_rate >= 0) { - g_spdk_nvmf_tgt_conf.acceptor_poll_rate = acceptor_poll_rate; + conf->acceptor_poll_rate = acceptor_poll_rate; } } +static struct spdk_nvmf_tgt_opts * +spdk_nvmf_parse_tgt_opts(void) +{ + struct spdk_nvmf_tgt_opts *opts; + struct spdk_conf_section *sp; + + opts = calloc(1, sizeof(*opts)); + if (!opts) { + SPDK_ERRLOG("calloc() failed for target options\n"); + return NULL; + } + + spdk_nvmf_tgt_opts_init(opts); + + sp = spdk_conf_find_section(NULL, "Nvmf"); + if (sp != NULL) { + spdk_nvmf_read_config_file_tgt_opts(sp, opts); + } + + return opts; +} + +static struct spdk_nvmf_tgt_conf * +spdk_nvmf_parse_tgt_conf(void) +{ + struct spdk_nvmf_tgt_conf *conf; + struct spdk_conf_section *sp; + + conf = calloc(1, sizeof(*conf)); + if (!conf) { + SPDK_ERRLOG("calloc() failed for target conf\n"); + return NULL; + } + + conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US; + + sp = spdk_conf_find_section(NULL, "Nvmf"); + if (sp != NULL) { + spdk_nvmf_read_config_file_tgt_conf(sp, conf); + } + + return conf; +} + static int spdk_nvmf_parse_nvmf_tgt(void) { - struct spdk_conf_section *sp; - struct spdk_nvmf_tgt_opts opts; int rc; - spdk_nvmf_tgt_opts_init(&opts); - g_spdk_nvmf_tgt_conf.acceptor_poll_rate = ACCEPT_TIMEOUT_US; - - sp = spdk_conf_find_section(NULL, "Nvmf"); - if (sp != NULL) { - spdk_nvmf_read_config_file_params(sp, &opts); + if (!g_spdk_nvmf_tgt_opts) { + g_spdk_nvmf_tgt_opts = spdk_nvmf_parse_tgt_opts(); + if (!g_spdk_nvmf_tgt_opts) { + SPDK_ERRLOG("spdk_nvmf_parse_tgt_opts() failed\n"); + return -1; + } } - g_spdk_nvmf_tgt = spdk_nvmf_tgt_create(&opts); + if (!g_spdk_nvmf_tgt_conf) { + g_spdk_nvmf_tgt_conf = spdk_nvmf_parse_tgt_conf(); + if (!g_spdk_nvmf_tgt_conf) { + SPDK_ERRLOG("spdk_nvmf_parse_tgt_conf() failed\n"); + return -1; + } + } + + g_spdk_nvmf_tgt = spdk_nvmf_tgt_create(g_spdk_nvmf_tgt_opts); + + free(g_spdk_nvmf_tgt_opts); + g_spdk_nvmf_tgt_opts = NULL; + if (!g_spdk_nvmf_tgt) { SPDK_ERRLOG("spdk_nvmf_tgt_create() failed\n"); return -1; diff --git a/lib/event/subsystems/nvmf/event_nvmf.h b/lib/event/subsystems/nvmf/event_nvmf.h index f14cba328..543d02a59 100644 --- a/lib/event/subsystems/nvmf/event_nvmf.h +++ b/lib/event/subsystems/nvmf/event_nvmf.h @@ -42,11 +42,14 @@ #include "spdk_internal/event.h" #include "spdk_internal/log.h" +#define ACCEPT_TIMEOUT_US 10000 /* 10ms */ + struct spdk_nvmf_tgt_conf { uint32_t acceptor_poll_rate; }; -extern struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf; +extern struct spdk_nvmf_tgt_opts *g_spdk_nvmf_tgt_opts; +extern struct spdk_nvmf_tgt_conf *g_spdk_nvmf_tgt_conf; extern struct spdk_nvmf_tgt *g_spdk_nvmf_tgt; diff --git a/lib/event/subsystems/nvmf/nvmf_rpc.c b/lib/event/subsystems/nvmf/nvmf_rpc.c index b2a4b2844..ba4d8ab63 100644 --- a/lib/event/subsystems/nvmf/nvmf_rpc.c +++ b/lib/event/subsystems/nvmf/nvmf_rpc.c @@ -1554,3 +1554,110 @@ nvmf_rpc_subsystem_allow_any_host(struct spdk_jsonrpc_request *request, } SPDK_RPC_REGISTER("nvmf_subsystem_allow_any_host", nvmf_rpc_subsystem_allow_any_host, SPDK_RPC_RUNTIME) + +static const struct spdk_json_object_decoder nvmf_rpc_subsystem_tgt_opts_decoder[] = { + {"max_queue_depth", offsetof(struct spdk_nvmf_tgt_opts, max_queue_depth), spdk_json_decode_uint16, true}, + {"max_qpairs_per_ctrlr", offsetof(struct spdk_nvmf_tgt_opts, max_qpairs_per_ctrlr), spdk_json_decode_uint16, true}, + {"in_capsule_data_size", offsetof(struct spdk_nvmf_tgt_opts, in_capsule_data_size), spdk_json_decode_uint32, true}, + {"max_io_size", offsetof(struct spdk_nvmf_tgt_opts, max_io_size), spdk_json_decode_uint32, true}, + {"max_subsystems", offsetof(struct spdk_nvmf_tgt_opts, max_subsystems), spdk_json_decode_uint32, true}, + {"io_unit_size", offsetof(struct spdk_nvmf_tgt_opts, io_unit_size), spdk_json_decode_uint32, true}, +}; + +static void +nvmf_rpc_subsystem_set_tgt_opts(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct spdk_nvmf_tgt_opts *opts; + struct spdk_json_write_ctx *w; + + if (g_spdk_nvmf_tgt_opts != NULL) { + SPDK_ERRLOG("this RPC must not be called more than once.\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Must not call more than once"); + return; + } + + opts = calloc(1, sizeof(*opts)); + if (opts == NULL) { + SPDK_ERRLOG("malloc() failed for target options\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Out of memory"); + return; + } + + spdk_nvmf_tgt_opts_init(opts); + + if (params != NULL) { + if (spdk_json_decode_object(params, nvmf_rpc_subsystem_tgt_opts_decoder, + SPDK_COUNTOF(nvmf_rpc_subsystem_tgt_opts_decoder), opts)) { + free(opts); + SPDK_ERRLOG("spdk_json_decode_object() failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + return; + } + } + + g_spdk_nvmf_tgt_opts = opts; + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} +SPDK_RPC_REGISTER("set_nvmf_target_options", nvmf_rpc_subsystem_set_tgt_opts, SPDK_RPC_STARTUP) + +static const struct spdk_json_object_decoder nvmf_rpc_subsystem_tgt_conf_decoder[] = { + {"acceptor_poll_rate", offsetof(struct spdk_nvmf_tgt_conf, acceptor_poll_rate), spdk_json_decode_uint32, true}, +}; + +static void +nvmf_rpc_subsystem_set_tgt_conf(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct spdk_nvmf_tgt_conf *conf; + struct spdk_json_write_ctx *w; + + if (g_spdk_nvmf_tgt_conf != NULL) { + SPDK_ERRLOG("this RPC must not be called more than once.\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Must not call more than once"); + return; + } + + conf = calloc(1, sizeof(*conf)); + if (conf == NULL) { + SPDK_ERRLOG("calloc() failed for target config\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, + "Out of memory"); + return; + } + + conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US; + + if (params != NULL) { + if (spdk_json_decode_object(params, nvmf_rpc_subsystem_tgt_conf_decoder, + SPDK_COUNTOF(nvmf_rpc_subsystem_tgt_conf_decoder), conf)) { + free(conf); + SPDK_ERRLOG("spdk_json_decode_object() failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + return; + } + } + + g_spdk_nvmf_tgt_conf = conf; + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} +SPDK_RPC_REGISTER("set_nvmf_target_config", nvmf_rpc_subsystem_set_tgt_conf, SPDK_RPC_STARTUP) diff --git a/lib/event/subsystems/nvmf/nvmf_tgt.c b/lib/event/subsystems/nvmf/nvmf_tgt.c index 6e4786f1d..73234efad 100644 --- a/lib/event/subsystems/nvmf/nvmf_tgt.c +++ b/lib/event/subsystems/nvmf/nvmf_tgt.c @@ -218,6 +218,7 @@ static void nvmf_tgt_destroy_done(void *ctx, int status) { g_tgt_state = NVMF_TGT_STOPPED; + free(g_spdk_nvmf_tgt_conf); nvmf_tgt_advance_state(); } @@ -278,7 +279,7 @@ nvmf_tgt_advance_state(void) } case NVMF_TGT_INIT_START_ACCEPTOR: g_acceptor_poller = spdk_poller_register(acceptor_poll, g_spdk_nvmf_tgt, - g_spdk_nvmf_tgt_conf.acceptor_poll_rate); + g_spdk_nvmf_tgt_conf->acceptor_poll_rate); SPDK_INFOLOG(SPDK_LOG_NVMF, "Acceptor running\n"); g_tgt_state = NVMF_TGT_RUNNING; break; diff --git a/scripts/rpc.py b/scripts/rpc.py index 25690053b..85136a373 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -819,6 +819,27 @@ if __name__ == "__main__": p.set_defaults(func=get_interfaces) # NVMe-oF + @call_cmd + def set_nvmf_target_options(args): + rpc.nvmf.set_nvmf_target_options(args.client, args) + + p = subparsers.add_parser('set_nvmf_target_options', help='Set NVMf target options') + p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/O per queue', type=int) + p.add_argument('-p', '--max-qpairs-per-session', help='Max number of SQ and CQ per session', type=int) + p.add_argument('-c', '--in-capsule-data-size', help='Max number of in-capsule data size', type=int) + p.add_argument('-i', '--max-io-size', help='Max I/O size', type=int) + p.add_argument('-x', '--max-subsystems', help='Max number of NVMf subsystems', type=int) + p.add_argument('-u', '--io-unit-size', help='I/O unit size', type=int) + p.set_defaults(func=set_nvmf_target_options) + + @call_cmd + def set_nvmf_target_config(args): + rpc.nvmf.set_nvmf_target_config(args.client, args) + + p = subparsers.add_parser('set_nvmf_target_config', help='Set NVMf target config') + p.add_argument('-r', '--acceptor-poll-rate', help='How often the acceptor polls for incoming connections', type=int) + p.set_defaults(func=set_nvmf_target_config) + @call_cmd def get_nvmf_subsystems(args): print_dict(rpc.nvmf.get_nvmf_subsystems(args.client, args)) diff --git a/scripts/rpc/nvmf.py b/scripts/rpc/nvmf.py index fdfb0881c..ba6ce45cf 100755 --- a/scripts/rpc/nvmf.py +++ b/scripts/rpc/nvmf.py @@ -1,3 +1,29 @@ +def set_nvmf_target_options(client, args): + params = {} + + if args.max_queue_depth: + params['max_queue_depth'] = args.max_queue_depth + if args.max_qpairs_per_session: + params['max_qpairs_per_session'] = args.max_qpairs_per_session + if args.in_capsule_data_size: + params['in_capsule_data_size'] = args.in_capsule_data_size + if args.max_io_size: + params['max_io_size'] = args.max_io_size + if args.max_subsystems: + params['max_subsystems'] = args.max_subsystems + if args.io_unit_size: + params['io_unit_size'] = args.io_unit_size + return client.call('set_nvmf_target_options', params) + + +def set_nvmf_target_config(client, args): + params = {} + + if args.acceptor_poll_rate: + params['acceptor_poll_rate'] = args.acceptor_poll_rate + return client.call('set_nvmf_target_config', params) + + def get_nvmf_subsystems(client, args): return client.call('get_nvmf_subsystems')