subsystem/nvmf: Add set_nvmf_target_options/config RPCs

Two new RPCs allocate and initialize options and conf dynamically.
Initialization of NVMf subsystem skips initialization of them
if they are already allocated.

To use and test them, add python script too.

Change-Id: I3be83c33edf9960cb4340d52abb47b7331a6fb92
Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/406932
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/event/subsystems/nvmf/conf.c

@@ -41,11 +41,10 @@
 #include "spdk/string.h"
 #include "spdk/util.h"
 
-#define ACCEPT_TIMEOUT_US		10000 /* 10ms */
-
 #define SPDK_NVMF_MAX_NAMESPACES (1 << 14)
 
-struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf;
+struct spdk_nvmf_tgt_opts *g_spdk_nvmf_tgt_opts = NULL;
+struct spdk_nvmf_tgt_conf *g_spdk_nvmf_tgt_conf = NULL;
 
 static int
 spdk_add_nvmf_discovery_subsystem(void)
@@ -65,15 +64,14 @@ spdk_add_nvmf_discovery_subsystem(void)
 }
 
 static void
-spdk_nvmf_read_config_file_params(struct spdk_conf_section *sp,
-				  struct spdk_nvmf_tgt_opts *opts)
+spdk_nvmf_read_config_file_tgt_opts(struct spdk_conf_section *sp,
+				    struct spdk_nvmf_tgt_opts *opts)
 {
 	int max_queue_depth;
 	int max_queues_per_sess;
 	int in_capsule_data_size;
 	int max_io_size;
 	int io_unit_size;
-	int acceptor_poll_rate;
 
 	max_queue_depth = spdk_conf_section_get_intval(sp, "MaxQueueDepth");
 	if (max_queue_depth >= 0) {
@@ -99,29 +97,90 @@ spdk_nvmf_read_config_file_params(struct spdk_conf_section *sp,
 	if (io_unit_size >= 0) {
 		opts->io_unit_size = io_unit_size;
 	}
+}
+
+static void
+spdk_nvmf_read_config_file_tgt_conf(struct spdk_conf_section *sp,
+				    struct spdk_nvmf_tgt_conf *conf)
+{
+	int acceptor_poll_rate;
 
 	acceptor_poll_rate = spdk_conf_section_get_intval(sp, "AcceptorPollRate");
 	if (acceptor_poll_rate >= 0) {
-		g_spdk_nvmf_tgt_conf.acceptor_poll_rate = acceptor_poll_rate;
+		conf->acceptor_poll_rate = acceptor_poll_rate;
 	}
 }
 
+static struct spdk_nvmf_tgt_opts *
+spdk_nvmf_parse_tgt_opts(void)
+{
+	struct spdk_nvmf_tgt_opts *opts;
+	struct spdk_conf_section *sp;
+
+	opts = calloc(1, sizeof(*opts));
+	if (!opts) {
+		SPDK_ERRLOG("calloc() failed for target options\n");
+		return NULL;
+	}
+
+	spdk_nvmf_tgt_opts_init(opts);
+
+	sp = spdk_conf_find_section(NULL, "Nvmf");
+	if (sp != NULL) {
+		spdk_nvmf_read_config_file_tgt_opts(sp, opts);
+	}
+
+	return opts;
+}
+
+static struct spdk_nvmf_tgt_conf *
+spdk_nvmf_parse_tgt_conf(void)
+{
+	struct spdk_nvmf_tgt_conf *conf;
+	struct spdk_conf_section *sp;
+
+	conf = calloc(1, sizeof(*conf));
+	if (!conf) {
+		SPDK_ERRLOG("calloc() failed for target conf\n");
+		return NULL;
+	}
+
+	conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US;
+
+	sp = spdk_conf_find_section(NULL, "Nvmf");
+	if (sp != NULL) {
+		spdk_nvmf_read_config_file_tgt_conf(sp, conf);
+	}
+
+	return conf;
+}
+
 static int
 spdk_nvmf_parse_nvmf_tgt(void)
 {
-	struct spdk_conf_section *sp;
-	struct spdk_nvmf_tgt_opts opts;
 	int rc;
 
-	spdk_nvmf_tgt_opts_init(&opts);
-	g_spdk_nvmf_tgt_conf.acceptor_poll_rate = ACCEPT_TIMEOUT_US;
-
-	sp = spdk_conf_find_section(NULL, "Nvmf");
-	if (sp != NULL) {
-		spdk_nvmf_read_config_file_params(sp, &opts);
+	if (!g_spdk_nvmf_tgt_opts) {
+		g_spdk_nvmf_tgt_opts = spdk_nvmf_parse_tgt_opts();
+		if (!g_spdk_nvmf_tgt_opts) {
+			SPDK_ERRLOG("spdk_nvmf_parse_tgt_opts() failed\n");
+			return -1;
+		}
 	}
 
-	g_spdk_nvmf_tgt = spdk_nvmf_tgt_create(&opts);
+	if (!g_spdk_nvmf_tgt_conf) {
+		g_spdk_nvmf_tgt_conf = spdk_nvmf_parse_tgt_conf();
+		if (!g_spdk_nvmf_tgt_conf) {
+			SPDK_ERRLOG("spdk_nvmf_parse_tgt_conf() failed\n");
+			return -1;
+		}
+	}
+
+	g_spdk_nvmf_tgt = spdk_nvmf_tgt_create(g_spdk_nvmf_tgt_opts);
+
+	free(g_spdk_nvmf_tgt_opts);
+	g_spdk_nvmf_tgt_opts = NULL;
+
 	if (!g_spdk_nvmf_tgt) {
 		SPDK_ERRLOG("spdk_nvmf_tgt_create() failed\n");
 		return -1;

lib/event/subsystems/nvmf/event_nvmf.h

@@ -42,11 +42,14 @@
 #include "spdk_internal/event.h"
 #include "spdk_internal/log.h"
 
+#define ACCEPT_TIMEOUT_US	10000 /* 10ms */
+
 struct spdk_nvmf_tgt_conf {
 	uint32_t acceptor_poll_rate;
 };
 
-extern struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf;
+extern struct spdk_nvmf_tgt_opts *g_spdk_nvmf_tgt_opts;
+extern struct spdk_nvmf_tgt_conf *g_spdk_nvmf_tgt_conf;
 
 extern struct spdk_nvmf_tgt *g_spdk_nvmf_tgt;
 

lib/event/subsystems/nvmf/nvmf_rpc.c

@@ -1554,3 +1554,110 @@ nvmf_rpc_subsystem_allow_any_host(struct spdk_jsonrpc_request *request,
 }
 SPDK_RPC_REGISTER("nvmf_subsystem_allow_any_host", nvmf_rpc_subsystem_allow_any_host,
 		  SPDK_RPC_RUNTIME)
+
+static const struct spdk_json_object_decoder nvmf_rpc_subsystem_tgt_opts_decoder[] = {
+	{"max_queue_depth", offsetof(struct spdk_nvmf_tgt_opts, max_queue_depth), spdk_json_decode_uint16, true},
+	{"max_qpairs_per_ctrlr", offsetof(struct spdk_nvmf_tgt_opts, max_qpairs_per_ctrlr), spdk_json_decode_uint16, true},
+	{"in_capsule_data_size", offsetof(struct spdk_nvmf_tgt_opts, in_capsule_data_size), spdk_json_decode_uint32, true},
+	{"max_io_size", offsetof(struct spdk_nvmf_tgt_opts, max_io_size), spdk_json_decode_uint32, true},
+	{"max_subsystems", offsetof(struct spdk_nvmf_tgt_opts, max_subsystems), spdk_json_decode_uint32, true},
+	{"io_unit_size", offsetof(struct spdk_nvmf_tgt_opts, io_unit_size), spdk_json_decode_uint32, true},
+};
+
+static void
+nvmf_rpc_subsystem_set_tgt_opts(struct spdk_jsonrpc_request *request,
+				const struct spdk_json_val *params)
+{
+	struct spdk_nvmf_tgt_opts *opts;
+	struct spdk_json_write_ctx *w;
+
+	if (g_spdk_nvmf_tgt_opts != NULL) {
+		SPDK_ERRLOG("this RPC must not be called more than once.\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Must not call more than once");
+		return;
+	}
+
+	opts = calloc(1, sizeof(*opts));
+	if (opts == NULL) {
+		SPDK_ERRLOG("malloc() failed for target options\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Out of memory");
+		return;
+	}
+
+	spdk_nvmf_tgt_opts_init(opts);
+
+	if (params != NULL) {
+		if (spdk_json_decode_object(params, nvmf_rpc_subsystem_tgt_opts_decoder,
+					    SPDK_COUNTOF(nvmf_rpc_subsystem_tgt_opts_decoder), opts)) {
+			free(opts);
+			SPDK_ERRLOG("spdk_json_decode_object() failed\n");
+			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+							 "Invalid parameters");
+			return;
+		}
+	}
+
+	g_spdk_nvmf_tgt_opts = opts;
+
+	w = spdk_jsonrpc_begin_result(request);
+	if (w == NULL) {
+		return;
+	}
+
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+}
+SPDK_RPC_REGISTER("set_nvmf_target_options", nvmf_rpc_subsystem_set_tgt_opts, SPDK_RPC_STARTUP)
+
+static const struct spdk_json_object_decoder nvmf_rpc_subsystem_tgt_conf_decoder[] = {
+	{"acceptor_poll_rate", offsetof(struct spdk_nvmf_tgt_conf, acceptor_poll_rate), spdk_json_decode_uint32, true},
+};
+
+static void
+nvmf_rpc_subsystem_set_tgt_conf(struct spdk_jsonrpc_request *request,
+				const struct spdk_json_val *params)
+{
+	struct spdk_nvmf_tgt_conf *conf;
+	struct spdk_json_write_ctx *w;
+
+	if (g_spdk_nvmf_tgt_conf != NULL) {
+		SPDK_ERRLOG("this RPC must not be called more than once.\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Must not call more than once");
+		return;
+	}
+
+	conf = calloc(1, sizeof(*conf));
+	if (conf == NULL) {
+		SPDK_ERRLOG("calloc() failed for target config\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR,
+						 "Out of memory");
+		return;
+	}
+
+	conf->acceptor_poll_rate = ACCEPT_TIMEOUT_US;
+
+	if (params != NULL) {
+		if (spdk_json_decode_object(params, nvmf_rpc_subsystem_tgt_conf_decoder,
+					    SPDK_COUNTOF(nvmf_rpc_subsystem_tgt_conf_decoder), conf)) {
+			free(conf);
+			SPDK_ERRLOG("spdk_json_decode_object() failed\n");
+			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+							 "Invalid parameters");
+			return;
+		}
+	}
+
+	g_spdk_nvmf_tgt_conf = conf;
+
+	w = spdk_jsonrpc_begin_result(request);
+	if (w == NULL) {
+		return;
+	}
+
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+}
+SPDK_RPC_REGISTER("set_nvmf_target_config", nvmf_rpc_subsystem_set_tgt_conf, SPDK_RPC_STARTUP)

lib/event/subsystems/nvmf/nvmf_tgt.c

@@ -218,6 +218,7 @@ static void
 nvmf_tgt_destroy_done(void *ctx, int status)
 {
 	g_tgt_state = NVMF_TGT_STOPPED;
+	free(g_spdk_nvmf_tgt_conf);
 	nvmf_tgt_advance_state();
 }
 
@@ -278,7 +279,7 @@ nvmf_tgt_advance_state(void)
 		}
 		case NVMF_TGT_INIT_START_ACCEPTOR:
 			g_acceptor_poller = spdk_poller_register(acceptor_poll, g_spdk_nvmf_tgt,
-					    g_spdk_nvmf_tgt_conf.acceptor_poll_rate);
+					    g_spdk_nvmf_tgt_conf->acceptor_poll_rate);
 			SPDK_INFOLOG(SPDK_LOG_NVMF, "Acceptor running\n");
 			g_tgt_state = NVMF_TGT_RUNNING;
 			break;

scripts/rpc.py

@@ -819,6 +819,27 @@ if __name__ == "__main__":
     p.set_defaults(func=get_interfaces)
 
     # NVMe-oF
+    @call_cmd
+    def set_nvmf_target_options(args):
+        rpc.nvmf.set_nvmf_target_options(args.client, args)
+
+    p = subparsers.add_parser('set_nvmf_target_options', help='Set NVMf target options')
+    p.add_argument('-q', '--max-queue-depth', help='Max number of outstanding I/O per queue', type=int)
+    p.add_argument('-p', '--max-qpairs-per-session', help='Max number of SQ and CQ per session', type=int)
+    p.add_argument('-c', '--in-capsule-data-size', help='Max number of in-capsule data size', type=int)
+    p.add_argument('-i', '--max-io-size', help='Max I/O size', type=int)
+    p.add_argument('-x', '--max-subsystems', help='Max number of NVMf subsystems', type=int)
+    p.add_argument('-u', '--io-unit-size', help='I/O unit size', type=int)
+    p.set_defaults(func=set_nvmf_target_options)
+
+    @call_cmd
+    def set_nvmf_target_config(args):
+        rpc.nvmf.set_nvmf_target_config(args.client, args)
+
+    p = subparsers.add_parser('set_nvmf_target_config', help='Set NVMf target config')
+    p.add_argument('-r', '--acceptor-poll-rate', help='How often the acceptor polls for incoming connections', type=int)
+    p.set_defaults(func=set_nvmf_target_config)
+
     @call_cmd
     def get_nvmf_subsystems(args):
         print_dict(rpc.nvmf.get_nvmf_subsystems(args.client, args))

scripts/rpc/nvmf.py

@@ -1,3 +1,29 @@
+def set_nvmf_target_options(client, args):
+    params = {}
+
+    if args.max_queue_depth:
+        params['max_queue_depth'] = args.max_queue_depth
+    if args.max_qpairs_per_session:
+        params['max_qpairs_per_session'] = args.max_qpairs_per_session
+    if args.in_capsule_data_size:
+        params['in_capsule_data_size'] = args.in_capsule_data_size
+    if args.max_io_size:
+        params['max_io_size'] = args.max_io_size
+    if args.max_subsystems:
+        params['max_subsystems'] = args.max_subsystems
+    if args.io_unit_size:
+        params['io_unit_size'] = args.io_unit_size
+    return client.call('set_nvmf_target_options', params)
+
+
+def set_nvmf_target_config(client, args):
+    params = {}
+
+    if args.acceptor_poll_rate:
+        params['acceptor_poll_rate'] = args.acceptor_poll_rate
+    return client.call('set_nvmf_target_config', params)
+
+
 def get_nvmf_subsystems(client, args):
     return client.call('get_nvmf_subsystems')