bdev/nvme: add RPC to enable and disable hotplug

Change-Id: Iac7340dd0df427ba4e9df5b1c52cfb732d011ae8
Signed-off-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-on: https://review.gerrithub.io/419093
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

doc/jsonrpc.md

@@ -934,6 +934,45 @@ Example response:
 }
 ~~~
 
+## set_bdev_nvme_hotplug {#rpc_set_bdev_nvme_hotplug}
+
+Change settings of the NVMe hotplug feature. If enabled, PCIe NVMe bdevs will be automatically discovered on insertion
+and deleted on removal.
+
+### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------
+enabled                 | Required | string      | True to enable, false to disable
+period_us               | Optional | number      | How often to poll for hot-insert and hot-remove events. Values: 0 - reset/use default or 1 to 10000000.
+
+### Example
+
+Example request:
+
+~~~
+request:
+{
+  "params": {
+    "enabled": true,
+    "period_us": 2000
+  },
+  "jsonrpc": "2.0",
+  "method": "set_bdev_nvme_hotplug",
+  "id": 1
+}
+~~~
+
+Example response:
+
+~~~
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~
+
 ## construct_nvme_bdev {#rpc_construct_nvme_bdev}
 
 Construct @ref bdev_config_nvme

lib/bdev/nvme/bdev_nvme.c

@@ -101,11 +101,13 @@ static struct spdk_bdev_nvme_opts g_opts = {
 	.nvme_adminq_poll_period_us = 1000000ULL,
 };
 
-static bool g_bdev_nvme_init_done = false;
+#define NVME_HOTPLUG_POLL_PERIOD_MAX			10000000ULL
+#define NVME_HOTPLUG_POLL_PERIOD_DEFAULT		100000ULL
 
 static int g_hot_insert_nvme_controller_index = 0;
-static int g_nvme_hotplug_poll_timeout_us = 0;
+static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT;
 static bool g_nvme_hotplug_enabled = false;
+static struct spdk_thread *g_bdev_nvme_init_thread;
 static struct spdk_poller *g_hotplug_poller;
 static char *g_nvme_hostnqn = NULL;
 static pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -1048,7 +1050,7 @@ spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts)
 int
 spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
 {
-	if (g_bdev_nvme_init_done) {
+	if (g_bdev_nvme_init_thread != NULL) {
 		return -EPERM;
 	}
 
@@ -1056,6 +1058,55 @@ spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts)
 
 	return 0;
 }
+struct set_nvme_hotplug_ctx {
+	uint64_t period_us;
+	bool enabled;
+	spdk_thread_fn fn;
+	void *fn_ctx;
+};
+
+static void
+set_nvme_hotplug_period_cb(void *_ctx)
+{
+	struct set_nvme_hotplug_ctx *ctx = _ctx;
+
+	spdk_poller_unregister(&g_hotplug_poller);
+	if (ctx->enabled) {
+		g_hotplug_poller = spdk_poller_register(bdev_nvme_hotplug, NULL, ctx->period_us);
+	}
+
+	g_nvme_hotplug_poll_period_us = ctx->period_us;
+	g_nvme_hotplug_enabled = ctx->enabled;
+	if (ctx->fn) {
+		ctx->fn(ctx->fn_ctx);
+	}
+
+	free(ctx);
+}
+
+int
+spdk_bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_thread_fn cb, void *cb_ctx)
+{
+	struct set_nvme_hotplug_ctx *ctx;
+
+	if (enabled == true && !spdk_process_is_primary()) {
+		return -EPERM;
+	}
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (ctx == NULL) {
+		return -ENOMEM;
+	}
+
+	period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us;
+	ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX);
+	ctx->enabled = enabled;
+	ctx->fn = cb;
+	ctx->fn_ctx = cb_ctx;
+
+	spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx);
+	return 0;
+}
 
 int
 spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
@@ -1156,8 +1207,10 @@ bdev_nvme_library_init(void)
 	struct nvme_probe_ctx *probe_ctx = NULL;
 	int retry_count;
 	uint32_t local_nvme_num = 0;
+	int64_t hotplug_period;
+	bool hotplug_enabled = g_nvme_hotplug_enabled;
 
-	g_bdev_nvme_init_done = true;
+	g_bdev_nvme_init_thread = spdk_get_thread();
 
 	sp = spdk_conf_find_section(NULL, "Nvme");
 	if (sp == NULL) {
@@ -1242,13 +1295,10 @@ bdev_nvme_library_init(void)
 	}
 
 	if (spdk_process_is_primary()) {
-		g_nvme_hotplug_enabled = spdk_conf_section_get_boolval(sp, "HotplugEnable", false);
+		hotplug_enabled = spdk_conf_section_get_boolval(sp, "HotplugEnable", false);
 	}
 
-	g_nvme_hotplug_poll_timeout_us = spdk_conf_section_get_intval(sp, "HotplugPollRate");
+	hotplug_period = spdk_conf_section_get_intval(sp, "HotplugPollRate");
-	if (g_nvme_hotplug_poll_timeout_us <= 0 || g_nvme_hotplug_poll_timeout_us > 100000) {
-		g_nvme_hotplug_poll_timeout_us = 100000;
-	}
 
 	g_nvme_hostnqn = spdk_conf_section_get_val(sp, "HostNQN");
 	probe_ctx->hostnqn = g_nvme_hostnqn;
@@ -1335,15 +1385,13 @@ bdev_nvme_library_init(void)
 		}
 	}
 
-	if (g_nvme_hotplug_enabled) {
+	rc = spdk_bdev_nvme_set_hotplug(hotplug_enabled, hotplug_period, NULL, NULL);
-		g_hotplug_poller = spdk_poller_register(bdev_nvme_hotplug, NULL,
+	if (rc) {
-							g_nvme_hotplug_poll_timeout_us);
+		SPDK_ERRLOG("Failed to setup hotplug (%d): %s", rc, spdk_strerror(rc));
+		rc = -1;
 	}
-
 end:
-	if (rc == 0) {
+	spdk_nvme_retry_count = g_opts.retry_count;
-		spdk_nvme_retry_count = g_opts.retry_count;
-	}
 
 	free(probe_ctx);
 	return rc;
@@ -1352,9 +1400,7 @@ end:
 static void
 bdev_nvme_library_fini(void)
 {
-	if (g_nvme_hotplug_enabled) {
+	spdk_poller_unregister(&g_hotplug_poller);
-		spdk_poller_unregister(&g_hotplug_poller);
-	}
 }
 
 static int
@@ -1680,7 +1726,7 @@ bdev_nvme_get_spdk_running_config(FILE *fp)
 	fprintf(fp, "\n"
 		"# Set how often the hotplug is processed for insert and remove events."
 		"# Units in microseconds.\n");
-	fprintf(fp, "HotplugPollRate %d\n", g_nvme_hotplug_poll_timeout_us);
+	fprintf(fp, "HotplugPollRate %"PRIu64"\n", g_nvme_hotplug_poll_period_us);
 	if (g_nvme_hostnqn) {
 		fprintf(fp, "HostNQN %s\n",  g_nvme_hostnqn);
 	}
@@ -1748,6 +1794,19 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w)
 		spdk_json_write_object_end(w);
 	}
 
+	/* Dump as last parameter to give all NVMe bdevs chance to be constructed
+	 * before enabling hotplug poller.
+	 */
+	spdk_json_write_object_begin(w);
+	spdk_json_write_named_string(w, "method", "set_bdev_nvme_hotplug");
+
+	spdk_json_write_named_object_begin(w, "params");
+	spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us);
+	spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled);
+	spdk_json_write_object_end(w);
+
+	spdk_json_write_object_end(w);
+
 	pthread_mutex_unlock(&g_bdev_nvme_mutex);
 	return 0;
 }

lib/bdev/nvme/bdev_nvme.h

@@ -85,6 +85,7 @@ struct nvme_bdev {
 
 void spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts);
 int spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts);
+int spdk_bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_thread_fn cb, void *cb_ctx);
 
 int spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid,
 			  const char *base_name,

lib/bdev/nvme/bdev_nvme_rpc.c

@@ -109,6 +109,54 @@ invalid:
 }
 SPDK_RPC_REGISTER("set_bdev_nvme_options", spdk_rpc_set_bdev_nvme_options, SPDK_RPC_STARTUP)
 
+struct rpc_bdev_nvme_hotplug {
+	bool enabled;
+	uint64_t period_us;
+};
+
+static const struct spdk_json_object_decoder rpc_bdev_nvme_hotplug_decoders[] = {
+	{"enable", offsetof(struct rpc_bdev_nvme_hotplug, enabled), spdk_json_decode_bool, false},
+	{"period_us", offsetof(struct rpc_bdev_nvme_hotplug, period_us), spdk_json_decode_uint64, true},
+};
+
+static void
+rpc_set_bdev_nvme_hotplug_done(void *ctx)
+{
+	struct spdk_jsonrpc_request *request = ctx;
+	struct spdk_json_write_ctx *w = spdk_jsonrpc_begin_result(request);
+
+	if (w != NULL) {
+		spdk_json_write_bool(w, true);
+		spdk_jsonrpc_end_result(request, w);
+	}
+}
+
+static void
+spdk_rpc_set_bdev_nvme_hotplug(struct spdk_jsonrpc_request *request,
+			       const struct spdk_json_val *params)
+{
+	struct rpc_bdev_nvme_hotplug req = {false, 0};
+	int rc;
+
+	if (spdk_json_decode_object(params, rpc_bdev_nvme_hotplug_decoders,
+				    SPDK_COUNTOF(rpc_bdev_nvme_hotplug_decoders), &req)) {
+		SPDK_ERRLOG("spdk_json_decode_object failed\n");
+		rc = -EINVAL;
+		goto invalid;
+	}
+
+	rc = spdk_bdev_nvme_set_hotplug(req.enabled, req.period_us, rpc_set_bdev_nvme_hotplug_done,
+					request);
+	if (rc) {
+		goto invalid;
+	}
+
+	return;
+invalid:
+	spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, spdk_strerror(-rc));
+}
+SPDK_RPC_REGISTER("set_bdev_nvme_hotplug", spdk_rpc_set_bdev_nvme_hotplug, SPDK_RPC_RUNTIME)
+
 struct rpc_construct_nvme {
 	char *name;
 	char *trtype;

scripts/rpc.py

@@ -238,6 +238,18 @@ if __name__ == "__main__":
                    help='How often the admin queue is polled for asynchronous events', type=int)
     p.set_defaults(func=set_bdev_nvme_options)
 
+    @call_cmd
+    def set_bdev_nvme_hotplug(args):
+        rpc.bdev.set_bdev_nvme_hotplug(args.client, enable=args.enable, period_us=args.period_us)
+
+    p = subparsers.add_parser('set_bdev_nvme_hotplug',
+                              help='Set hotplug options for bdev nvme type.')
+    p.add_argument('-d', '--disable', dest='enable', default=False, action='store_false', help="Disable hotplug (default)")
+    p.add_argument('-e', '--enable', dest='enable', action='store_true', help="Enable hotplug")
+    p.add_argument('-r', '--period-us',
+                   help='How often the hotplug is processed for insert and remove events', type=int)
+    p.set_defaults(func=set_bdev_nvme_hotplug)
+
     @call_cmd
     def construct_nvme_bdev(args):
         print_array(rpc.bdev.construct_nvme_bdev(args.client,

scripts/rpc/bdev.py

@@ -173,6 +173,21 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry
     return client.call('set_bdev_nvme_options', params)
 
 
+def set_bdev_nvme_hotplug(client, enable, period_us=None):
+    """Set options for the bdev nvme. This is startup command.
+
+    Args:
+       enable: True to enable hotplug, False to disable.
+       period_us: how often the hotplug is processed for insert and remove events. Set 0 to reset to default. (optional)
+    """
+    params = {'enable': enable}
+
+    if period_us:
+        params['period_us'] = period_us
+
+    return client.call('set_bdev_nvme_hotplug', params)
+
+
 def construct_nvme_bdev(client, name, trtype, traddr, adrfam=None, trsvcid=None, subnqn=None):
     """Construct NVMe namespace block device.
 

test/json_config/clear_config.py

@@ -76,6 +76,9 @@ def clear_bdev_subsystem(args, bdev_config):
         if destroy_method:
             args.client.call(destroy_method, {bdev_name_key: bdev_name})
 
+    ''' Disable and reset hotplug '''
+    rpc.bdev.set_bdev_nvme_hotplug(args.client, False)
+
 
 def get_nvmf_destroy_method(nvmf):
     destroy_method_map = {'construct_nvmf_subsystem': "delete_nvmf_subsystem",

test/json_config/config_filter.py

@@ -3,13 +3,14 @@ import json
 import argparse
 
 
-def filter_methods(filename, do_remove_startup_rpcs):
+def filter_methods(filename, do_remove_global_rpcs):
-    startup_rpcs = [
+    global_rpcs = [
         'set_iscsi_options',
         'set_nvmf_target_config',
         'set_nvmf_target_options',
         'set_bdev_options',
-        'set_bdev_nvme_options'
+        'set_bdev_nvme_options',
+        'set_bdev_nvme_hotplug',
     ]
 
     with open(filename) as json_file:
@@ -20,8 +21,8 @@ def filter_methods(filename, do_remove_startup_rpcs):
             s_config = []
             for config in s['config']:
                 m_name = config['method']
-                is_startup_rpc = m_name in startup_rpcs
+                is_global_rpc = m_name in global_rpcs
-                if do_remove_startup_rpcs != is_startup_rpc:
+                if do_remove_global_rpcs != is_global_rpc:
                     s_config.append(config)
         else:
             s_config = None