diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 6deaaa16a..02d71b66e 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -934,6 +934,45 @@ Example response: } ~~~ +## set_bdev_nvme_hotplug {#rpc_set_bdev_nvme_hotplug} + +Change settings of the NVMe hotplug feature. If enabled, PCIe NVMe bdevs will be automatically discovered on insertion +and deleted on removal. + +### Parameters + +Name | Optional | Type | Description +----------------------- | -------- | ----------- | ----------- +enabled | Required | string | True to enable, false to disable +period_us | Optional | number | How often to poll for hot-insert and hot-remove events. Values: 0 - reset/use default or 1 to 10000000. + +### Example + +Example request: + +~~~ +request: +{ + "params": { + "enabled": true, + "period_us": 2000 + }, + "jsonrpc": "2.0", + "method": "set_bdev_nvme_hotplug", + "id": 1 +} +~~~ + +Example response: + +~~~ +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + ## construct_nvme_bdev {#rpc_construct_nvme_bdev} Construct @ref bdev_config_nvme diff --git a/lib/bdev/nvme/bdev_nvme.c b/lib/bdev/nvme/bdev_nvme.c index faf8bb75d..4bc40cec1 100644 --- a/lib/bdev/nvme/bdev_nvme.c +++ b/lib/bdev/nvme/bdev_nvme.c @@ -101,11 +101,13 @@ static struct spdk_bdev_nvme_opts g_opts = { .nvme_adminq_poll_period_us = 1000000ULL, }; -static bool g_bdev_nvme_init_done = false; +#define NVME_HOTPLUG_POLL_PERIOD_MAX 10000000ULL +#define NVME_HOTPLUG_POLL_PERIOD_DEFAULT 100000ULL static int g_hot_insert_nvme_controller_index = 0; -static int g_nvme_hotplug_poll_timeout_us = 0; +static uint64_t g_nvme_hotplug_poll_period_us = NVME_HOTPLUG_POLL_PERIOD_DEFAULT; static bool g_nvme_hotplug_enabled = false; +static struct spdk_thread *g_bdev_nvme_init_thread; static struct spdk_poller *g_hotplug_poller; static char *g_nvme_hostnqn = NULL; static pthread_mutex_t g_bdev_nvme_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -1048,7 +1050,7 @@ spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts) int spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) { - if (g_bdev_nvme_init_done) { + if (g_bdev_nvme_init_thread != NULL) { return -EPERM; } @@ -1056,6 +1058,55 @@ spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) return 0; } +struct set_nvme_hotplug_ctx { + uint64_t period_us; + bool enabled; + spdk_thread_fn fn; + void *fn_ctx; +}; + +static void +set_nvme_hotplug_period_cb(void *_ctx) +{ + struct set_nvme_hotplug_ctx *ctx = _ctx; + + spdk_poller_unregister(&g_hotplug_poller); + if (ctx->enabled) { + g_hotplug_poller = spdk_poller_register(bdev_nvme_hotplug, NULL, ctx->period_us); + } + + g_nvme_hotplug_poll_period_us = ctx->period_us; + g_nvme_hotplug_enabled = ctx->enabled; + if (ctx->fn) { + ctx->fn(ctx->fn_ctx); + } + + free(ctx); +} + +int +spdk_bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_thread_fn cb, void *cb_ctx) +{ + struct set_nvme_hotplug_ctx *ctx; + + if (enabled == true && !spdk_process_is_primary()) { + return -EPERM; + } + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return -ENOMEM; + } + + period_us = period_us == 0 ? NVME_HOTPLUG_POLL_PERIOD_DEFAULT : period_us; + ctx->period_us = spdk_min(period_us, NVME_HOTPLUG_POLL_PERIOD_MAX); + ctx->enabled = enabled; + ctx->fn = cb; + ctx->fn_ctx = cb_ctx; + + spdk_thread_send_msg(g_bdev_nvme_init_thread, set_nvme_hotplug_period_cb, ctx); + return 0; +} int spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, @@ -1156,8 +1207,10 @@ bdev_nvme_library_init(void) struct nvme_probe_ctx *probe_ctx = NULL; int retry_count; uint32_t local_nvme_num = 0; + int64_t hotplug_period; + bool hotplug_enabled = g_nvme_hotplug_enabled; - g_bdev_nvme_init_done = true; + g_bdev_nvme_init_thread = spdk_get_thread(); sp = spdk_conf_find_section(NULL, "Nvme"); if (sp == NULL) { @@ -1242,13 +1295,10 @@ bdev_nvme_library_init(void) } if (spdk_process_is_primary()) { - g_nvme_hotplug_enabled = spdk_conf_section_get_boolval(sp, "HotplugEnable", false); + hotplug_enabled = spdk_conf_section_get_boolval(sp, "HotplugEnable", false); } - g_nvme_hotplug_poll_timeout_us = spdk_conf_section_get_intval(sp, "HotplugPollRate"); - if (g_nvme_hotplug_poll_timeout_us <= 0 || g_nvme_hotplug_poll_timeout_us > 100000) { - g_nvme_hotplug_poll_timeout_us = 100000; - } + hotplug_period = spdk_conf_section_get_intval(sp, "HotplugPollRate"); g_nvme_hostnqn = spdk_conf_section_get_val(sp, "HostNQN"); probe_ctx->hostnqn = g_nvme_hostnqn; @@ -1335,15 +1385,13 @@ bdev_nvme_library_init(void) } } - if (g_nvme_hotplug_enabled) { - g_hotplug_poller = spdk_poller_register(bdev_nvme_hotplug, NULL, - g_nvme_hotplug_poll_timeout_us); + rc = spdk_bdev_nvme_set_hotplug(hotplug_enabled, hotplug_period, NULL, NULL); + if (rc) { + SPDK_ERRLOG("Failed to setup hotplug (%d): %s", rc, spdk_strerror(rc)); + rc = -1; } - end: - if (rc == 0) { - spdk_nvme_retry_count = g_opts.retry_count; - } + spdk_nvme_retry_count = g_opts.retry_count; free(probe_ctx); return rc; @@ -1352,9 +1400,7 @@ end: static void bdev_nvme_library_fini(void) { - if (g_nvme_hotplug_enabled) { - spdk_poller_unregister(&g_hotplug_poller); - } + spdk_poller_unregister(&g_hotplug_poller); } static int @@ -1680,7 +1726,7 @@ bdev_nvme_get_spdk_running_config(FILE *fp) fprintf(fp, "\n" "# Set how often the hotplug is processed for insert and remove events." "# Units in microseconds.\n"); - fprintf(fp, "HotplugPollRate %d\n", g_nvme_hotplug_poll_timeout_us); + fprintf(fp, "HotplugPollRate %"PRIu64"\n", g_nvme_hotplug_poll_period_us); if (g_nvme_hostnqn) { fprintf(fp, "HostNQN %s\n", g_nvme_hostnqn); } @@ -1748,6 +1794,19 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w) spdk_json_write_object_end(w); } + /* Dump as last parameter to give all NVMe bdevs chance to be constructed + * before enabling hotplug poller. + */ + spdk_json_write_object_begin(w); + spdk_json_write_named_string(w, "method", "set_bdev_nvme_hotplug"); + + spdk_json_write_named_object_begin(w, "params"); + spdk_json_write_named_uint64(w, "period_us", g_nvme_hotplug_poll_period_us); + spdk_json_write_named_bool(w, "enable", g_nvme_hotplug_enabled); + spdk_json_write_object_end(w); + + spdk_json_write_object_end(w); + pthread_mutex_unlock(&g_bdev_nvme_mutex); return 0; } diff --git a/lib/bdev/nvme/bdev_nvme.h b/lib/bdev/nvme/bdev_nvme.h index bd4ee1835..f48c696b0 100644 --- a/lib/bdev/nvme/bdev_nvme.h +++ b/lib/bdev/nvme/bdev_nvme.h @@ -85,6 +85,7 @@ struct nvme_bdev { void spdk_bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts); int spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts); +int spdk_bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_thread_fn cb, void *cb_ctx); int spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, const char *base_name, diff --git a/lib/bdev/nvme/bdev_nvme_rpc.c b/lib/bdev/nvme/bdev_nvme_rpc.c index 382f49923..a9154d4e5 100644 --- a/lib/bdev/nvme/bdev_nvme_rpc.c +++ b/lib/bdev/nvme/bdev_nvme_rpc.c @@ -109,6 +109,54 @@ invalid: } SPDK_RPC_REGISTER("set_bdev_nvme_options", spdk_rpc_set_bdev_nvme_options, SPDK_RPC_STARTUP) +struct rpc_bdev_nvme_hotplug { + bool enabled; + uint64_t period_us; +}; + +static const struct spdk_json_object_decoder rpc_bdev_nvme_hotplug_decoders[] = { + {"enable", offsetof(struct rpc_bdev_nvme_hotplug, enabled), spdk_json_decode_bool, false}, + {"period_us", offsetof(struct rpc_bdev_nvme_hotplug, period_us), spdk_json_decode_uint64, true}, +}; + +static void +rpc_set_bdev_nvme_hotplug_done(void *ctx) +{ + struct spdk_jsonrpc_request *request = ctx; + struct spdk_json_write_ctx *w = spdk_jsonrpc_begin_result(request); + + if (w != NULL) { + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); + } +} + +static void +spdk_rpc_set_bdev_nvme_hotplug(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_bdev_nvme_hotplug req = {false, 0}; + int rc; + + if (spdk_json_decode_object(params, rpc_bdev_nvme_hotplug_decoders, + SPDK_COUNTOF(rpc_bdev_nvme_hotplug_decoders), &req)) { + SPDK_ERRLOG("spdk_json_decode_object failed\n"); + rc = -EINVAL; + goto invalid; + } + + rc = spdk_bdev_nvme_set_hotplug(req.enabled, req.period_us, rpc_set_bdev_nvme_hotplug_done, + request); + if (rc) { + goto invalid; + } + + return; +invalid: + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, spdk_strerror(-rc)); +} +SPDK_RPC_REGISTER("set_bdev_nvme_hotplug", spdk_rpc_set_bdev_nvme_hotplug, SPDK_RPC_RUNTIME) + struct rpc_construct_nvme { char *name; char *trtype; diff --git a/scripts/rpc.py b/scripts/rpc.py index 60f6b056b..24dc31a70 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -238,6 +238,18 @@ if __name__ == "__main__": help='How often the admin queue is polled for asynchronous events', type=int) p.set_defaults(func=set_bdev_nvme_options) + @call_cmd + def set_bdev_nvme_hotplug(args): + rpc.bdev.set_bdev_nvme_hotplug(args.client, enable=args.enable, period_us=args.period_us) + + p = subparsers.add_parser('set_bdev_nvme_hotplug', + help='Set hotplug options for bdev nvme type.') + p.add_argument('-d', '--disable', dest='enable', default=False, action='store_false', help="Disable hotplug (default)") + p.add_argument('-e', '--enable', dest='enable', action='store_true', help="Enable hotplug") + p.add_argument('-r', '--period-us', + help='How often the hotplug is processed for insert and remove events', type=int) + p.set_defaults(func=set_bdev_nvme_hotplug) + @call_cmd def construct_nvme_bdev(args): print_array(rpc.bdev.construct_nvme_bdev(args.client, diff --git a/scripts/rpc/bdev.py b/scripts/rpc/bdev.py index 3132cfabc..ada9448ea 100755 --- a/scripts/rpc/bdev.py +++ b/scripts/rpc/bdev.py @@ -173,6 +173,21 @@ def set_bdev_nvme_options(client, action_on_timeout=None, timeout_us=None, retry return client.call('set_bdev_nvme_options', params) +def set_bdev_nvme_hotplug(client, enable, period_us=None): + """Set options for the bdev nvme. This is startup command. + + Args: + enable: True to enable hotplug, False to disable. + period_us: how often the hotplug is processed for insert and remove events. Set 0 to reset to default. (optional) + """ + params = {'enable': enable} + + if period_us: + params['period_us'] = period_us + + return client.call('set_bdev_nvme_hotplug', params) + + def construct_nvme_bdev(client, name, trtype, traddr, adrfam=None, trsvcid=None, subnqn=None): """Construct NVMe namespace block device. diff --git a/test/json_config/clear_config.py b/test/json_config/clear_config.py index 05ef42911..bb83f0072 100755 --- a/test/json_config/clear_config.py +++ b/test/json_config/clear_config.py @@ -76,6 +76,9 @@ def clear_bdev_subsystem(args, bdev_config): if destroy_method: args.client.call(destroy_method, {bdev_name_key: bdev_name}) + ''' Disable and reset hotplug ''' + rpc.bdev.set_bdev_nvme_hotplug(args.client, False) + def get_nvmf_destroy_method(nvmf): destroy_method_map = {'construct_nvmf_subsystem': "delete_nvmf_subsystem", diff --git a/test/json_config/config_filter.py b/test/json_config/config_filter.py index deead102c..41bef46a7 100755 --- a/test/json_config/config_filter.py +++ b/test/json_config/config_filter.py @@ -3,13 +3,14 @@ import json import argparse -def filter_methods(filename, do_remove_startup_rpcs): - startup_rpcs = [ +def filter_methods(filename, do_remove_global_rpcs): + global_rpcs = [ 'set_iscsi_options', 'set_nvmf_target_config', 'set_nvmf_target_options', 'set_bdev_options', - 'set_bdev_nvme_options' + 'set_bdev_nvme_options', + 'set_bdev_nvme_hotplug', ] with open(filename) as json_file: @@ -20,8 +21,8 @@ def filter_methods(filename, do_remove_startup_rpcs): s_config = [] for config in s['config']: m_name = config['method'] - is_startup_rpc = m_name in startup_rpcs - if do_remove_startup_rpcs != is_startup_rpc: + is_global_rpc = m_name in global_rpcs + if do_remove_global_rpcs != is_global_rpc: s_config.append(config) else: s_config = None