From 344249069d80cb1ef7fe0a964ab6712ccc6951dd Mon Sep 17 00:00:00 2001 From: Krzysztof Karas Date: Mon, 17 Oct 2022 11:49:29 +0000 Subject: [PATCH] event: add runtime cpu lock configuration Allow CPU core locks to be enabled and disabled during runtime. This feature will be useful in cases like SPDK hot upgrade, where locking should be disabled temporarily. Change-Id: I9bc7292fd964abffc7214d074d191f38b13583c3 Signed-off-by: Krzysztof Karas Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15031 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- CHANGELOG.md | 3 ++ doc/jsonrpc.md | 76 +++++++++++++++++++++++++++++++++++++ lib/event/app.c | 50 ++++++++++++++++++++++++ python/spdk/rpc/__init__.py | 10 +++++ scripts/rpc.py | 14 +++++++ 5 files changed, 153 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21dd549ea..cc19a289a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ SPDK application instances. Added a command line switch to disable CPU locks on SPDK startup. +Added RPCs framework_enable_cpumask_locks and framework_disable_cpumask_locks to enable +and disable CPU core locks in runtime. + ## v22.09 ### accel diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index f46c5cb1a..4aab0bef9 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -797,6 +797,82 @@ Example response: } ~~~ +### framework_enable_cpumask_locks + +Enable CPU core lock files to block multiple SPDK applications from running on the same cpumask. +The CPU core locks are enabled by default, unless user specified `--disable-cpumask-locks` command +line option when launching SPDK. + +This RPC may be called after locks have already been enabled, with no effect and no error response. + +#### Parameters + +This method has no parameters. + +#### Response + +true on success + +#### Example + +Example request: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_enable_cpumask_locks" +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### framework_disable_cpumask_locks + +Disable CPU core lock files. The locks can also be disabled during startup, when +user specifies `--disable-cpumask-locks` command line option during SPDK launch. + +This RPC may be called after locks have already been disabled, with no effect and no error +response. + +#### Parameters + +This method has no parameters. + +#### Response + +true on success + +#### Example + +Example request: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_disable_cpumask_locks" +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + ### thread_get_stats {#rpc_thread_get_stats} Retrieve current statistics of all the threads. diff --git a/lib/event/app.c b/lib/event/app.c index 2aebcd0f4..3252a3107 100644 --- a/lib/event/app.c +++ b/lib/event/app.c @@ -525,6 +525,7 @@ unclaim_cpu_cores(void) if (g_core_locks[i] != -1) { snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", i); close(g_core_locks[i]); + g_core_locks[i] = -1; unlink(core_name); } } @@ -546,6 +547,11 @@ claim_cpu_cores(uint32_t *failed_core) }; SPDK_ENV_FOREACH_CORE(core) { + if (g_core_locks[core] != -1) { + /* If this core is locked already, do not try lock it again. */ + continue; + } + snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", core); core_fd = open(core_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (core_fd == -1) { @@ -1211,3 +1217,47 @@ rpc_framework_wait_init(struct spdk_jsonrpc_request *request, } SPDK_RPC_REGISTER("framework_wait_init", rpc_framework_wait_init, SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) + +static void +rpc_framework_disable_cpumask_locks(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + if (params != NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "framework_disable_cpumask_locks" + "requires no arguments"); + return; + } + + unclaim_cpu_cores(); + spdk_jsonrpc_send_bool_response(request, true); +} +SPDK_RPC_REGISTER("framework_disable_cpumask_locks", rpc_framework_disable_cpumask_locks, + SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) + +static void +rpc_framework_enable_cpumask_locks(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + char msg[128]; + int rc; + uint32_t failed_core; + + if (params != NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "framework_enable_cpumask_locks" + "requires no arguments"); + return; + } + + rc = claim_cpu_cores(&failed_core); + if (rc) { + snprintf(msg, sizeof(msg), "Failed to claim CPU core: %" PRIu32, failed_core); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} +SPDK_RPC_REGISTER("framework_enable_cpumask_locks", rpc_framework_enable_cpumask_locks, + SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) diff --git a/python/spdk/rpc/__init__.py b/python/spdk/rpc/__init__.py index 668598ed5..eb2ba7737 100644 --- a/python/spdk/rpc/__init__.py +++ b/python/spdk/rpc/__init__.py @@ -39,6 +39,16 @@ def framework_wait_init(client): return client.call('framework_wait_init') +def framework_disable_cpumask_locks(client): + """ Disable CPU core lock files.""" + return client.call('framework_disable_cpumask_locks') + + +def framework_enable_cpumask_locks(client): + """ Enable CPU core lock files.""" + return client.call('framework_enable_cpumask_locks') + + def rpc_get_methods(client, current=None, include_aliases=None): """Get list of supported RPC methods. Args: diff --git a/scripts/rpc.py b/scripts/rpc.py index 92262fb94..950ee99a2 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -179,6 +179,20 @@ if __name__ == "__main__": 'framework_get_scheduler', help='Display currently set scheduler and its properties.') p.set_defaults(func=framework_get_scheduler) + def framework_disable_cpumask_locks(args): + rpc.framework_disable_cpumask_locks(args.client) + + p = subparsers.add_parser('framework_disable_cpumask_locks', + help='Disable CPU core lock files.') + p.set_defaults(func=framework_disable_cpumask_locks) + + def framework_enable_cpumask_locks(args): + rpc.framework_enable_cpumask_locks(args.client) + + p = subparsers.add_parser('framework_enable_cpumask_locks', + help='Enable CPU core lock files.') + p.set_defaults(func=framework_enable_cpumask_locks) + # bdev def bdev_set_options(args): rpc.bdev.bdev_set_options(args.client,