diff --git a/CHANGELOG.md b/CHANGELOG.md index 21dd549ea..cc19a289a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ SPDK application instances. Added a command line switch to disable CPU locks on SPDK startup. +Added RPCs framework_enable_cpumask_locks and framework_disable_cpumask_locks to enable +and disable CPU core locks in runtime. + ## v22.09 ### accel diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index f46c5cb1a..4aab0bef9 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -797,6 +797,82 @@ Example response: } ~~~ +### framework_enable_cpumask_locks + +Enable CPU core lock files to block multiple SPDK applications from running on the same cpumask. +The CPU core locks are enabled by default, unless user specified `--disable-cpumask-locks` command +line option when launching SPDK. + +This RPC may be called after locks have already been enabled, with no effect and no error response. + +#### Parameters + +This method has no parameters. + +#### Response + +true on success + +#### Example + +Example request: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_enable_cpumask_locks" +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + +### framework_disable_cpumask_locks + +Disable CPU core lock files. The locks can also be disabled during startup, when +user specifies `--disable-cpumask-locks` command line option during SPDK launch. + +This RPC may be called after locks have already been disabled, with no effect and no error +response. + +#### Parameters + +This method has no parameters. + +#### Response + +true on success + +#### Example + +Example request: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "method": "framework_disable_cpumask_locks" +} +~~~ + +Example response: + +~~~json +{ + "jsonrpc": "2.0", + "id": 1, + "result": true +} +~~~ + ### thread_get_stats {#rpc_thread_get_stats} Retrieve current statistics of all the threads. diff --git a/lib/event/app.c b/lib/event/app.c index 2aebcd0f4..3252a3107 100644 --- a/lib/event/app.c +++ b/lib/event/app.c @@ -525,6 +525,7 @@ unclaim_cpu_cores(void) if (g_core_locks[i] != -1) { snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", i); close(g_core_locks[i]); + g_core_locks[i] = -1; unlink(core_name); } } @@ -546,6 +547,11 @@ claim_cpu_cores(uint32_t *failed_core) }; SPDK_ENV_FOREACH_CORE(core) { + if (g_core_locks[core] != -1) { + /* If this core is locked already, do not try lock it again. */ + continue; + } + snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", core); core_fd = open(core_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if (core_fd == -1) { @@ -1211,3 +1217,47 @@ rpc_framework_wait_init(struct spdk_jsonrpc_request *request, } SPDK_RPC_REGISTER("framework_wait_init", rpc_framework_wait_init, SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) + +static void +rpc_framework_disable_cpumask_locks(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + if (params != NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "framework_disable_cpumask_locks" + "requires no arguments"); + return; + } + + unclaim_cpu_cores(); + spdk_jsonrpc_send_bool_response(request, true); +} +SPDK_RPC_REGISTER("framework_disable_cpumask_locks", rpc_framework_disable_cpumask_locks, + SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) + +static void +rpc_framework_enable_cpumask_locks(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + char msg[128]; + int rc; + uint32_t failed_core; + + if (params != NULL) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "framework_enable_cpumask_locks" + "requires no arguments"); + return; + } + + rc = claim_cpu_cores(&failed_core); + if (rc) { + snprintf(msg, sizeof(msg), "Failed to claim CPU core: %" PRIu32, failed_core); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg); + return; + } + + spdk_jsonrpc_send_bool_response(request, true); +} +SPDK_RPC_REGISTER("framework_enable_cpumask_locks", rpc_framework_enable_cpumask_locks, + SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME) diff --git a/python/spdk/rpc/__init__.py b/python/spdk/rpc/__init__.py index 668598ed5..eb2ba7737 100644 --- a/python/spdk/rpc/__init__.py +++ b/python/spdk/rpc/__init__.py @@ -39,6 +39,16 @@ def framework_wait_init(client): return client.call('framework_wait_init') +def framework_disable_cpumask_locks(client): + """ Disable CPU core lock files.""" + return client.call('framework_disable_cpumask_locks') + + +def framework_enable_cpumask_locks(client): + """ Enable CPU core lock files.""" + return client.call('framework_enable_cpumask_locks') + + def rpc_get_methods(client, current=None, include_aliases=None): """Get list of supported RPC methods. Args: diff --git a/scripts/rpc.py b/scripts/rpc.py index 92262fb94..950ee99a2 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -179,6 +179,20 @@ if __name__ == "__main__": 'framework_get_scheduler', help='Display currently set scheduler and its properties.') p.set_defaults(func=framework_get_scheduler) + def framework_disable_cpumask_locks(args): + rpc.framework_disable_cpumask_locks(args.client) + + p = subparsers.add_parser('framework_disable_cpumask_locks', + help='Disable CPU core lock files.') + p.set_defaults(func=framework_disable_cpumask_locks) + + def framework_enable_cpumask_locks(args): + rpc.framework_enable_cpumask_locks(args.client) + + p = subparsers.add_parser('framework_enable_cpumask_locks', + help='Enable CPU core lock files.') + p.set_defaults(func=framework_enable_cpumask_locks) + # bdev def bdev_set_options(args): rpc.bdev.bdev_set_options(args.client,