event: add runtime cpu lock configuration

Allow CPU core locks to be enabled and disabled
during runtime. This feature will be useful
in cases like SPDK hot upgrade, where
locking should be disabled temporarily.

Change-Id: I9bc7292fd964abffc7214d074d191f38b13583c3
Signed-off-by: Krzysztof Karas <krzysztof.karas@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15031
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>

CHANGELOG.md

@@ -34,6 +34,9 @@ SPDK application instances.
 
 Added a command line switch to disable CPU locks on SPDK startup.
 
+Added RPCs framework_enable_cpumask_locks and framework_disable_cpumask_locks to enable
+and disable CPU core locks in runtime.
+
 ## v22.09
 
 ### accel

doc/jsonrpc.md

@@ -797,6 +797,82 @@ Example response:
 }
 ~~~
 
+### framework_enable_cpumask_locks
+
+Enable CPU core lock files to block multiple SPDK applications from running on the same cpumask.
+The CPU core locks are enabled by default, unless user specified `--disable-cpumask-locks` command
+line option when launching SPDK.
+
+This RPC may be called after locks have already been enabled, with no effect and no error response.
+
+#### Parameters
+
+This method has no parameters.
+
+#### Response
+
+true on success
+
+#### Example
+
+Example request:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "framework_enable_cpumask_locks"
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~
+
+### framework_disable_cpumask_locks
+
+Disable CPU core lock files. The locks can also be disabled during startup, when
+user specifies `--disable-cpumask-locks` command line option during SPDK launch.
+
+This RPC may be called after locks have already been disabled, with no effect and no error
+response.
+
+#### Parameters
+
+This method has no parameters.
+
+#### Response
+
+true on success
+
+#### Example
+
+Example request:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "framework_disable_cpumask_locks"
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~
+
 ### thread_get_stats {#rpc_thread_get_stats}
 
 Retrieve current statistics of all the threads.

lib/event/app.c

@@ -525,6 +525,7 @@ unclaim_cpu_cores(void)
 		if (g_core_locks[i] != -1) {
 			snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", i);
 			close(g_core_locks[i]);
+			g_core_locks[i] = -1;
 			unlink(core_name);
 		}
 	}
@@ -546,6 +547,11 @@ claim_cpu_cores(uint32_t *failed_core)
 	};
 
 	SPDK_ENV_FOREACH_CORE(core) {
+		if (g_core_locks[core] != -1) {
+			/* If this core is locked already, do not try lock it again. */
+			continue;
+		}
+
 		snprintf(core_name, sizeof(core_name), "/var/tmp/spdk_cpu_lock_%03d", core);
 		core_fd = open(core_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
 		if (core_fd == -1) {
@@ -1211,3 +1217,47 @@ rpc_framework_wait_init(struct spdk_jsonrpc_request *request,
 }
 SPDK_RPC_REGISTER("framework_wait_init", rpc_framework_wait_init,
 		  SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME)
+
+static void
+rpc_framework_disable_cpumask_locks(struct spdk_jsonrpc_request *request,
+				    const struct spdk_json_val *params)
+{
+	if (params != NULL) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "framework_disable_cpumask_locks"
+						 "requires no arguments");
+		return;
+	}
+
+	unclaim_cpu_cores();
+	spdk_jsonrpc_send_bool_response(request, true);
+}
+SPDK_RPC_REGISTER("framework_disable_cpumask_locks", rpc_framework_disable_cpumask_locks,
+		  SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME)
+
+static void
+rpc_framework_enable_cpumask_locks(struct spdk_jsonrpc_request *request,
+				   const struct spdk_json_val *params)
+{
+	char msg[128];
+	int rc;
+	uint32_t failed_core;
+
+	if (params != NULL) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "framework_enable_cpumask_locks"
+						 "requires no arguments");
+		return;
+	}
+
+	rc = claim_cpu_cores(&failed_core);
+	if (rc) {
+		snprintf(msg, sizeof(msg), "Failed to claim CPU core: %" PRIu32, failed_core);
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, msg);
+		return;
+	}
+
+	spdk_jsonrpc_send_bool_response(request, true);
+}
+SPDK_RPC_REGISTER("framework_enable_cpumask_locks", rpc_framework_enable_cpumask_locks,
+		  SPDK_RPC_STARTUP | SPDK_RPC_RUNTIME)

python/spdk/rpc/__init__.py

@@ -39,6 +39,16 @@ def framework_wait_init(client):
     return client.call('framework_wait_init')
 
 
+def framework_disable_cpumask_locks(client):
+    """ Disable CPU core lock files."""
+    return client.call('framework_disable_cpumask_locks')
+
+
+def framework_enable_cpumask_locks(client):
+    """ Enable CPU core lock files."""
+    return client.call('framework_enable_cpumask_locks')
+
+
 def rpc_get_methods(client, current=None, include_aliases=None):
     """Get list of supported RPC methods.
     Args:

scripts/rpc.py

@@ -179,6 +179,20 @@ if __name__ == "__main__":
         'framework_get_scheduler', help='Display currently set scheduler and its properties.')
     p.set_defaults(func=framework_get_scheduler)
 
+    def framework_disable_cpumask_locks(args):
+        rpc.framework_disable_cpumask_locks(args.client)
+
+    p = subparsers.add_parser('framework_disable_cpumask_locks',
+                              help='Disable CPU core lock files.')
+    p.set_defaults(func=framework_disable_cpumask_locks)
+
+    def framework_enable_cpumask_locks(args):
+        rpc.framework_enable_cpumask_locks(args.client)
+
+    p = subparsers.add_parser('framework_enable_cpumask_locks',
+                              help='Enable CPU core lock files.')
+    p.set_defaults(func=framework_enable_cpumask_locks)
+
     # bdev
     def bdev_set_options(args):
         rpc.bdev.bdev_set_options(args.client,