lib/accel: add RPC to enable override of opcode to engine

Docs explaining how to use the RPC are in the next patch in the
series.

Signed-off-by: paul luse <paul.e.luse@intel.com>
Change-Id: I7dab8fdbeb90cdfde8b3e916ed6d19930ad36e66
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12848
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

CHANGELOG.md

@@ -39,6 +39,9 @@ writes (read-modify-write) are not.
 Added a new runtime RPC `accel_get_opc_assignments` to get a list of current opcode to engine
 assignements.
 
+Added a new startup RPC `accel_assign_opc` to assign/override a specific opcode to
+an engine.
+
 ## v22.05
 
 ### sock

doc/jsonrpc.md

@@ -1587,6 +1587,43 @@ Example response:
 }
 ~~~
 
+### accel_assign_opc {#rpc_accel_assign_opc}
+
+Manually assign an operation to an engine.
+
+#### Parameters
+
+Name                    | Optional | Type        | Description
+----------------------- | -------- | ----------- | -----------------
+opname                  | Required | string      | name of operation
+engine                  | Required | string      | name of engine
+
+#### Example
+
+Example request:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "method": "accel_assign_opc",
+  "id": 1,
+  "params": {
+    "opanme": "copy",
+    "engine": "software"
+  }
+}
+~~~
+
+Example response:
+
+~~~json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": true
+}
+~~~
+
 ### dsa_scan_accel_engine {#rpc_dsa_scan_accel_engine}
 
 Set config and enable dsa accel engine offload.

include/spdk/accel_engine.h

@@ -275,6 +275,19 @@ int spdk_accel_submit_decompress(struct spdk_io_channel *ch, void *dst, void *sr
  */
 int spdk_accel_get_opc_engine_name(enum accel_opcode opcode, const char **engine_name);
 
+/**
+ * Override the assignment of an opcode to an engine.
+ *
+ * \param opcode Accel Framework Opcode enum value. Valid codes can be retrieved using
+ * `accel_get_opc_assignments` or `spdk_accel_get_opc_name`.
+ * \param name Name of the engine to assign. Valid engine names may be retrieved
+ * with `spdk_accel_get_opc_engine_name`
+ *
+ * \return 0 if a valid opcode name was provided. -EINVAL for invalid opcode
+ *  or if the framework has started (cannot change engines after startup)
+ */
+int spdk_accel_assign_opc(enum accel_opcode opcode, const char *name);
+
 struct spdk_json_write_ctx;
 
 /**

lib/accel/accel_engine.c

@@ -35,6 +35,7 @@ static size_t g_max_accel_module_size = 0;
 static struct spdk_accel_module_if *g_accel_engine_module = NULL;
 static spdk_accel_fini_cb g_fini_cb_fn = NULL;
 static void *g_fini_cb_arg = NULL;
+static bool g_engine_started = false;
 
 /* Global list of registered accelerator modules */
 static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =
@@ -46,6 +47,7 @@ static TAILQ_HEAD(, spdk_accel_engine) g_engine_list =
 
 /* Global array mapping capabilities to engines */
 static struct spdk_accel_engine *g_engines_opc[ACCEL_OPC_LAST] = {};
+static char *g_engines_opc_override[ACCEL_OPC_LAST] = {};
 
 static int sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task);
 
@@ -87,6 +89,25 @@ _accel_for_each_engine(struct engine_info *info, _accel_for_each_engine_fn fn)
 	}
 }
 
+int
+spdk_accel_assign_opc(enum accel_opcode opcode, const char *name)
+{
+	if (g_engine_started == true) {
+		/* we don't allow re-assignment once things have started */
+		return -EINVAL;
+	}
+
+	if (opcode >= ACCEL_OPC_LAST) {
+		/* invalid opcode */
+		return -EINVAL;
+	}
+
+	/* engine selection will be validated after the framework starts. */
+	g_engines_opc_override[opcode] = strdup(name);
+
+	return 0;
+}
+
 static struct spdk_accel_engine *
 _engine_find_by_name(const char *name)
 {
@@ -585,6 +606,7 @@ spdk_accel_engine_initialize(void)
 	enum accel_opcode op;
 	struct spdk_accel_engine *accel_engine = NULL;
 
+	g_engine_started = true;
 	accel_engine_module_initialize();
 
 	/* Create our priority global map of opcodes to engines, we populate starting
@@ -601,6 +623,23 @@ spdk_accel_engine_initialize(void)
 			}
 		}
 	}
+
+	/* Now lets check for overrides and apply all that exist */
+	for (op = 0; op < ACCEL_OPC_LAST; op++) {
+		if (g_engines_opc_override[op] != NULL) {
+			accel_engine = _engine_find_by_name(g_engines_opc_override[op]);
+			if (accel_engine == NULL) {
+				SPDK_ERRLOG("Invalid engine name of %s\n", g_engines_opc_override[op]);
+				return -EINVAL;
+			}
+			if (accel_engine->supports_opcode(op) == false) {
+				SPDK_ERRLOG("Engine %s does not support op code %d\n", accel_engine->name, op);
+				return -EINVAL;
+			}
+			g_engines_opc[op] = accel_engine;
+		}
+	}
+
 #ifdef DEBUG
 	for (op = 0; op < ACCEL_OPC_LAST; op++) {
 		assert(g_engines_opc[op] != NULL);
@@ -668,11 +707,19 @@ spdk_accel_engine_module_finish(void)
 void
 spdk_accel_engine_finish(spdk_accel_fini_cb cb_fn, void *cb_arg)
 {
+	enum accel_opcode op;
+
 	assert(cb_fn != NULL);
 
 	g_fini_cb_fn = cb_fn;
 	g_fini_cb_arg = cb_arg;
 
+	for (op = 0; op < ACCEL_OPC_LAST; op++) {
+		if (g_engines_opc_override[op] != NULL) {
+			free(g_engines_opc_override[op]);
+		}
+	}
+
 	spdk_io_device_unregister(&spdk_accel_module_list, NULL);
 	spdk_accel_engine_module_finish();
 }

lib/accel/accel_engine_rpc.c

@@ -122,3 +122,68 @@ rpc_accel_get_engine_info(struct spdk_jsonrpc_request *request,
 }
 SPDK_RPC_REGISTER("accel_get_engine_info", rpc_accel_get_engine_info,
 		  SPDK_RPC_RUNTIME)
+
+struct rpc_accel_assign_opc {
+	char *opname;
+	char *engine;
+};
+
+static const struct spdk_json_object_decoder rpc_accel_assign_opc_decoders[] = {
+	{"opname", offsetof(struct rpc_accel_assign_opc, opname), spdk_json_decode_string},
+	{"engine", offsetof(struct rpc_accel_assign_opc, engine), spdk_json_decode_string},
+};
+
+static void
+free_accel_assign_opc(struct rpc_accel_assign_opc *r)
+{
+	free(r->opname);
+	free(r->engine);
+}
+
+static void
+rpc_accel_assign_opc(struct spdk_jsonrpc_request *request,
+		     const struct spdk_json_val *params)
+{
+	struct rpc_accel_assign_opc req = {};
+	enum accel_opcode opcode;
+	bool found = false;
+	int rc;
+
+	if (spdk_json_decode_object(params, rpc_accel_assign_opc_decoders,
+				    SPDK_COUNTOF(rpc_accel_assign_opc_decoders),
+				    &req)) {
+		SPDK_DEBUGLOG(accel, "spdk_json_decode_object failed\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_PARSE_ERROR,
+						 "spdk_json_decode_object failed");
+		goto cleanup;
+	}
+
+	for (opcode = 0; opcode < ACCEL_OPC_LAST; opcode++) {
+		if (strcmp(g_opcode_strings[opcode], req.opname) == 0) {
+			found = true;
+			break;
+		}
+	}
+
+	if (found == false) {
+		SPDK_DEBUGLOG(accel, "Invalid operation name\n");
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "spdk_json_decode_object failed");
+		goto cleanup;
+	}
+
+	rc = spdk_accel_assign_opc(opcode, req.engine);
+	if (rc) {
+		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+						 "error assigning opcode");
+		goto cleanup;
+	}
+
+	SPDK_NOTICELOG("Operation %s will be assigned to engine %s\n", req.opname, req.engine);
+	spdk_jsonrpc_send_bool_response(request, true);
+
+cleanup:
+	free_accel_assign_opc(&req);
+
+}
+SPDK_RPC_REGISTER("accel_assign_opc", rpc_accel_assign_opc, SPDK_RPC_STARTUP)

lib/accel/spdk_accel.map

@@ -17,6 +17,7 @@
         spdk_accel_submit_compress;
         spdk_accel_submit_decompress;
 	spdk_accel_get_opc_engine_name;
+	spdk_accel_assign_opc;
 	spdk_accel_write_config_json;
 
 	# functions needed by modules

python/spdk/rpc/accel.py

@@ -7,4 +7,19 @@ def accel_get_opc_assignments(client):
 def accel_get_engine_info(client):
     """Get list of valid engine names and their operations.
     """
-    return client.call('accel_get_engine_info')
+    return client.call('accel_get_engine_names')
+
+
+def accel_assign_opc(client, opname, engine):
+    """Manually assign an operation to an engine.
+
+    Args:
+        opname: name of operation
+        engine: name of engine
+    """
+    params = {
+        'opname': opname,
+        'engine': engine,
+    }
+
+    return client.call('accel_assign_opc', params)

scripts/rpc.py

@@ -2575,6 +2575,14 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
     p = subparsers.add_parser('accel_get_engine_info', help='Get list of valid engine names and their operations.')
     p.set_defaults(func=accel_get_engine_info)
 
+    def accel_assign_opc(args):
+        rpc.accel.accel_assign_opc(args.client, opname=args.opname, engine=args.engine)
+
+    p = subparsers.add_parser('accel_assign_opc', help='Manually assign an operation to an engine.')
+    p.add_argument('-o', '--opname', help='opname')
+    p.add_argument('-e', '--engine', help='name of engine')
+    p.set_defaults(func=accel_assign_opc)
+
     # ioat
     def ioat_scan_accel_engine(args):
         rpc.ioat.ioat_scan_accel_engine(args.client)