copy/ioat: Add scan_ioat_copy_engine RPC

Add an new RPC, scan_ioat_copy_engine, to scan IOATs for copy engine
dynamically. This patch is to keep compatibility to current config file.

Change-Id: Id1378fcda04fc5a868e373acc076bc34eeca01ae
Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/411842
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/copy/ioat/Makefile

@@ -35,6 +35,6 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
 LIBNAME = copy_ioat
-C_SRCS = copy_engine_ioat.c
+C_SRCS = copy_engine_ioat.c copy_engine_ioat_rpc.c
 
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/copy/ioat/copy_engine_ioat.c

@@ -31,6 +31,8 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "copy_engine_ioat.h"
+
 #include "spdk/stdinc.h"
 
 #include "spdk_internal/copy_engine.h"
@@ -42,8 +44,6 @@
 #include "spdk/thread.h"
 #include "spdk/ioat.h"
 
-#define IOAT_MAX_CHANNELS		64
-
 static bool g_ioat_enable = false;
 
 struct ioat_probe_ctx {
@@ -278,14 +278,56 @@ attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *
 	TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
 }
 
-static int
-copy_engine_ioat_init(void)
+void
+copy_engine_ioat_enable_probe(void)
 {
-	struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Ioat");
-	const char *val, *pci_bdf;
-	int i;
+	g_ioat_enable = true;
+}
+
+static int
+copy_engine_ioat_add_whitelist_device(const char *pci_bdf)
+{
+	if (pci_bdf == NULL) {
+		return -1;
+	}
+
+	if (g_probe_ctx.num_whitelist_devices >= IOAT_MAX_CHANNELS) {
+		SPDK_ERRLOG("Ioat whitelist is full (max size is %d)\n",
+			    IOAT_MAX_CHANNELS);
+		return -1;
+	}
+
+	if (spdk_pci_addr_parse(&g_probe_ctx.whitelist[g_probe_ctx.num_whitelist_devices],
+				pci_bdf) < 0) {
+		SPDK_ERRLOG("Invalid address %s\n", pci_bdf);
+		return -1;
+	}
+
+	g_probe_ctx.num_whitelist_devices++;
+
+	return 0;
+}
+
+int
+copy_engine_ioat_add_whitelist_devices(const char *pci_bdfs[], size_t num_pci_bdfs)
+{
+	size_t i;
+
+	for (i = 0; i < num_pci_bdfs; i++) {
+		if (copy_engine_ioat_add_whitelist_device(pci_bdfs[i]) < 0) {
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static int
+copy_engine_ioat_read_config_file_params(struct spdk_conf_section *sp)
+{
+	int i;
+	char *val, *pci_bdf;
 
-	if (sp != NULL) {
 	if (spdk_conf_section_get_boolval(sp, "Enable", false)) {
 		g_ioat_enable = true;
 		/* Enable Ioat */
@@ -303,18 +345,32 @@ copy_engine_ioat_init(void)
 	}
 
 	/* Init the whitelist */
-		for (i = 0; i < IOAT_MAX_CHANNELS; i++) {
+	for (i = 0; ; i++) {
 		pci_bdf = spdk_conf_section_get_nmval(sp, "Whitelist", i, 0);
 		if (!pci_bdf) {
 			break;
 		}
 
-			if (spdk_pci_addr_parse(&g_probe_ctx.whitelist[g_probe_ctx.num_whitelist_devices],
-						pci_bdf) < 0) {
-				SPDK_ERRLOG("Invalid Ioat Whitelist address %s\n", pci_bdf);
+		if (copy_engine_ioat_add_whitelist_device(pci_bdf) < 0) {
 			return -1;
 		}
-			g_probe_ctx.num_whitelist_devices++;
+	}
+
+	return 0;
+}
+
+static int
+copy_engine_ioat_init(void)
+{
+	struct spdk_conf_section *sp;
+	int rc;
+
+	sp = spdk_conf_find_section(NULL, "Ioat");
+	if (sp != NULL) {
+		rc = copy_engine_ioat_read_config_file_params(sp);
+		if (rc != 0) {
+			SPDK_ERRLOG("copy_engine_ioat_read_config_file_params() failed\n");
+			return rc;
 		}
 	}
 

lib/copy/ioat/copy_engine_ioat.h

@@ -0,0 +1,44 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_COPY_ENGINE_IOAT_H
+#define SPDK_COPY_ENGINE_IOAT_H
+
+#include "spdk/stdinc.h"
+
+#define IOAT_MAX_CHANNELS	64
+
+int copy_engine_ioat_add_whitelist_devices(const char *pci_bdfs[], size_t num_pci_bdfs);
+void copy_engine_ioat_enable_probe(void);
+
+#endif /* SPDK_COPY_ENGINE_IOAT_H */

lib/copy/ioat/copy_engine_ioat_rpc.c

@@ -0,0 +1,118 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "copy_engine_ioat.h"
+
+#include "spdk/rpc.h"
+#include "spdk/util.h"
+#include "spdk/event.h"
+
+struct rpc_pci_whitelist {
+	size_t num_bdfs;
+	char *bdfs[IOAT_MAX_CHANNELS];
+};
+
+static int
+decode_rpc_pci_whitelist(const struct spdk_json_val *val, void *out)
+{
+	struct rpc_pci_whitelist *pci_whitelist = out;
+
+	return spdk_json_decode_array(val, spdk_json_decode_string, pci_whitelist->bdfs,
+				      IOAT_MAX_CHANNELS, &pci_whitelist->num_bdfs, sizeof(char *));
+}
+
+static void
+free_rpc_pci_whitelist(struct rpc_pci_whitelist *list)
+{
+	size_t i;
+
+	for (i = 0; i < list->num_bdfs; i++) {
+		free(list->bdfs[i]);
+	}
+}
+
+struct rpc_copy_engine_ioat {
+	struct rpc_pci_whitelist pci_whitelist;
+};
+
+static void
+free_rpc_copy_engine_ioat(struct rpc_copy_engine_ioat *p)
+{
+	free_rpc_pci_whitelist(&p->pci_whitelist);
+}
+
+static const struct spdk_json_object_decoder rpc_copy_engine_ioat_decoder[] = {
+	{"pci_whitelist", offsetof(struct rpc_copy_engine_ioat, pci_whitelist), decode_rpc_pci_whitelist},
+};
+
+static void
+spdk_rpc_scan_copy_engine_ioat(struct spdk_jsonrpc_request *request,
+			       const struct spdk_json_val *params)
+{
+	struct rpc_copy_engine_ioat req = {};
+	struct spdk_json_write_ctx *w;
+	int rc;
+
+	if (params != NULL) {
+		if (spdk_json_decode_object(params, rpc_copy_engine_ioat_decoder,
+					    SPDK_COUNTOF(rpc_copy_engine_ioat_decoder),
+					    &req)) {
+			free_rpc_copy_engine_ioat(&req);
+			SPDK_ERRLOG("spdk_json_decode_object() failed\n");
+			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+							 "Invalid parameters");
+			return;
+		}
+
+		rc = copy_engine_ioat_add_whitelist_devices((const char **)req.pci_whitelist.bdfs,
+				req.pci_whitelist.num_bdfs);
+		free_rpc_copy_engine_ioat(&req);
+		if (rc < 0) {
+			SPDK_ERRLOG("copy_engine_ioat_add_whitelist_devices() failed\n");
+			spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS,
+							 "Invalid parameters");
+			return;
+		}
+	}
+
+	copy_engine_ioat_enable_probe();
+
+	w = spdk_jsonrpc_begin_result(request);
+	if (w == NULL) {
+		return;
+	}
+
+	spdk_json_write_bool(w, true);
+	spdk_jsonrpc_end_result(request, w);
+}
+SPDK_RPC_REGISTER("scan_ioat_copy_engine", spdk_rpc_scan_copy_engine_ioat, SPDK_RPC_STARTUP)

scripts/rpc.py

@@ -1237,6 +1237,20 @@ if __name__ == "__main__":
     p.add_argument('name', help='Name for the bdev')
     p.set_defaults(func=construct_virtio_pci_blk_bdev)
 
+    # ioat
+    @call_cmd
+    def scan_ioat_copy_engine(args):
+        pci_whitelist = []
+        if args.pci_whitelist:
+            for w in args.pci_whitelist.strip().split(" "):
+                pci_whitelist.append(w)
+        rpc.ioat.scan_ioat_copy_engine(args.client, pci_whitelist)
+
+    p = subparsers.add_parser('scan_ioat_copy_engine', help='Set scan and enable IOAT copy engine offload.')
+    p.add_argument('-w', '--pci-whitelist', help="""Whitespace-separated list of PCI addresses in
+    domain:bus:device.function format or domain.bus.device.function format""")
+    p.set_defaults(func=scan_ioat_copy_engine)
+
     args = parser.parse_args()
 
     try:

scripts/rpc/__init__.py

@@ -3,6 +3,7 @@ import sys
 
 from . import app
 from . import bdev
+from . import ioat
 from . import iscsi
 from . import log
 from . import lvol

scripts/rpc/ioat.py

@@ -0,0 +1,12 @@
+def scan_ioat_copy_engine(client, pci_whitelist):
+    """Scan and enable IOAT copy engine.
+
+    Args:
+        pci_whitelist: Python list of PCI addresses in
+                       domain:bus:device.function format or
+                       domain.bus.device.function format
+    """
+    params = {}
+    if pci_whitelist:
+        params['pci_whitelist'] = pci_whitelist
+    return client.call('scan_ioat_copy_engine', params)