From ce2e68289b13d3a8b22439cecd9570407f5405a0 Mon Sep 17 00:00:00 2001 From: Shuhei Matsumoto Date: Wed, 13 Jun 2018 08:31:20 +0900 Subject: [PATCH] copy/ioat: Add scan_ioat_copy_engine RPC Add an new RPC, scan_ioat_copy_engine, to scan IOATs for copy engine dynamically. This patch is to keep compatibility to current config file. Change-Id: Id1378fcda04fc5a868e373acc076bc34eeca01ae Signed-off-by: Shuhei Matsumoto Reviewed-on: https://review.gerrithub.io/411842 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Jim Harris --- lib/copy/ioat/Makefile | 2 +- lib/copy/ioat/copy_engine_ioat.c | 124 +++++++++++++++++++-------- lib/copy/ioat/copy_engine_ioat.h | 44 ++++++++++ lib/copy/ioat/copy_engine_ioat_rpc.c | 118 +++++++++++++++++++++++++ scripts/rpc.py | 14 +++ scripts/rpc/__init__.py | 1 + scripts/rpc/ioat.py | 12 +++ 7 files changed, 280 insertions(+), 35 deletions(-) create mode 100644 lib/copy/ioat/copy_engine_ioat.h create mode 100644 lib/copy/ioat/copy_engine_ioat_rpc.c create mode 100644 scripts/rpc/ioat.py diff --git a/lib/copy/ioat/Makefile b/lib/copy/ioat/Makefile index dda418796..3d19e38f2 100644 --- a/lib/copy/ioat/Makefile +++ b/lib/copy/ioat/Makefile @@ -35,6 +35,6 @@ SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) include $(SPDK_ROOT_DIR)/mk/spdk.common.mk LIBNAME = copy_ioat -C_SRCS = copy_engine_ioat.c +C_SRCS = copy_engine_ioat.c copy_engine_ioat_rpc.c include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/lib/copy/ioat/copy_engine_ioat.c b/lib/copy/ioat/copy_engine_ioat.c index 6e1401cde..625521a5e 100644 --- a/lib/copy/ioat/copy_engine_ioat.c +++ b/lib/copy/ioat/copy_engine_ioat.c @@ -31,6 +31,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include "copy_engine_ioat.h" + #include "spdk/stdinc.h" #include "spdk_internal/copy_engine.h" @@ -42,8 +44,6 @@ #include "spdk/thread.h" #include "spdk/ioat.h" -#define IOAT_MAX_CHANNELS 64 - static bool g_ioat_enable = false; struct ioat_probe_ctx { @@ -278,43 +278,99 @@ attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan * TAILQ_INSERT_TAIL(&g_devices, dev, tailq); } +void +copy_engine_ioat_enable_probe(void) +{ + g_ioat_enable = true; +} + +static int +copy_engine_ioat_add_whitelist_device(const char *pci_bdf) +{ + if (pci_bdf == NULL) { + return -1; + } + + if (g_probe_ctx.num_whitelist_devices >= IOAT_MAX_CHANNELS) { + SPDK_ERRLOG("Ioat whitelist is full (max size is %d)\n", + IOAT_MAX_CHANNELS); + return -1; + } + + if (spdk_pci_addr_parse(&g_probe_ctx.whitelist[g_probe_ctx.num_whitelist_devices], + pci_bdf) < 0) { + SPDK_ERRLOG("Invalid address %s\n", pci_bdf); + return -1; + } + + g_probe_ctx.num_whitelist_devices++; + + return 0; +} + +int +copy_engine_ioat_add_whitelist_devices(const char *pci_bdfs[], size_t num_pci_bdfs) +{ + size_t i; + + for (i = 0; i < num_pci_bdfs; i++) { + if (copy_engine_ioat_add_whitelist_device(pci_bdfs[i]) < 0) { + return -1; + } + } + + return 0; +} + +static int +copy_engine_ioat_read_config_file_params(struct spdk_conf_section *sp) +{ + int i; + char *val, *pci_bdf; + + if (spdk_conf_section_get_boolval(sp, "Enable", false)) { + g_ioat_enable = true; + /* Enable Ioat */ + } + + val = spdk_conf_section_get_val(sp, "Disable"); + if (val != NULL) { + SPDK_WARNLOG("\"Disable\" option is deprecated and will be removed in a future release.\n"); + SPDK_WARNLOG("IOAT is now disabled by default. It may be enabled by \"Enable Yes\"\n"); + + if (g_ioat_enable && (strcasecmp(val, "Yes") == 0)) { + SPDK_ERRLOG("\"Enable Yes\" and \"Disable Yes\" cannot be set at the same time\n"); + return -1; + } + } + + /* Init the whitelist */ + for (i = 0; ; i++) { + pci_bdf = spdk_conf_section_get_nmval(sp, "Whitelist", i, 0); + if (!pci_bdf) { + break; + } + + if (copy_engine_ioat_add_whitelist_device(pci_bdf) < 0) { + return -1; + } + } + + return 0; +} + static int copy_engine_ioat_init(void) { - struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Ioat"); - const char *val, *pci_bdf; - int i; + struct spdk_conf_section *sp; + int rc; + sp = spdk_conf_find_section(NULL, "Ioat"); if (sp != NULL) { - if (spdk_conf_section_get_boolval(sp, "Enable", false)) { - g_ioat_enable = true; - /* Enable Ioat */ - } - - val = spdk_conf_section_get_val(sp, "Disable"); - if (val != NULL) { - SPDK_WARNLOG("\"Disable\" option is deprecated and will be removed in a future release.\n"); - SPDK_WARNLOG("IOAT is now disabled by default. It may be enabled by \"Enable Yes\"\n"); - - if (g_ioat_enable && (strcasecmp(val, "Yes") == 0)) { - SPDK_ERRLOG("\"Enable Yes\" and \"Disable Yes\" cannot be set at the same time\n"); - return -1; - } - } - - /* Init the whitelist */ - for (i = 0; i < IOAT_MAX_CHANNELS; i++) { - pci_bdf = spdk_conf_section_get_nmval(sp, "Whitelist", i, 0); - if (!pci_bdf) { - break; - } - - if (spdk_pci_addr_parse(&g_probe_ctx.whitelist[g_probe_ctx.num_whitelist_devices], - pci_bdf) < 0) { - SPDK_ERRLOG("Invalid Ioat Whitelist address %s\n", pci_bdf); - return -1; - } - g_probe_ctx.num_whitelist_devices++; + rc = copy_engine_ioat_read_config_file_params(sp); + if (rc != 0) { + SPDK_ERRLOG("copy_engine_ioat_read_config_file_params() failed\n"); + return rc; } } diff --git a/lib/copy/ioat/copy_engine_ioat.h b/lib/copy/ioat/copy_engine_ioat.h new file mode 100644 index 000000000..ae69fb2d0 --- /dev/null +++ b/lib/copy/ioat/copy_engine_ioat.h @@ -0,0 +1,44 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPDK_COPY_ENGINE_IOAT_H +#define SPDK_COPY_ENGINE_IOAT_H + +#include "spdk/stdinc.h" + +#define IOAT_MAX_CHANNELS 64 + +int copy_engine_ioat_add_whitelist_devices(const char *pci_bdfs[], size_t num_pci_bdfs); +void copy_engine_ioat_enable_probe(void); + +#endif /* SPDK_COPY_ENGINE_IOAT_H */ diff --git a/lib/copy/ioat/copy_engine_ioat_rpc.c b/lib/copy/ioat/copy_engine_ioat_rpc.c new file mode 100644 index 000000000..ae03fdb16 --- /dev/null +++ b/lib/copy/ioat/copy_engine_ioat_rpc.c @@ -0,0 +1,118 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "copy_engine_ioat.h" + +#include "spdk/rpc.h" +#include "spdk/util.h" +#include "spdk/event.h" + +struct rpc_pci_whitelist { + size_t num_bdfs; + char *bdfs[IOAT_MAX_CHANNELS]; +}; + +static int +decode_rpc_pci_whitelist(const struct spdk_json_val *val, void *out) +{ + struct rpc_pci_whitelist *pci_whitelist = out; + + return spdk_json_decode_array(val, spdk_json_decode_string, pci_whitelist->bdfs, + IOAT_MAX_CHANNELS, &pci_whitelist->num_bdfs, sizeof(char *)); +} + +static void +free_rpc_pci_whitelist(struct rpc_pci_whitelist *list) +{ + size_t i; + + for (i = 0; i < list->num_bdfs; i++) { + free(list->bdfs[i]); + } +} + +struct rpc_copy_engine_ioat { + struct rpc_pci_whitelist pci_whitelist; +}; + +static void +free_rpc_copy_engine_ioat(struct rpc_copy_engine_ioat *p) +{ + free_rpc_pci_whitelist(&p->pci_whitelist); +} + +static const struct spdk_json_object_decoder rpc_copy_engine_ioat_decoder[] = { + {"pci_whitelist", offsetof(struct rpc_copy_engine_ioat, pci_whitelist), decode_rpc_pci_whitelist}, +}; + +static void +spdk_rpc_scan_copy_engine_ioat(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct rpc_copy_engine_ioat req = {}; + struct spdk_json_write_ctx *w; + int rc; + + if (params != NULL) { + if (spdk_json_decode_object(params, rpc_copy_engine_ioat_decoder, + SPDK_COUNTOF(rpc_copy_engine_ioat_decoder), + &req)) { + free_rpc_copy_engine_ioat(&req); + SPDK_ERRLOG("spdk_json_decode_object() failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + return; + } + + rc = copy_engine_ioat_add_whitelist_devices((const char **)req.pci_whitelist.bdfs, + req.pci_whitelist.num_bdfs); + free_rpc_copy_engine_ioat(&req); + if (rc < 0) { + SPDK_ERRLOG("copy_engine_ioat_add_whitelist_devices() failed\n"); + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "Invalid parameters"); + return; + } + } + + copy_engine_ioat_enable_probe(); + + w = spdk_jsonrpc_begin_result(request); + if (w == NULL) { + return; + } + + spdk_json_write_bool(w, true); + spdk_jsonrpc_end_result(request, w); +} +SPDK_RPC_REGISTER("scan_ioat_copy_engine", spdk_rpc_scan_copy_engine_ioat, SPDK_RPC_STARTUP) diff --git a/scripts/rpc.py b/scripts/rpc.py index 3270d44d0..50f4fb42f 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -1237,6 +1237,20 @@ if __name__ == "__main__": p.add_argument('name', help='Name for the bdev') p.set_defaults(func=construct_virtio_pci_blk_bdev) + # ioat + @call_cmd + def scan_ioat_copy_engine(args): + pci_whitelist = [] + if args.pci_whitelist: + for w in args.pci_whitelist.strip().split(" "): + pci_whitelist.append(w) + rpc.ioat.scan_ioat_copy_engine(args.client, pci_whitelist) + + p = subparsers.add_parser('scan_ioat_copy_engine', help='Set scan and enable IOAT copy engine offload.') + p.add_argument('-w', '--pci-whitelist', help="""Whitespace-separated list of PCI addresses in + domain:bus:device.function format or domain.bus.device.function format""") + p.set_defaults(func=scan_ioat_copy_engine) + args = parser.parse_args() try: diff --git a/scripts/rpc/__init__.py b/scripts/rpc/__init__.py index 2d39fc1d5..68d54d3fc 100755 --- a/scripts/rpc/__init__.py +++ b/scripts/rpc/__init__.py @@ -3,6 +3,7 @@ import sys from . import app from . import bdev +from . import ioat from . import iscsi from . import log from . import lvol diff --git a/scripts/rpc/ioat.py b/scripts/rpc/ioat.py new file mode 100644 index 000000000..958e18bb4 --- /dev/null +++ b/scripts/rpc/ioat.py @@ -0,0 +1,12 @@ +def scan_ioat_copy_engine(client, pci_whitelist): + """Scan and enable IOAT copy engine. + + Args: + pci_whitelist: Python list of PCI addresses in + domain:bus:device.function format or + domain.bus.device.function format + """ + params = {} + if pci_whitelist: + params['pci_whitelist'] = pci_whitelist + return client.call('scan_ioat_copy_engine', params)