diff --git a/lib/copy/Makefile b/lib/copy/Makefile index dd48f29f8..d357d3d9b 100644 --- a/lib/copy/Makefile +++ b/lib/copy/Makefile @@ -38,4 +38,6 @@ CFLAGS += $(DPDK_INC) LIBNAME = copy C_SRCS = copy_engine.c +DIRS-y = ioat + include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/lib/copy/ioat/Makefile b/lib/copy/ioat/Makefile new file mode 100644 index 000000000..348122381 --- /dev/null +++ b/lib/copy/ioat/Makefile @@ -0,0 +1,41 @@ +# +# BSD LICENSE +# +# Copyright (c) Intel Corporation. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +CFLAGS += $(DPDK_INC) +LIBNAME = copy_ioat +C_SRCS = copy_engine_ioat.c + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/lib/copy/ioat/copy_engine_ioat.c b/lib/copy/ioat/copy_engine_ioat.c new file mode 100644 index 000000000..7e57e6df0 --- /dev/null +++ b/lib/copy/ioat/copy_engine_ioat.c @@ -0,0 +1,290 @@ +/*- + * BSD LICENSE + * + * Copyright (c) Intel Corporation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include "spdk/copy_engine.h" +#include "spdk/vtophys.h" +#include "spdk/conf.h" +#include "spdk/log.h" +#include "spdk/event.h" +#include "spdk/pci.h" + +#include "spdk/ioat.h" + +#define IOAT_MAX_CHANNELS 64 + +struct ioat_device { + struct spdk_ioat_chan *ioat; + /** linked list pointer for device list */ + TAILQ_ENTRY(ioat_device) tailq; +}; + +static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices); +static int g_unbindfromkernel = 0; +static int g_ioat_channel_count = 0; +static struct spdk_ioat_chan *g_ioat_chan[RTE_MAX_LCORE]; + +struct ioat_whitelist { + uint32_t bus; + uint32_t dev; + uint32_t func; +}; + +static int +ioat_find_dev_by_whitelist_bdf(struct spdk_pci_device *dev, + struct ioat_whitelist *whitelist, + int num_whitelist_devices) +{ + int i; + + for (i = 0; i < num_whitelist_devices; i++) { + if (spdk_pci_device_get_bus(dev) == whitelist[i].bus && + spdk_pci_device_get_dev(dev) == whitelist[i].dev && + spdk_pci_device_get_func(dev) == whitelist[i].func) + return 1; + } + return 0; +} + + +struct ioat_task { + copy_completion_cb cb; +}; + +static int copy_engine_ioat_init(void); +static void copy_engine_ioat_exit(void); + +static int +copy_engine_ioat_get_ctx_size(void) +{ + return sizeof(struct ioat_task) + sizeof(struct copy_task); +} + +SPDK_COPY_MODULE_REGISTER(copy_engine_ioat_init, copy_engine_ioat_exit, NULL, + copy_engine_ioat_get_ctx_size) + +static void +copy_engine_ioat_exit(void) +{ + struct ioat_device *dev; + + while (!TAILQ_EMPTY(&g_devices)) { + dev = TAILQ_FIRST(&g_devices); + TAILQ_REMOVE(&g_devices, dev, tailq); + spdk_ioat_detach(dev->ioat); + rte_free(dev); + } + return; +} + +static void +ioat_done(void *cb_arg) +{ + struct copy_task *copy_req; + struct ioat_task *ioat_task = cb_arg; + + copy_req = (struct copy_task *) + ((uintptr_t)ioat_task - + offsetof(struct copy_task, offload_ctx)); + + ioat_task->cb(copy_req, 0); +} + +static int64_t +ioat_copy_submit(void *cb_arg, void *dst, void *src, uint64_t nbytes, + copy_completion_cb cb) +{ + struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; + struct spdk_ioat_chan *chan = g_ioat_chan[rte_lcore_id()]; + + RTE_VERIFY(chan != NULL); + + ioat_task->cb = cb; + + return spdk_ioat_submit_copy(chan, ioat_task, ioat_done, dst, src, nbytes); +} + +static void +ioat_check_io(void) +{ + struct spdk_ioat_chan *chan = g_ioat_chan[rte_lcore_id()]; + + RTE_VERIFY(chan != NULL); + spdk_ioat_process_events(chan); +} + +static struct spdk_copy_engine ioat_copy_engine = { + .copy = ioat_copy_submit, + .check_io = ioat_check_io, +}; + +struct ioat_probe_ctx { + int num_whitelist_devices; + struct ioat_whitelist whitelist[IOAT_MAX_CHANNELS]; +}; + +static bool +probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) +{ + struct ioat_probe_ctx *ctx = cb_ctx; + + SPDK_NOTICELOG(" Found matching device at %d:%d:%d vendor:0x%04x device:0x%04x\n name:%s\n", + spdk_pci_device_get_bus(pci_dev), + spdk_pci_device_get_dev(pci_dev), + spdk_pci_device_get_func(pci_dev), + spdk_pci_device_get_vendor_id(pci_dev), + spdk_pci_device_get_device_id(pci_dev), + spdk_pci_device_get_device_name(pci_dev)); + + if (ctx->num_whitelist_devices > 0 && + !ioat_find_dev_by_whitelist_bdf(pci_dev, ctx->whitelist, ctx->num_whitelist_devices)) { + return false; + } + + if (spdk_pci_device_has_non_uio_driver(pci_dev)) { + if (g_unbindfromkernel && ctx->num_whitelist_devices > 0) { + if (spdk_pci_device_switch_to_uio_driver(pci_dev)) { + return false; + } + } else { + SPDK_WARNLOG("Device has kernel ioat driver attached, skipping...\n"); + return false; + } + } else { + if (spdk_pci_device_bind_uio_driver(pci_dev)) { + SPDK_WARNLOG("Device %s %d:%d:%d bind to uio driver failed\n", + spdk_pci_device_get_device_name(pci_dev), + spdk_pci_device_get_bus(pci_dev), + spdk_pci_device_get_dev(pci_dev), + spdk_pci_device_get_func(pci_dev)); + return false; + } + } + + /* Claim the device in case conflict with other process */ + if (spdk_pci_device_claim(pci_dev) != 0) { + return false; + } + + return true; +} + +static void +attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) +{ + struct ioat_device *dev; + + dev = rte_malloc(NULL, sizeof(*dev), 0); + if (dev == NULL) { + SPDK_ERRLOG("Failed to allocate device struct\n"); + return; + } + + dev->ioat = ioat; + TAILQ_INSERT_TAIL(&g_devices, dev, tailq); + g_ioat_channel_count++; +} + +static int +copy_engine_ioat_init(void) +{ + struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Ioat"); + const char *val, *pci_bdf; + int i; + struct ioat_probe_ctx probe_ctx = {}; + int lcore; + struct ioat_device *dev; + + if (sp != NULL) { + val = spdk_conf_section_get_val(sp, "Disable"); + if (val != NULL) { + /* Disable Ioat */ + if (!strcmp(val, "Yes")) { + return 0; + } + } + /*Init the whitelist*/ + for (i = 0; i < IOAT_MAX_CHANNELS; i++) { + pci_bdf = spdk_conf_section_get_nmval(sp, "Whitelist", i, 0); + if (!pci_bdf) + break; + sscanf(pci_bdf, "%02x:%02x.%1u", + &probe_ctx.whitelist[probe_ctx.num_whitelist_devices].bus, + &probe_ctx.whitelist[probe_ctx.num_whitelist_devices].dev, + &probe_ctx.whitelist[probe_ctx.num_whitelist_devices].func); + probe_ctx.num_whitelist_devices++; + } + + val = spdk_conf_section_get_val(sp, "UnbindFromKernel"); + if (val != NULL) { + if (!strcmp(val, "Yes")) { + g_unbindfromkernel = 1; + } + } + } + + if (spdk_ioat_probe(&probe_ctx, probe_cb, attach_cb) != 0) { + SPDK_ERRLOG("spdk_ioat_probe() failed\n"); + return -1; + } + + /* We only handle the case where we have enough channels */ + if (g_ioat_channel_count < spdk_app_get_core_count()) { + SPDK_ERRLOG("Not enough IOAT channels for all cores\n"); + copy_engine_ioat_exit(); + return 0; + } + + /* Assign channels to lcores in the active core mask */ + dev = TAILQ_FIRST(&g_devices); + /* we use u64 as CPU core mask */ + for (lcore = 0; lcore < RTE_MAX_LCORE && lcore < 64; lcore++) { + if ((spdk_app_get_core_mask() & (1ULL << lcore))) { + g_ioat_chan[lcore] = dev->ioat; + dev = TAILQ_NEXT(dev, tailq); + } + } + + SPDK_NOTICELOG("Ioat Copy Engine Offload Enabled\n"); + spdk_copy_engine_register(&ioat_copy_engine); + + return 0; +}