copy_engine: add I/OAT copy engine module
Change-Id: I670ab7c32aa7571def22c06dde091972808f215b Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp
parent
746dc47d62
commit
fc07014a54
@ -38,4 +38,6 @@ CFLAGS += $(DPDK_INC)
|
||||
LIBNAME = copy
|
||||
C_SRCS = copy_engine.c
|
||||
|
||||
DIRS-y = ioat
|
||||
|
||||
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk
|
||||
|
||||
41
lib/copy/ioat/Makefile
Normal file
41
lib/copy/ioat/Makefile
Normal file
@ -0,0 +1,41 @@
|
||||
#
|
||||
# BSD LICENSE
|
||||
#
|
||||
# Copyright (c) Intel Corporation.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in
|
||||
# the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
# * Neither the name of Intel Corporation nor the names of its
|
||||
# contributors may be used to endorse or promote products derived
|
||||
# from this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
|
||||
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
|
||||
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
|
||||
|
||||
CFLAGS += $(DPDK_INC)
|
||||
LIBNAME = copy_ioat
|
||||
C_SRCS = copy_engine_ioat.c
|
||||
|
||||
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk
|
||||
290
lib/copy/ioat/copy_engine_ioat.c
Normal file
290
lib/copy/ioat/copy_engine_ioat.c
Normal file
@ -0,0 +1,290 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include <rte_config.h>
|
||||
#include <rte_malloc.h>
|
||||
#include <rte_memcpy.h>
|
||||
#include <rte_lcore.h>
|
||||
#include <rte_debug.h>
|
||||
|
||||
#include "spdk/copy_engine.h"
|
||||
#include "spdk/vtophys.h"
|
||||
#include "spdk/conf.h"
|
||||
#include "spdk/log.h"
|
||||
#include "spdk/event.h"
|
||||
#include "spdk/pci.h"
|
||||
|
||||
#include "spdk/ioat.h"
|
||||
|
||||
#define IOAT_MAX_CHANNELS 64
|
||||
|
||||
struct ioat_device {
|
||||
struct spdk_ioat_chan *ioat;
|
||||
/** linked list pointer for device list */
|
||||
TAILQ_ENTRY(ioat_device) tailq;
|
||||
};
|
||||
|
||||
static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices);
|
||||
static int g_unbindfromkernel = 0;
|
||||
static int g_ioat_channel_count = 0;
|
||||
static struct spdk_ioat_chan *g_ioat_chan[RTE_MAX_LCORE];
|
||||
|
||||
struct ioat_whitelist {
|
||||
uint32_t bus;
|
||||
uint32_t dev;
|
||||
uint32_t func;
|
||||
};
|
||||
|
||||
static int
|
||||
ioat_find_dev_by_whitelist_bdf(struct spdk_pci_device *dev,
|
||||
struct ioat_whitelist *whitelist,
|
||||
int num_whitelist_devices)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < num_whitelist_devices; i++) {
|
||||
if (spdk_pci_device_get_bus(dev) == whitelist[i].bus &&
|
||||
spdk_pci_device_get_dev(dev) == whitelist[i].dev &&
|
||||
spdk_pci_device_get_func(dev) == whitelist[i].func)
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
struct ioat_task {
|
||||
copy_completion_cb cb;
|
||||
};
|
||||
|
||||
static int copy_engine_ioat_init(void);
|
||||
static void copy_engine_ioat_exit(void);
|
||||
|
||||
static int
|
||||
copy_engine_ioat_get_ctx_size(void)
|
||||
{
|
||||
return sizeof(struct ioat_task) + sizeof(struct copy_task);
|
||||
}
|
||||
|
||||
SPDK_COPY_MODULE_REGISTER(copy_engine_ioat_init, copy_engine_ioat_exit, NULL,
|
||||
copy_engine_ioat_get_ctx_size)
|
||||
|
||||
static void
|
||||
copy_engine_ioat_exit(void)
|
||||
{
|
||||
struct ioat_device *dev;
|
||||
|
||||
while (!TAILQ_EMPTY(&g_devices)) {
|
||||
dev = TAILQ_FIRST(&g_devices);
|
||||
TAILQ_REMOVE(&g_devices, dev, tailq);
|
||||
spdk_ioat_detach(dev->ioat);
|
||||
rte_free(dev);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
static void
|
||||
ioat_done(void *cb_arg)
|
||||
{
|
||||
struct copy_task *copy_req;
|
||||
struct ioat_task *ioat_task = cb_arg;
|
||||
|
||||
copy_req = (struct copy_task *)
|
||||
((uintptr_t)ioat_task -
|
||||
offsetof(struct copy_task, offload_ctx));
|
||||
|
||||
ioat_task->cb(copy_req, 0);
|
||||
}
|
||||
|
||||
static int64_t
|
||||
ioat_copy_submit(void *cb_arg, void *dst, void *src, uint64_t nbytes,
|
||||
copy_completion_cb cb)
|
||||
{
|
||||
struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
|
||||
struct spdk_ioat_chan *chan = g_ioat_chan[rte_lcore_id()];
|
||||
|
||||
RTE_VERIFY(chan != NULL);
|
||||
|
||||
ioat_task->cb = cb;
|
||||
|
||||
return spdk_ioat_submit_copy(chan, ioat_task, ioat_done, dst, src, nbytes);
|
||||
}
|
||||
|
||||
static void
|
||||
ioat_check_io(void)
|
||||
{
|
||||
struct spdk_ioat_chan *chan = g_ioat_chan[rte_lcore_id()];
|
||||
|
||||
RTE_VERIFY(chan != NULL);
|
||||
spdk_ioat_process_events(chan);
|
||||
}
|
||||
|
||||
static struct spdk_copy_engine ioat_copy_engine = {
|
||||
.copy = ioat_copy_submit,
|
||||
.check_io = ioat_check_io,
|
||||
};
|
||||
|
||||
struct ioat_probe_ctx {
|
||||
int num_whitelist_devices;
|
||||
struct ioat_whitelist whitelist[IOAT_MAX_CHANNELS];
|
||||
};
|
||||
|
||||
static bool
|
||||
probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev)
|
||||
{
|
||||
struct ioat_probe_ctx *ctx = cb_ctx;
|
||||
|
||||
SPDK_NOTICELOG(" Found matching device at %d:%d:%d vendor:0x%04x device:0x%04x\n name:%s\n",
|
||||
spdk_pci_device_get_bus(pci_dev),
|
||||
spdk_pci_device_get_dev(pci_dev),
|
||||
spdk_pci_device_get_func(pci_dev),
|
||||
spdk_pci_device_get_vendor_id(pci_dev),
|
||||
spdk_pci_device_get_device_id(pci_dev),
|
||||
spdk_pci_device_get_device_name(pci_dev));
|
||||
|
||||
if (ctx->num_whitelist_devices > 0 &&
|
||||
!ioat_find_dev_by_whitelist_bdf(pci_dev, ctx->whitelist, ctx->num_whitelist_devices)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (spdk_pci_device_has_non_uio_driver(pci_dev)) {
|
||||
if (g_unbindfromkernel && ctx->num_whitelist_devices > 0) {
|
||||
if (spdk_pci_device_switch_to_uio_driver(pci_dev)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
SPDK_WARNLOG("Device has kernel ioat driver attached, skipping...\n");
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if (spdk_pci_device_bind_uio_driver(pci_dev)) {
|
||||
SPDK_WARNLOG("Device %s %d:%d:%d bind to uio driver failed\n",
|
||||
spdk_pci_device_get_device_name(pci_dev),
|
||||
spdk_pci_device_get_bus(pci_dev),
|
||||
spdk_pci_device_get_dev(pci_dev),
|
||||
spdk_pci_device_get_func(pci_dev));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/* Claim the device in case conflict with other process */
|
||||
if (spdk_pci_device_claim(pci_dev) != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void
|
||||
attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat)
|
||||
{
|
||||
struct ioat_device *dev;
|
||||
|
||||
dev = rte_malloc(NULL, sizeof(*dev), 0);
|
||||
if (dev == NULL) {
|
||||
SPDK_ERRLOG("Failed to allocate device struct\n");
|
||||
return;
|
||||
}
|
||||
|
||||
dev->ioat = ioat;
|
||||
TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
|
||||
g_ioat_channel_count++;
|
||||
}
|
||||
|
||||
static int
|
||||
copy_engine_ioat_init(void)
|
||||
{
|
||||
struct spdk_conf_section *sp = spdk_conf_find_section(NULL, "Ioat");
|
||||
const char *val, *pci_bdf;
|
||||
int i;
|
||||
struct ioat_probe_ctx probe_ctx = {};
|
||||
int lcore;
|
||||
struct ioat_device *dev;
|
||||
|
||||
if (sp != NULL) {
|
||||
val = spdk_conf_section_get_val(sp, "Disable");
|
||||
if (val != NULL) {
|
||||
/* Disable Ioat */
|
||||
if (!strcmp(val, "Yes")) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
/*Init the whitelist*/
|
||||
for (i = 0; i < IOAT_MAX_CHANNELS; i++) {
|
||||
pci_bdf = spdk_conf_section_get_nmval(sp, "Whitelist", i, 0);
|
||||
if (!pci_bdf)
|
||||
break;
|
||||
sscanf(pci_bdf, "%02x:%02x.%1u",
|
||||
&probe_ctx.whitelist[probe_ctx.num_whitelist_devices].bus,
|
||||
&probe_ctx.whitelist[probe_ctx.num_whitelist_devices].dev,
|
||||
&probe_ctx.whitelist[probe_ctx.num_whitelist_devices].func);
|
||||
probe_ctx.num_whitelist_devices++;
|
||||
}
|
||||
|
||||
val = spdk_conf_section_get_val(sp, "UnbindFromKernel");
|
||||
if (val != NULL) {
|
||||
if (!strcmp(val, "Yes")) {
|
||||
g_unbindfromkernel = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (spdk_ioat_probe(&probe_ctx, probe_cb, attach_cb) != 0) {
|
||||
SPDK_ERRLOG("spdk_ioat_probe() failed\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* We only handle the case where we have enough channels */
|
||||
if (g_ioat_channel_count < spdk_app_get_core_count()) {
|
||||
SPDK_ERRLOG("Not enough IOAT channels for all cores\n");
|
||||
copy_engine_ioat_exit();
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Assign channels to lcores in the active core mask */
|
||||
dev = TAILQ_FIRST(&g_devices);
|
||||
/* we use u64 as CPU core mask */
|
||||
for (lcore = 0; lcore < RTE_MAX_LCORE && lcore < 64; lcore++) {
|
||||
if ((spdk_app_get_core_mask() & (1ULL << lcore))) {
|
||||
g_ioat_chan[lcore] = dev->ioat;
|
||||
dev = TAILQ_NEXT(dev, tailq);
|
||||
}
|
||||
}
|
||||
|
||||
SPDK_NOTICELOG("Ioat Copy Engine Offload Enabled\n");
|
||||
spdk_copy_engine_register(&ioat_copy_engine);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user