ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ioat: add user-mode Intel I/OAT driver

Browse Source 
The ioat driver supports DMA engine copy offload hardware available on
Intel Xeon platforms.

Change-Id: Ida0b17b25816576948ddb1b0443587e0f09574d4
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2015-12-03 14:30:38 -07:00
parent 8925d9dec0
commit d4ab30ba33
32 changed files with 5256 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CONFIG
View File

@ -45,3 +45,7 @@ CONFIG_DPDK_DIR?=/path/to/dpdk
# Header file to use for NVMe implementation specific functions.
# Defaults to depending on DPDK.
CONFIG_NVME_IMPL?=nvme_impl.h
# Header file to use for IOAT implementation specific functions.
# Defaults to depending on DPDK.
CONFIG_IOAT_IMPL?=ioat_impl.h

4
README.md
View File

@ -82,8 +82,8 @@ Hugepages and Device Binding
============================
Before running an SPDK application, some hugepages must be allocated and
any NVMe devices must be unbound from the native NVMe kernel driver.
any NVMe and I/OAT devices must be unbound from the native kernel drivers.
SPDK includes scripts to automate this process on both Linux and FreeBSD.
1) scripts/configure_hugepages.sh
2) scripts/unbind_nvme.sh
2) scripts/unbind.sh

3
autotest.sh
View File

@ -38,7 +38,7 @@ timing_enter afterboot
./scripts/configure_hugepages.sh 1024
timing_exit afterboot
./scripts/unbind_nvme.sh
./scripts/unbind.sh
#####################
# Unit Tests
@ -48,6 +48,7 @@ timing_enter lib
time test/lib/nvme/nvme.sh
time test/lib/memory/memory.sh
time test/lib/ioat/ioat.sh
timing_exit lib

2367
doc/Doxyfile.ioat Normal file
View File

File diff suppressed because it is too large Load Diff

2
doc/Makefile
View File

@ -1,4 +1,4 @@
DOXYFILES = Doxyfile.nvme
DOXYFILES = Doxyfile.ioat Doxyfile.nvme
OUTPUT_DIRS = $(patsubst Doxyfile.%,output.%,$(DOXYFILES))
all: doc

48
doc/ioat.index.txt Normal file
View File

@ -0,0 +1,48 @@
/*
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*!
\mainpage SPDK Userspace I/OAT Driver
\section interface Public Interface
- ioat.h
\section key_functions Key Functions
- ioat_attach() \copybrief ioat_attach()
- ioat_register_thread() \copybrief ioat_register_thread()
- ioat_submit_copy() \copybrief ioat_submit_copy()
*/

2
examples/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(CURDIR)/..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y += nvme
DIRS-y += ioat nvme
.PHONY: all clean $(DIRS-y)

44
examples/ioat/Makefile Normal file
View File

@ -0,0 +1,44 @@
#
# BSD LICENSE
#
# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(CURDIR)/../..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y += perf verify
.PHONY: all clean $(DIRS-y)
all: $(DIRS-y)
clean: $(DIRS-y)
include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

1
examples/ioat/perf/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
perf

57
examples/ioat/perf/Makefile Normal file
View File

@ -0,0 +1,57 @@
#
# BSD LICENSE
#
# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(CURDIR)/../../..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
APP = perf
C_SRCS := perf.c
CFLAGS += -I. $(DPDK_INC)
SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/ioat/libspdk_ioat.a \
$(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
$(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a
LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt
all: $(APP)
$(APP): $(OBJS) $(SPDK_LIBS)
$(LINK_C)
clean:
$(CLEAN_C) $(APP)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

435
examples/ioat/perf/perf.c Normal file
View File

@ -0,0 +1,435 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <rte_config.h>
#include <rte_malloc.h>
#include <rte_eal.h>
#include <rte_lcore.h>
#include <rte_cycles.h>
#include <rte_mempool.h>
#include <pciaccess.h>
#include "spdk/ioat.h"
#include "spdk/pci.h"
#include "spdk/string.h"
struct user_config {
int xfer_size_bytes;
int queue_depth;
int time_in_sec;
bool verify;
char *core_mask;
};
struct ioat_device {
struct ioat_channel *ioat;
TAILQ_ENTRY(ioat_device) tailq;
};
static TAILQ_HEAD(, ioat_device) g_devices;
static struct user_config g_user_config;
struct thread_entry {
uint64_t xfer_completed;
uint64_t xfer_failed;
uint64_t current_queue_depth;
unsigned lcore_id;
bool is_draining;
struct rte_mempool *data_pool;
struct rte_mempool *task_pool;
};
struct ioat_task {
struct thread_entry *thread_entry;
void *src;
void *dst;
};
static void submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_task,
void *dst, void *src);
static void
construct_user_config(struct user_config *self)
{
self->xfer_size_bytes = 4096;
self->queue_depth = 256;
self->time_in_sec = 10;
self->verify = false;
self->core_mask = "0x1";
}
static void
dump_user_config(struct user_config *self)
{
printf("User configuration:\n");
printf("Transfer size: %u bytes\n", self->xfer_size_bytes);
printf("Queue depth: %u\n", self->queue_depth);
printf("Run time: %u seconds\n", self->time_in_sec);
printf("Core mask: %s\n", self->core_mask);
printf("Verify: %s\n\n", self->verify ? "Yes" : "No");
}
static void
ioat_exit(void)
{
struct ioat_device *dev;
while (!TAILQ_EMPTY(&g_devices)) {
dev = TAILQ_FIRST(&g_devices);
TAILQ_REMOVE(&g_devices, dev, tailq);
ioat_detach(dev->ioat);
rte_free(dev);
}
}
static void
ioat_done(void *cb_arg)
{
struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
struct thread_entry *thread_entry = ioat_task->thread_entry;
if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) {
thread_entry->xfer_failed++;
} else {
thread_entry->xfer_completed++;
}
thread_entry->current_queue_depth--;
if (thread_entry->is_draining) {
rte_mempool_put(thread_entry->data_pool, ioat_task->src);
rte_mempool_put(thread_entry->data_pool, ioat_task->dst);
rte_mempool_put(thread_entry->task_pool, ioat_task);
} else {
submit_single_xfer(thread_entry, ioat_task, ioat_task->dst, ioat_task->src);
}
}
static int
ioat_init(void)
{
struct pci_device_iterator *iter;
struct pci_device *pci_dev;
int err = 0;
struct pci_id_match match;
struct ioat_device *dev;
pci_system_init();
TAILQ_INIT(&g_devices);
match.vendor_id = PCI_MATCH_ANY;
match.subvendor_id = PCI_MATCH_ANY;
match.subdevice_id = PCI_MATCH_ANY;
match.device_id = PCI_MATCH_ANY;
match.device_class = 0x088000;
match.device_class_mask = 0xFFFFFF;
iter = pci_id_match_iterator_create(&match);
while ((pci_dev = pci_device_next(iter)) != NULL) {
/* Check if the PCI devices is a supported IOAT channel. */
if (!(ioat_pci_device_match_id(pci_dev->vendor_id,
pci_dev->device_id))) {
continue;
}
printf(" Found matching device at %d:%d:%d "
"vendor:0x%04x device:0x%04x\n name:%s\n",
pci_dev->bus, pci_dev->dev, pci_dev->func,
pci_dev->vendor_id, pci_dev->device_id,
pci_device_get_device_name(pci_dev));
if (pci_device_has_non_null_driver(pci_dev)) {
printf("Device has kernel driver, skipping...\n");
continue;
}
pci_device_probe(pci_dev);
dev = rte_malloc(NULL, sizeof(*dev), 0);
if (dev == NULL) {
printf("Failed to allocate device struct\n");
err = -1;
goto cleanup;
}
dev->ioat = ioat_attach(pci_dev);
if (dev->ioat == NULL) {
rte_free(dev);
/* Likely no device found. */
err = -1;
goto cleanup;
}
TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
}
cleanup:
pci_iterator_destroy(iter);
if (err != 0) {
ioat_exit();
}
return 0;
}
static void
usage(char *program_name)
{
printf("%s options\n", program_name);
printf("\t[-h help message]\n");
printf("\t[-c core mask for distributing I/O submission/completion work]\n");
printf("\t[-q queue depth]\n");
printf("\t[-s transfer size in bytes]\n");
printf("\t[-t time in seconds]\n");
printf("\t[-v verify copy result if this switch is on]\n");
}
static int
parse_args(int argc, char **argv)
{
int op;
construct_user_config(&g_user_config);
while ((op = getopt(argc, argv, "c:hq:s:t:v")) != -1) {
switch (op) {
case 's':
g_user_config.xfer_size_bytes = atoi(optarg);
break;
case 'q':
g_user_config.queue_depth = atoi(optarg);
break;
case 't':
g_user_config.time_in_sec = atoi(optarg);
break;
case 'c':
g_user_config.core_mask = optarg;
break;
case 'v':
g_user_config.verify = true;
break;
case 'h':
usage(argv[0]);
exit(0);
default:
usage(argv[0]);
return 1;
}
}
if (!g_user_config.xfer_size_bytes || !g_user_config.queue_depth ||
!g_user_config.time_in_sec || !g_user_config.core_mask) {
usage(argv[0]);
return 1;
}
optind = 1;
return 0;
}
static void
drain_io(struct thread_entry *thread_entry)
{
while (thread_entry->current_queue_depth > 0) {
ioat_process_events();
}
}
static void
submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_task, void *dst,
void *src)
{
ioat_task->thread_entry = thread_entry;
ioat_task->src = src;
ioat_task->dst = dst;
ioat_submit_copy(ioat_task, ioat_done, dst, src, g_user_config.xfer_size_bytes);
thread_entry->current_queue_depth++;
}
static void
submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth)
{
while (queue_depth-- > 0) {
void *src = NULL, *dst = NULL;
struct ioat_task *ioat_task = NULL;
rte_mempool_get(thread_entry->data_pool, &src);
rte_mempool_get(thread_entry->data_pool, &dst);
rte_mempool_get(thread_entry->task_pool, (void **)&ioat_task);
submit_single_xfer(thread_entry, ioat_task, dst, src);
}
}
static int
work_fn(void *arg)
{
char buf_pool_name[20], task_pool_name[20];
uint64_t tsc_end;
struct thread_entry *t = (struct thread_entry *)arg;
t->lcore_id = rte_lcore_id();
snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", rte_lcore_id());
t->data_pool = rte_mempool_create(buf_pool_name, 512, g_user_config.xfer_size_bytes, 0, 0, NULL,
NULL,
NULL, NULL, SOCKET_ID_ANY, 0);
t->task_pool = rte_mempool_create(task_pool_name, 512, sizeof(struct ioat_task), 0, 0, NULL, NULL,
NULL, NULL, SOCKET_ID_ANY, 0);
if (!t->data_pool || !t->task_pool) {
fprintf(stderr, "Could not allocate buffer pool.\n");
return 1;
}
if (ioat_register_thread() != 0) {
fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",
rte_lcore_id());
return 0;
}
tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();
// begin to submit transfers
submit_xfers(t, g_user_config.queue_depth);
while (rte_get_timer_cycles() < tsc_end) {
ioat_process_events();
}
// begin to drain io
t->is_draining = true;
drain_io(t);
ioat_unregister_thread();
return 0;
}
static int
init(void)
{
char *core_mask_conf;
core_mask_conf = sprintf_alloc("-c %s", g_user_config.core_mask);
if (!core_mask_conf) {
return 1;
}
char *ealargs[] = {"perf", core_mask_conf, "-n 4", "--no-pci"};
if (rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs) < 0) {
free(core_mask_conf);
fprintf(stderr, "Could not init eal\n");
return 1;
}
free(core_mask_conf);
if (ioat_init() != 0) {
fprintf(stderr, "Could not init ioat\n");
return 1;
}
return 0;
}
static int
dump_result(struct thread_entry *threads, int len)
{
int i;
uint64_t total_completed = 0;
uint64_t total_failed = 0;
uint64_t total_xfer_per_sec, total_bw_in_MBps;
printf("lcore Transfers Bandwidth Failed\n");
printf("--------------------------------------------\n");
for (i = 0; i < len; i++) {
struct thread_entry *t = &threads[i];
uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec;
uint64_t bw_in_MBps = (t->xfer_completed * g_user_config.xfer_size_bytes) /
(g_user_config.time_in_sec * 1024 * 1024);
total_completed += t->xfer_completed;
total_failed += t->xfer_failed;
if (xfer_per_sec) {
printf("%5d %10" PRIu64 "/s %10" PRIu64 " MB/s %6" PRIu64 "\n",
t->lcore_id, xfer_per_sec, bw_in_MBps, t->xfer_failed);
}
}
total_xfer_per_sec = total_completed / g_user_config.time_in_sec;
total_bw_in_MBps = (total_completed * g_user_config.xfer_size_bytes) /
(g_user_config.time_in_sec * 1024 * 1024);
printf("============================================\n");
printf("Total: %10" PRIu64 "/s %10" PRIu64 " MB/s %6" PRIu64 "\n",
total_xfer_per_sec, total_bw_in_MBps, total_failed);
return total_failed ? 1 : 0;
}
int
main(int argc, char **argv)
{
unsigned lcore_id;
struct thread_entry threads[RTE_MAX_LCORE] = {};
if (parse_args(argc, argv) != 0) {
return 1;
}
if (init() != 0) {
return 1;
}
dump_user_config(&g_user_config);
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);
}
if (work_fn(&threads[rte_get_master_lcore()]) != 0) {
return 1;
}
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
if (rte_eal_wait_lcore(lcore_id) != 0) {
return 1;
}
}
return dump_result(threads, RTE_MAX_LCORE);
}

1
examples/ioat/verify/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
verify

57
examples/ioat/verify/Makefile Normal file
View File

@ -0,0 +1,57 @@
#
# BSD LICENSE
#
# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(CURDIR)/../../..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
APP = verify
C_SRCS := verify.c
CFLAGS += -I. $(DPDK_INC)
SPDK_LIBS += $(SPDK_ROOT_DIR)/lib/ioat/libspdk_ioat.a \
$(SPDK_ROOT_DIR)/lib/util/libspdk_util.a \
$(SPDK_ROOT_DIR)/lib/memory/libspdk_memory.a
LIBS += $(SPDK_LIBS) -lpciaccess -lpthread $(DPDK_LIB) -lrt
all: $(APP)
$(APP): $(OBJS) $(SPDK_LIBS)
$(LINK_C)
clean:
$(CLEAN_C) $(APP)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

439
examples/ioat/verify/verify.c Normal file
View File

@ -0,0 +1,439 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <rte_config.h>
#include <rte_lcore.h>
#include <rte_malloc.h>
#include <rte_eal.h>
#include <rte_cycles.h>
#include <rte_mempool.h>
#include <pciaccess.h>
#include "spdk/ioat.h"
#include "spdk/pci.h"
#include "spdk/string.h"
#define SRC_BUFFER_SIZE (512*1024)
struct user_config {
int queue_depth;
int time_in_sec;
char *core_mask;
};
struct ioat_device {
struct ioat_channel *ioat;
TAILQ_ENTRY(ioat_device) tailq;
};
static TAILQ_HEAD(, ioat_device) g_devices;
static struct user_config g_user_config;
struct thread_entry {
uint64_t xfer_completed;
uint64_t xfer_failed;
uint64_t current_queue_depth;
unsigned lcore_id;
bool is_draining;
struct rte_mempool *data_pool;
struct rte_mempool *task_pool;
};
struct ioat_task {
struct thread_entry *thread_entry;
void *buffer;
int len;
void *src;
void *dst;
};
static __thread unsigned int seed = 0;
static unsigned char *g_src;
static void submit_single_xfer(struct ioat_task *ioat_task);
static void
construct_user_config(struct user_config *self)
{
self->queue_depth = 32;
self->time_in_sec = 10;
self->core_mask = "0x1";
}
static void
dump_user_config(struct user_config *self)
{
printf("User configuration:\n");
printf("Run time: %u seconds\n", self->time_in_sec);
printf("Core mask: %s\n", self->core_mask);
printf("Queue depth: %u\n", self->queue_depth);
}
static void
ioat_exit(void)
{
struct ioat_device *dev;
while (!TAILQ_EMPTY(&g_devices)) {
dev = TAILQ_FIRST(&g_devices);
TAILQ_REMOVE(&g_devices, dev, tailq);
ioat_detach(dev->ioat);
rte_free(dev);
}
}
static void prepare_ioat_task(struct thread_entry *thread_entry, struct ioat_task *ioat_task)
{
int len;
int src_offset;
int dst_offset;
src_offset = rand_r(&seed) % SRC_BUFFER_SIZE;
len = rand_r(&seed) % (SRC_BUFFER_SIZE - src_offset);
dst_offset = rand_r(&seed) % (SRC_BUFFER_SIZE - len);
memset(ioat_task->buffer, 0, SRC_BUFFER_SIZE);
ioat_task->len = len;
ioat_task->src = g_src + src_offset;
ioat_task->dst = ioat_task->buffer + dst_offset;
ioat_task->thread_entry = thread_entry;
}
static void
ioat_done(void *cb_arg)
{
struct ioat_task *ioat_task = (struct ioat_task *)cb_arg;
struct thread_entry *thread_entry = ioat_task->thread_entry;
if (memcmp(ioat_task->src, ioat_task->dst, ioat_task->len)) {
thread_entry->xfer_failed++;
} else {
thread_entry->xfer_completed++;
}
thread_entry->current_queue_depth--;
if (thread_entry->is_draining) {
rte_mempool_put(thread_entry->data_pool, ioat_task->buffer);
rte_mempool_put(thread_entry->task_pool, ioat_task);
} else {
prepare_ioat_task(thread_entry, ioat_task);
submit_single_xfer(ioat_task);
}
}
static int
ioat_init(void)
{
struct pci_device_iterator *iter;
struct pci_device *pci_dev;
int err = 0;
struct pci_id_match match;
struct ioat_device *dev;
pci_system_init();
TAILQ_INIT(&g_devices);
match.vendor_id = PCI_MATCH_ANY;
match.subvendor_id = PCI_MATCH_ANY;
match.subdevice_id = PCI_MATCH_ANY;
match.device_id = PCI_MATCH_ANY;
match.device_class = 0x088000;
match.device_class_mask = 0xFFFFFF;
iter = pci_id_match_iterator_create(&match);
while ((pci_dev = pci_device_next(iter)) != NULL) {
/* Check if the PCI devices is a supported IOAT channel. */
if (!(ioat_pci_device_match_id(pci_dev->vendor_id,
pci_dev->device_id))) {
continue;
}
printf(" Found matching device at %d:%d:%d "
"vendor:0x%04x device:0x%04x\n name:%s\n",
pci_dev->bus, pci_dev->dev, pci_dev->func,
pci_dev->vendor_id, pci_dev->device_id,
pci_device_get_device_name(pci_dev));
if (pci_device_has_non_null_driver(pci_dev)) {
printf("Device has kernel driver, skipping...\n");
continue;
}
pci_device_probe(pci_dev);
dev = malloc(sizeof(*dev));
if (dev == NULL) {
printf("Failed to allocate device struct\n");
err = -1;
goto cleanup;
}
memset(dev, 0, sizeof(*dev));
dev->ioat = ioat_attach(pci_dev);
if (dev->ioat == NULL) {
free(dev);
/* Likely no device found. */
err = -1;
goto cleanup;
}
TAILQ_INSERT_TAIL(&g_devices, dev, tailq);
}
cleanup:
pci_iterator_destroy(iter);
if (err != 0) {
ioat_exit();
}
return 0;
}
static void
usage(char *program_name)
{
printf("%s options\n", program_name);
printf("\t[-h help message]\n");
printf("\t[-c core mask for distributing I/O submission/completion work]\n");
printf("\t[-t time in seconds]\n");
printf("\t[-q queue depth]\n");
}
static int
parse_args(int argc, char **argv)
{
int op;
construct_user_config(&g_user_config);
while ((op = getopt(argc, argv, "c:ht:q:")) != -1) {
switch (op) {
case 't':
g_user_config.time_in_sec = atoi(optarg);
break;
case 'c':
g_user_config.core_mask = optarg;
break;
case 'q':
g_user_config.queue_depth = atoi(optarg);
break;
case 'h':
usage(argv[0]);
exit(0);
default:
usage(argv[0]);
return 1;
}
}
if (!g_user_config.time_in_sec || !g_user_config.core_mask || !g_user_config.queue_depth) {
usage(argv[0]);
return 1;
}
optind = 1;
return 0;
}
static void
drain_xfers(struct thread_entry *thread_entry)
{
while (thread_entry->current_queue_depth > 0) {
ioat_process_events();
}
}
static void
submit_single_xfer(struct ioat_task *ioat_task)
{
ioat_submit_copy(ioat_task, ioat_done, ioat_task->dst, ioat_task->src, ioat_task->len);
ioat_task->thread_entry->current_queue_depth++;
}
static void
submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth)
{
while (queue_depth-- > 0) {
struct ioat_task *ioat_task = NULL;
rte_mempool_get(thread_entry->task_pool, (void **)&ioat_task);
rte_mempool_get(thread_entry->data_pool, &(ioat_task->buffer));
prepare_ioat_task(thread_entry, ioat_task);
submit_single_xfer(ioat_task);
}
}
static int
work_fn(void *arg)
{
uint64_t tsc_end;
char buf_pool_name[20], task_pool_name[20];
struct thread_entry *t = (struct thread_entry *)arg;
t->lcore_id = rte_lcore_id();
snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", rte_lcore_id());
t->data_pool = rte_mempool_create(buf_pool_name, g_user_config.queue_depth, SRC_BUFFER_SIZE, 0, 0,
NULL, NULL,
NULL, NULL, SOCKET_ID_ANY, 0);
t->task_pool = rte_mempool_create(task_pool_name, g_user_config.queue_depth,
sizeof(struct ioat_task), 0, 0, NULL, NULL,
NULL, NULL, SOCKET_ID_ANY, 0);
if (!t->data_pool || !t->task_pool) {
fprintf(stderr, "Could not allocate buffer pool.\n");
return 1;
}
if (ioat_register_thread() != 0) {
fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",
rte_lcore_id());
return 0;
}
tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();
submit_xfers(t, g_user_config.queue_depth);
while (rte_get_timer_cycles() < tsc_end) {
ioat_process_events();
}
t->is_draining = true;
drain_xfers(t);
ioat_unregister_thread();
return 0;
}
static int
init_src_buffer(void)
{
int i;
g_src = rte_malloc(NULL, SRC_BUFFER_SIZE, 512);
if (g_src == NULL) {
fprintf(stderr, "Allocate src buffer failed\n");
return -1;
}
for (i = 0; i < SRC_BUFFER_SIZE / 4; i++) {
memset((g_src + (4 * i)), i, 4);
}
return 0;
}
static int
init(void)
{
char *core_mask_conf;
core_mask_conf = sprintf_alloc("-c %s", g_user_config.core_mask);
if (!core_mask_conf) {
return 1;
}
char *ealargs[] = {"verify", core_mask_conf, "-n 4", "--no-pci"};
if (rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs) < 0) {
free(core_mask_conf);
fprintf(stderr, "Could not init eal\n");
return 1;
}
free(core_mask_conf);
if (init_src_buffer() != 0) {
fprintf(stderr, "Could not init src buffer\n");
return 1;
}
if (ioat_init() != 0) {
fprintf(stderr, "Could not init ioat\n");
return 1;
}
return 0;
}
static int
dump_result(struct thread_entry *threads, int len)
{
int i;
uint64_t total_completed = 0;
uint64_t total_failed = 0;
for (i = 0; i < len; i++) {
struct thread_entry *t = &threads[i];
total_completed += t->xfer_completed;
total_failed += t->xfer_failed;
if (t->xfer_completed || t->xfer_failed)
printf("lcore = %d, success = %ld, failed = %ld \n",
t->lcore_id, t->xfer_completed, t->xfer_failed);
}
return total_failed ? 1 : 0;
}
int
main(int argc, char **argv)
{
unsigned lcore_id;
struct thread_entry threads[RTE_MAX_LCORE] = {};
if (parse_args(argc, argv) != 0) {
return 1;
}
if (init() != 0) {
return 1;
}
dump_user_config(&g_user_config);
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);
}
if (work_fn(&threads[rte_get_master_lcore()]) != 0) {
return 1;
}
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
if (rte_eal_wait_lcore(lcore_id) != 0) {
return 1;
}
}
rte_free(g_src);
return dump_result(threads, RTE_MAX_LCORE);
}

101
include/spdk/ioat.h Normal file
View File

@ -0,0 +1,101 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* This file defines the public interface to the I/OAT DMA engine driver.
*/
#ifndef __IOAT_H__
#define __IOAT_H__
#include <inttypes.h>
#include <stdbool.h>
/**
* Signature for callback function invoked when a request is completed.
*/
typedef void (*ioat_callback_t)(void *arg);
/**
* Returns true if vendor_id and device_id match a known IOAT PCI device ID.
*/
bool ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id);
/**
* Attach an I/OAT PCI device to the I/OAT userspace driver.
*
* To stop using the the device and release its associated resources,
* call \ref ioat_detach with the ioat_channel instance returned by this function.
*/
struct ioat_channel *ioat_attach(void *device);
/**
* Detaches specified device returned by \ref ioat_attach() from the I/OAT driver.
*/
int ioat_detach(struct ioat_channel *ioat);
/**
* Request a DMA engine channel for the calling thread.
*
* Must be called before submitting any requests from a thread.
*
* The \ref ioat_unregister_thread() function can be called to release the channel.
*/
int ioat_register_thread(void);
/**
* Unregister the current thread's I/OAT channel.
*
* This function can be called after \ref ioat_register_thread() to release the thread's
* DMA engine channel for use by other threads.
*/
void ioat_unregister_thread(void);
/**
* Submit a DMA engine memory copy request.
*
* Before submitting any requests on a thread, the thread must be registered
* using the \ref ioat_register_thread() function.
*/
int64_t ioat_submit_copy(void *cb_arg, ioat_callback_t cb_fn,
void *dst, const void *src, uint64_t nbytes);
/*
* Check for completed requests on the current thread.
*
* Before submitting any requests on a thread, the thread must be registered
* using the \ref ioat_register_thread() function.
*/
void ioat_process_events(void);
#endif

268
include/spdk/ioat_spec.h Normal file
View File

@ -0,0 +1,268 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __IOAT_SPEC_H__
#define __IOAT_SPEC_H__
#include <inttypes.h>
#define IOAT_INTRCTRL_MASTER_INT_EN 0x01
#define IOAT_VER_3_0 0x30
#define IOAT_VER_3_3 0x33
/* DMA Channel Registers */
#define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK 0xF000
#define IOAT_CHANCTRL_COMPL_DCA_EN 0x0200
#define IOAT_CHANCTRL_CHANNEL_IN_USE 0x0100
#define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL 0x0020
#define IOAT_CHANCTRL_ERR_INT_EN 0x0010
#define IOAT_CHANCTRL_ANY_ERR_ABORT_EN 0x0008
#define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004
#define IOAT_CHANCTRL_INT_REARM 0x0001
struct ioat_registers {
uint8_t chancnt;
uint8_t xfercap;
uint8_t genctrl;
uint8_t intrctrl;
uint32_t attnstatus;
uint8_t cbver; /* 0x08 */
uint8_t reserved4[0x3]; /* 0x09 */
uint16_t intrdelay; /* 0x0C */
uint16_t cs_status; /* 0x0E */
uint32_t dmacapability; /* 0x10 */
uint8_t reserved5[0x6C]; /* 0x14 */
uint16_t chanctrl; /* 0x80 */
uint8_t reserved6[0x2]; /* 0x82 */
uint8_t chancmd; /* 0x84 */
uint8_t reserved3[1]; /* 0x85 */
uint16_t dmacount; /* 0x86 */
uint64_t chansts; /* 0x88 */
uint64_t chainaddr; /* 0x90 */
uint64_t chancmp; /* 0x98 */
uint8_t reserved2[0x8]; /* 0xA0 */
uint32_t chanerr; /* 0xA8 */
uint32_t chanerrmask; /* 0xAC */
} __attribute__((packed));
#define IOAT_CHANCMD_RESET 0x20
#define IOAT_CHANCMD_SUSPEND 0x04
#define IOAT_CHANSTS_STATUS 0x7ULL
#define IOAT_CHANSTS_ACTIVE 0x0
#define IOAT_CHANSTS_IDLE 0x1
#define IOAT_CHANSTS_SUSPENDED 0x2
#define IOAT_CHANSTS_HALTED 0x3
#define IOAT_CHANSTS_ARMED 0x4
#define IOAT_CHANSTS_UNAFFILIATED_ERROR 0x8ULL
#define IOAT_CHANSTS_SOFT_ERROR 0x10ULL
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK (~0x3FULL)
#define IOAT_CHANCMP_ALIGN 8 /* CHANCMP address must be 64-bit aligned */
struct ioat_dma_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
struct {
uint32_t int_enable: 1;
uint32_t src_snoop_disable: 1;
uint32_t dest_snoop_disable: 1;
uint32_t completion_update: 1;
uint32_t fence: 1;
uint32_t null: 1;
uint32_t src_page_break: 1;
uint32_t dest_page_break: 1;
uint32_t bundle: 1;
uint32_t dest_dca: 1;
uint32_t hint: 1;
uint32_t reserved: 13;
#define IOAT_OP_COPY 0x00
uint32_t op: 8;
} control;
} u;
uint64_t src_addr;
uint64_t dest_addr;
uint64_t next;
uint64_t reserved;
uint64_t reserved2;
uint64_t user1;
uint64_t user2;
};
struct ioat_fill_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
struct {
uint32_t int_enable: 1;
uint32_t reserved: 1;
uint32_t dest_snoop_disable: 1;
uint32_t completion_update: 1;
uint32_t fence: 1;
uint32_t reserved2: 2;
uint32_t dest_page_break: 1;
uint32_t bundle: 1;
uint32_t reserved3: 15;
#define IOAT_OP_FILL 0x01
uint32_t op: 8;
} control;
} u;
uint64_t src_data;
uint64_t dest_addr;
uint64_t next;
uint64_t reserved;
uint64_t next_dest_addr;
uint64_t user1;
uint64_t user2;
};
struct ioat_xor_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
struct {
uint32_t int_enable: 1;
uint32_t src_snoop_disable: 1;
uint32_t dest_snoop_disable: 1;
uint32_t completion_update: 1;
uint32_t fence: 1;
uint32_t src_count: 3;
uint32_t bundle: 1;
uint32_t dest_dca: 1;
uint32_t hint: 1;
uint32_t reserved: 13;
#define IOAT_OP_XOR 0x87
#define IOAT_OP_XOR_VAL 0x88
uint32_t op: 8;
} control;
} u;
uint64_t src_addr;
uint64_t dest_addr;
uint64_t next;
uint64_t src_addr2;
uint64_t src_addr3;
uint64_t src_addr4;
uint64_t src_addr5;
};
struct ioat_xor_ext_hw_descriptor {
uint64_t src_addr6;
uint64_t src_addr7;
uint64_t src_addr8;
uint64_t next;
uint64_t reserved[4];
};
struct ioat_pq_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
struct {
uint32_t int_enable: 1;
uint32_t src_snoop_disable: 1;
uint32_t dest_snoop_disable: 1;
uint32_t completion_update: 1;
uint32_t fence: 1;
uint32_t src_count: 3;
uint32_t bundle: 1;
uint32_t dest_dca: 1;
uint32_t hint: 1;
uint32_t p_disable: 1;
uint32_t q_disable: 1;
uint32_t reserved: 11;
#define IOAT_OP_PQ 0x89
#define IOAT_OP_PQ_VAL 0x8a
uint32_t op: 8;
} control;
} u;
uint64_t src_addr;
uint64_t p_addr;
uint64_t next;
uint64_t src_addr2;
uint64_t src_addr3;
uint8_t coef[8];
uint64_t q_addr;
};
struct ioat_pq_ext_hw_descriptor {
uint64_t src_addr4;
uint64_t src_addr5;
uint64_t src_addr6;
uint64_t next;
uint64_t src_addr7;
uint64_t src_addr8;
uint64_t reserved[2];
};
struct ioat_pq_update_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
struct {
uint32_t int_enable: 1;
uint32_t src_snoop_disable: 1;
uint32_t dest_snoop_disable: 1;
uint32_t completion_update: 1;
uint32_t fence: 1;
uint32_t src_cnt: 3;
uint32_t bundle: 1;
uint32_t dest_dca: 1;
uint32_t hint: 1;
uint32_t p_disable: 1;
uint32_t q_disable: 1;
uint32_t reserved: 3;
uint32_t coef: 8;
#define IOAT_OP_PQ_UP 0x8b
uint32_t op: 8;
} control;
} u;
uint64_t src_addr;
uint64_t p_addr;
uint64_t next;
uint64_t src_addr2;
uint64_t p_src;
uint64_t q_src;
uint64_t q_addr;
};
struct ioat_raw_hw_descriptor {
uint64_t field[8];
};
#endif /* __IOAT_SPEC_H__ */

2
lib/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(CURDIR)/..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y += memory util nvme
DIRS-y += memory util nvme ioat
.PHONY: all clean $(DIRS-y)

51
lib/ioat/Makefile Normal file
View File

@ -0,0 +1,51 @@
#
# BSD LICENSE
#
# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(CURDIR)/../..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += $(DPDK_INC) -include $(CONFIG_IOAT_IMPL)
C_SRCS = ioat.c
LIB = libspdk_ioat.a
all: $(LIB)
clean:
$(CLEAN_C)
$(LIB): $(OBJS)
$(LIB_C)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

738
lib/ioat/ioat.c Normal file
View File

@ -0,0 +1,738 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "ioat_internal.h"
#include "ioat_pci.h"
/** List of channels that have been attached but are not yet assigned to a thread.
*
* Must hold g_ioat_driver.lock while manipulating this list.
*/
static SLIST_HEAD(, ioat_channel) ioat_free_channels;
/** IOAT channel assigned to this thread (or NULL if not assigned yet). */
static __thread struct ioat_channel *ioat_thread_channel;
struct ioat_driver {
ioat_mutex_t lock;
};
static struct ioat_driver g_ioat_driver = {
.lock = IOAT_MUTEX_INITIALIZER,
};
struct pci_device_id {
uint16_t vendor;
uint16_t device;
};
static const struct pci_device_id ioat_pci_table[] = {
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2},
{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3},
};
bool
ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id)
{
size_t i;
const struct pci_device_id *ids;
for (i = 0; i < sizeof(ioat_pci_table) / sizeof(struct pci_device_id); i++) {
ids = &ioat_pci_table[i];
if (ids->device == device_id && ids->vendor == vendor_id) {
return true;
}
}
return false;
}
static uint64_t
ioat_mmio_read_8(volatile uint64_t *addr)
{
uint64_t val;
volatile uint32_t *addr32 = (volatile uint32_t *)addr;
if (IOAT_64BIT_IO) {
val = *addr;
} else {
/* Must read lower 4 bytes before upper 4 bytes. */
val = addr32[0];
val |= (uint64_t)addr32[1] << 32;
}
return val;
}
static void
ioat_mmio_write_8(volatile uint64_t *addr, uint64_t val)
{
volatile uint32_t *addr32 = (volatile uint32_t *)addr;
if (IOAT_64BIT_IO) {
*addr = val;
} else {
addr32[0] = (uint32_t)val;
addr32[1] = (uint32_t)(val >> 32);
}
}
static uint64_t
ioat_get_chansts(struct ioat_channel *ioat)
{
return ioat_mmio_read_8(&ioat->regs->chansts);
}
static void
ioat_write_chancmp(struct ioat_channel *ioat, uint64_t addr)
{
ioat_mmio_write_8(&ioat->regs->chancmp, addr);
}
static void
ioat_write_chainaddr(struct ioat_channel *ioat, uint64_t addr)
{
ioat_mmio_write_8(&ioat->regs->chainaddr, addr);
}
static inline void
ioat_suspend(struct ioat_channel *ioat)
{
ioat->regs->chancmd = IOAT_CHANCMD_SUSPEND;
}
static inline void
ioat_reset(struct ioat_channel *ioat)
{
ioat->regs->chancmd = IOAT_CHANCMD_RESET;
}
static inline uint32_t
ioat_reset_pending(struct ioat_channel *ioat)
{
uint8_t cmd;
cmd = ioat->regs->chancmd;
return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
}
static int
ioat_map_pci_bar(struct ioat_channel *ioat)
{
int regs_bar, rc;
void *addr;
regs_bar = 0;
rc = ioat_pcicfg_map_bar(ioat->device, regs_bar, 0, &addr);
if (rc != 0 || addr == NULL) {
ioat_printf(ioat, "%s: pci_device_map_range failed with error code %d\n",
__func__, rc);
return -1;
}
ioat->regs = (volatile struct ioat_registers *)addr;
return 0;
}
static int
ioat_unmap_pci_bar(struct ioat_channel *ioat)
{
int rc = 0;
void *addr = (void *)ioat->regs;
if (addr) {
rc = ioat_pcicfg_unmap_bar(ioat->device, 0, addr);
}
return rc;
}
static inline uint32_t
ioat_get_active(struct ioat_channel *ioat)
{
return (ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1);
}
static inline uint32_t
ioat_get_ring_space(struct ioat_channel *ioat)
{
return (1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1;
}
static struct ioat_descriptor *
ioat_alloc_ring_entry(struct ioat_channel *ioat)
{
struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
desc = malloc(sizeof(*desc));
if (desc == NULL) {
return NULL;
}
hw_desc = ioat_zmalloc(NULL, sizeof(*hw_desc), 64, &desc->hw_desc_bus_addr);
if (hw_desc == NULL) {
free(desc);
return NULL;
}
if (desc->hw_desc_bus_addr == 0) {
free(desc);
ioat_free(hw_desc);
return NULL;
}
desc->u.dma = hw_desc;
return desc;
}
static void
ioat_free_ring_entry(struct ioat_channel *ioat, struct ioat_descriptor *desc)
{
if (desc) {
ioat_free(desc->u.dma);
free(desc);
}
}
static struct ioat_descriptor *
ioat_get_ring_entry(struct ioat_channel *ioat, uint32_t index)
{
return ioat->ring[index % (1 << ioat->ring_size_order)];
}
static void
ioat_submit_single(struct ioat_channel *ioat)
{
ioat->head++;
}
static void
ioat_flush(struct ioat_channel *ioat)
{
ioat->regs->dmacount = (uint16_t)ioat->head;
}
static struct ioat_descriptor *
ioat_prep_null(struct ioat_channel *ioat)
{
struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
if (ioat_get_ring_space(ioat) < 1) {
return NULL;
}
desc = ioat_get_ring_entry(ioat, ioat->head);
hw_desc = desc->u.dma;
hw_desc->u.control_raw = 0;
hw_desc->u.control.op = IOAT_OP_COPY;
hw_desc->u.control.null = 1;
hw_desc->u.control.completion_update = 1;
hw_desc->size = 8;
hw_desc->src_addr = 0;
hw_desc->dest_addr = 0;
desc->callback_fn = NULL;
desc->callback_arg = NULL;
ioat_submit_single(ioat);
return desc;
}
static struct ioat_descriptor *
ioat_prep_copy(struct ioat_channel *ioat, uint64_t dst,
uint64_t src, uint32_t len)
{
struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
ioat_assert(len <= ioat->max_xfer_size);
if (ioat_get_ring_space(ioat) < 1) {
return NULL;
}
desc = ioat_get_ring_entry(ioat, ioat->head);
hw_desc = desc->u.dma;
hw_desc->u.control_raw = 0;
hw_desc->u.control.op = IOAT_OP_COPY;
hw_desc->u.control.completion_update = 1;
hw_desc->size = len;
hw_desc->src_addr = src;
hw_desc->dest_addr = dst;
desc->callback_fn = NULL;
desc->callback_arg = NULL;
ioat_submit_single(ioat);
return desc;
}
static int ioat_reset_hw(struct ioat_channel *ioat)
{
int timeout;
uint64_t status;
uint32_t chanerr;
status = ioat_get_chansts(ioat);
if (is_ioat_active(status) || is_ioat_idle(status)) {
ioat_suspend(ioat);
}
timeout = 20; /* in milliseconds */
while (is_ioat_active(status) || is_ioat_idle(status)) {
ioat_delay_us(1000);
timeout--;
if (timeout == 0) {
ioat_printf(ioat, "%s: timed out waiting for suspend\n", __func__);
return -1;
}
status = ioat_get_chansts(ioat);
}
/*
* Clear any outstanding errors.
* CHANERR is write-1-to-clear, so write the current CHANERR bits back to reset everything.
*/
chanerr = ioat->regs->chanerr;
ioat->regs->chanerr = chanerr;
ioat_reset(ioat);
timeout = 20;
while (ioat_reset_pending(ioat)) {
ioat_delay_us(1000);
timeout--;
if (timeout == 0) {
ioat_printf(ioat, "%s: timed out waiting for reset\n", __func__);
return -1;
}
}
return 0;
}
static void
ioat_process_channel_events(struct ioat_channel *ioat)
{
struct ioat_descriptor *desc;
uint64_t status, completed_descriptor;
if (ioat->head == ioat->tail) {
return;
}
status = *ioat->comp_update;
completed_descriptor = status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
if (is_ioat_halted(status)) {
ioat_printf(ioat, "%s: Channel halted (%x)\n", __func__, ioat->regs->chanerr);
/* TODO: report error */
return;
}
if (completed_descriptor == ioat->last_seen) {
return;
}
while (1) {
desc = ioat_get_ring_entry(ioat, ioat->tail);
if (desc->callback_fn) {
desc->callback_fn(desc->callback_arg);
}
ioat->tail++;
if (desc->hw_desc_bus_addr == completed_descriptor)
break;
}
ioat->last_seen = desc->hw_desc_bus_addr;
}
static int
ioat_channel_destruct(struct ioat_channel *ioat)
{
ioat_unmap_pci_bar(ioat);
if (ioat->ring) {
int i;
for (i = 0; i < (1 << ioat->ring_size_order); i++)
ioat_free_ring_entry(ioat, ioat->ring[i]);
free(ioat->ring);
}
if (ioat->comp_update) {
ioat_free((void *)ioat->comp_update);
ioat->comp_update = NULL;
}
return 0;
}
static int
ioat_channel_start(struct ioat_channel *ioat)
{
struct ioat_descriptor **ring;
uint8_t xfercap, version;
uint64_t status;
int i, num_descriptors;
struct ioat_descriptor *next;
struct ioat_dma_hw_descriptor *dma_hw_desc;
uint64_t comp_update_bus_addr;
if (ioat_map_pci_bar(ioat) != 0) {
ioat_printf(ioat, "%s: ioat_map_pci_bar() failed\n", __func__);
return -1;
}
version = ioat->regs->cbver;
if (version < IOAT_VER_3_0) {
ioat_printf(ioat, "%s: unsupported IOAT version %u.%u\n",
__func__, version >> 4, version & 0xF);
return -1;
}
xfercap = ioat->regs->xfercap;
/* Only bits [4:0] are valid. */
xfercap &= 0x1f;
if (xfercap == 0) {
/* 0 means 4 GB max transfer size. */
ioat->max_xfer_size = 1ULL << 32;
} else if (xfercap < 12) {
/* XFCERCAP must be at least 12 (4 KB) according to the spec. */
ioat_printf(ioat, "%s: invalid XFERCAP value %u\n", __func__, xfercap);
return -1;
} else {
ioat->max_xfer_size = 1U << xfercap;
}
ioat->comp_update = ioat_zmalloc(NULL, sizeof(*ioat->comp_update), IOAT_CHANCMP_ALIGN,
&comp_update_bus_addr);
if (ioat->comp_update == NULL) {
return -1;
}
ioat->ring_size_order = IOAT_DEFAULT_ORDER;
num_descriptors = 1 << ioat->ring_size_order;
ioat->ring = calloc(num_descriptors, sizeof(*ring));
if (!ioat->ring) {
ioat_channel_destruct(ioat);
return -1;
}
ring = ioat->ring;
for (i = 0; i < num_descriptors; i++) {
ring[i] = ioat_alloc_ring_entry(ioat);
if (!ring[i]) {
ioat_channel_destruct(ioat);
return -1;
}
}
for (i = 0; i < num_descriptors - 1; i++) {
next = ring[i + 1];
dma_hw_desc = ring[i]->u.dma;
dma_hw_desc->next = next->hw_desc_bus_addr;
}
ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr;
ioat->head = 0;
ioat->tail = 0;
ioat->last_seen = 0;
ioat_reset_hw(ioat);
ioat->regs->chanctrl = IOAT_CHANCTRL_ANY_ERR_ABORT_EN;
ioat_write_chancmp(ioat, comp_update_bus_addr);
ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr);
ioat_prep_null(ioat);
ioat_flush(ioat);
i = 100;
while (i-- > 0) {
ioat_delay_us(100);
status = ioat_get_chansts(ioat);
if (is_ioat_idle(status))
break;
}
if (is_ioat_idle(status)) {
ioat_process_channel_events(ioat);
} else {
ioat_printf(ioat, "%s: could not start channel: status = %p\n error = %#x\n",
__func__, (void *)status, ioat->regs->chanerr);
ioat_channel_destruct(ioat);
return -1;
}
return 0;
}
struct ioat_channel *
ioat_attach(void *device)
{
struct ioat_driver *driver = &g_ioat_driver;
struct ioat_channel *ioat;
uint32_t cmd_reg;
ioat = malloc(sizeof(struct ioat_channel));
if (ioat == NULL) {
return NULL;
}
/* Enable PCI busmaster. */
ioat_pcicfg_read32(device, &cmd_reg, 4);
cmd_reg |= 0x4;
ioat_pcicfg_write32(device, cmd_reg, 4);
ioat->device = device;
if (ioat_channel_start(ioat) != 0) {
free(ioat);
return NULL;
}
ioat_mutex_lock(&driver->lock);
SLIST_INSERT_HEAD(&ioat_free_channels, ioat, next);
ioat_mutex_unlock(&driver->lock);
return ioat;
}
int
ioat_detach(struct ioat_channel *ioat)
{
struct ioat_driver *driver = &g_ioat_driver;
/* ioat should be in the free list (not registered to a thread)
* when calling ioat_detach().
*/
ioat_mutex_lock(&driver->lock);
SLIST_REMOVE(&ioat_free_channels, ioat, ioat_channel, next);
ioat_mutex_unlock(&driver->lock);
ioat_channel_destruct(ioat);
free(ioat);
return 0;
}
int
ioat_register_thread(void)
{
struct ioat_driver *driver = &g_ioat_driver;
if (ioat_thread_channel) {
ioat_printf(NULL, "%s: thread already registered\n", __func__);
return -1;
}
ioat_mutex_lock(&driver->lock);
ioat_thread_channel = SLIST_FIRST(&ioat_free_channels);
if (ioat_thread_channel) {
SLIST_REMOVE_HEAD(&ioat_free_channels, next);
}
ioat_mutex_unlock(&driver->lock);
return ioat_thread_channel ? 0 : -1;
}
void
ioat_unregister_thread(void)
{
struct ioat_driver *driver = &g_ioat_driver;
if (!ioat_thread_channel) {
return;
}
ioat_mutex_lock(&driver->lock);
SLIST_INSERT_HEAD(&ioat_free_channels, ioat_thread_channel, next);
ioat_thread_channel = NULL;
ioat_mutex_unlock(&driver->lock);
}
#define min(a, b) (((a)<(b))?(a):(b))
#define _2MB_PAGE(ptr) ((ptr) & ~(0x200000 - 1))
#define _2MB_OFFSET(ptr) ((ptr) & (0x200000 - 1))
int64_t
ioat_submit_copy(void *cb_arg, ioat_callback_t cb_fn,
void *dst, const void *src, uint64_t nbytes)
{
struct ioat_channel *ioat;
struct ioat_descriptor *last_desc;
uint64_t remaining, op_size;
uint64_t vdst, vsrc;
uint64_t vdst_page, vsrc_page;
uint64_t pdst_page, psrc_page;
uint32_t orig_head;
ioat = ioat_thread_channel;
if (!ioat) {
return -1;
}
orig_head = ioat->head;
vdst = (uint64_t)dst;
vsrc = (uint64_t)src;
vsrc_page = _2MB_PAGE(vsrc);
vdst_page = _2MB_PAGE(vdst);
psrc_page = ioat_vtophys((void *)vsrc_page);
pdst_page = ioat_vtophys((void *)vdst_page);
remaining = nbytes;
while (remaining) {
op_size = remaining;
op_size = min(op_size, (0x200000 - _2MB_OFFSET(vsrc)));
op_size = min(op_size, (0x200000 - _2MB_OFFSET(vdst)));
op_size = min(op_size, ioat->max_xfer_size);
remaining -= op_size;
last_desc = ioat_prep_copy(ioat,
pdst_page + _2MB_OFFSET(vdst),
psrc_page + _2MB_OFFSET(vsrc),
op_size);
if (remaining == 0 || last_desc == NULL) {
break;
}
vsrc += op_size;
vdst += op_size;
if (_2MB_PAGE(vsrc) != vsrc_page) {
vsrc_page = _2MB_PAGE(vsrc);
psrc_page = ioat_vtophys((void *)vsrc_page);
}
if (_2MB_PAGE(vdst) != vdst_page) {
vdst_page = _2MB_PAGE(vdst);
pdst_page = ioat_vtophys((void *)vdst_page);
}
}
/* Issue null descriptor for null transfer */
if (nbytes == 0) {
last_desc = ioat_prep_null(ioat);
}
if (last_desc) {
last_desc->callback_fn = cb_fn;
last_desc->callback_arg = cb_arg;
} else {
/*
* Ran out of descriptors in the ring - reset head to leave things as they were
* in case we managed to fill out any descriptors.
*/
ioat->head = orig_head;
return -1;
}
ioat_flush(ioat);
return nbytes;
}
void ioat_process_events(void)
{
if (!ioat_thread_channel) {
return;
}
ioat_process_channel_events(ioat_thread_channel);
}

93
lib/ioat/ioat_impl.h Normal file
View File

@ -0,0 +1,93 @@
#ifndef __IOAT_IMPL_H__
#define __IOAT_IMPL_H__
#include <assert.h>
#include <pthread.h>
#include <pciaccess.h>
#include <stdio.h>
#include <rte_malloc.h>
#include <rte_config.h>
#include <rte_atomic.h>
#include <rte_cycles.h>
#include "spdk/vtophys.h"
/**
* \file
*
* This file describes the functions required to integrate
* the userspace IOAT driver for a specific implementation. This
* implementation is specific for DPDK. Users would revise it as
* necessary for their own particular environment if not using it
* within the SPDK framework.
*/
/**
* Allocate a pinned, physically contiguous memory buffer with the
* given size and alignment.
*/
static inline void *
ioat_zmalloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr)
{
void *buf = rte_zmalloc(tag, size, align);
*phys_addr = rte_malloc_virt2phy(buf);
return buf;
}
/**
* Free a memory buffer previously allocated with ioat_zmalloc.
*/
#define ioat_free(buf) rte_free(buf)
/**
* Return the physical address for the specified virtual address.
*/
#define ioat_vtophys(buf) vtophys(buf)
/**
* Delay us.
*/
#define ioat_delay_us(us) rte_delay_us(us)
/**
* Assert a condition and panic/abort as desired. Failures of these
* assertions indicate catastrophic failures within the driver.
*/
#define ioat_assert(check) assert(check)
/**
* Log or print a message from the driver.
*/
#define ioat_printf(chan, fmt, args...) printf(fmt, ##args)
/**
*
*/
#define ioat_pcicfg_read32(handle, var, offset) pci_device_cfg_read_u32(handle, var, offset)
#define ioat_pcicfg_write32(handle, var, offset) pci_device_cfg_write_u32(handle, var, offset)
static inline int
ioat_pcicfg_map_bar(void *devhandle, uint32_t bar, uint32_t read_only, void **mapped_addr)
{
struct pci_device *dev = devhandle;
uint32_t flags = (read_only ? 0 : PCI_DEV_MAP_FLAG_WRITABLE);
return pci_device_map_range(dev, dev->regions[bar].base_addr, 4096,
flags, mapped_addr);
}
static inline int
ioat_pcicfg_unmap_bar(void *devhandle, uint32_t bar, void *addr)
{
struct pci_device *dev = devhandle;
return pci_device_unmap_range(dev, addr, dev->regions[bar].size);
}
typedef pthread_mutex_t ioat_mutex_t;
#define ioat_mutex_lock pthread_mutex_lock
#define ioat_mutex_unlock pthread_mutex_unlock
#define IOAT_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
#endif /* __IOAT_IMPL_H__ */

116
lib/ioat/ioat_internal.h Normal file
View File

@ -0,0 +1,116 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __IOAT_INTERNAL_H__
#define __IOAT_INTERNAL_H__
#include "spdk/ioat.h"
#include "spdk/ioat_spec.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <inttypes.h>
#include "spdk/queue.h"
/* Allocate 2 << 15 (32K) descriptors per channel by default. */
#define IOAT_DEFAULT_ORDER 15
#ifdef __x86_64__
#define IOAT_64BIT_IO 1 /* Can do atomic 64-bit memory read/write (over PCIe) */
#else
#define IOAT_64BIT_IO 0
#endif
struct ioat_descriptor {
ioat_callback_t callback_fn;
void *callback_arg;
union {
struct ioat_dma_hw_descriptor *dma;
struct ioat_fill_hw_descriptor *fill;
struct ioat_xor_hw_descriptor *xor;
struct ioat_xor_ext_hw_descriptor *xor_ext;
struct ioat_pq_hw_descriptor *pq;
struct ioat_pq_ext_hw_descriptor *pq_ext;
struct ioat_raw_hw_descriptor *raw;
} u;
uint64_t hw_desc_bus_addr;
};
/* One of these per allocated PCI device. */
struct ioat_channel {
SLIST_ENTRY(ioat_channel) next;
/* Opaque handle to upper layer */
void *device;
uint64_t max_xfer_size;
volatile struct ioat_registers *regs;
volatile uint64_t *comp_update;
uint32_t head;
uint32_t tail;
uint32_t ring_size_order;
uint64_t last_seen;
struct ioat_descriptor **ring;
};
static inline uint32_t
is_ioat_active(uint64_t status)
{
return (status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE;
}
static inline uint32_t
is_ioat_idle(uint64_t status)
{
return (status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_IDLE;
}
static inline uint32_t
is_ioat_halted(uint64_t status)
{
return (status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED;
}
static inline uint32_t
is_ioat_suspended(uint64_t status)
{
return (status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED;
}
#endif /* __IOAT_INTERNAL_H__ */

94
lib/ioat/ioat_pci.h Normal file
View File

@ -0,0 +1,94 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __IOAT_PCI_H__
#define __IOAT_PCI_H__
#define PCI_DEVICE_ID_INTEL_IOAT_SNB0 0x3c20
#define PCI_DEVICE_ID_INTEL_IOAT_SNB1 0x3c21
#define PCI_DEVICE_ID_INTEL_IOAT_SNB2 0x3c22
#define PCI_DEVICE_ID_INTEL_IOAT_SNB3 0x3c23
#define PCI_DEVICE_ID_INTEL_IOAT_SNB4 0x3c24
#define PCI_DEVICE_ID_INTEL_IOAT_SNB5 0x3c25
#define PCI_DEVICE_ID_INTEL_IOAT_SNB6 0x3c26
#define PCI_DEVICE_ID_INTEL_IOAT_SNB7 0x3c27
#define PCI_DEVICE_ID_INTEL_IOAT_SNB8 0x3c2e
#define PCI_DEVICE_ID_INTEL_IOAT_SNB9 0x3c2f
#define PCI_DEVICE_ID_INTEL_IOAT_IVB0 0x0e20
#define PCI_DEVICE_ID_INTEL_IOAT_IVB1 0x0e21
#define PCI_DEVICE_ID_INTEL_IOAT_IVB2 0x0e22
#define PCI_DEVICE_ID_INTEL_IOAT_IVB3 0x0e23
#define PCI_DEVICE_ID_INTEL_IOAT_IVB4 0x0e24
#define PCI_DEVICE_ID_INTEL_IOAT_IVB5 0x0e25
#define PCI_DEVICE_ID_INTEL_IOAT_IVB6 0x0e26
#define PCI_DEVICE_ID_INTEL_IOAT_IVB7 0x0e27
#define PCI_DEVICE_ID_INTEL_IOAT_IVB8 0x0e2e
#define PCI_DEVICE_ID_INTEL_IOAT_IVB9 0x0e2f
#define PCI_DEVICE_ID_INTEL_IOAT_HSW0 0x2f20
#define PCI_DEVICE_ID_INTEL_IOAT_HSW1 0x2f21
#define PCI_DEVICE_ID_INTEL_IOAT_HSW2 0x2f22
#define PCI_DEVICE_ID_INTEL_IOAT_HSW3 0x2f23
#define PCI_DEVICE_ID_INTEL_IOAT_HSW4 0x2f24
#define PCI_DEVICE_ID_INTEL_IOAT_HSW5 0x2f25
#define PCI_DEVICE_ID_INTEL_IOAT_HSW6 0x2f26
#define PCI_DEVICE_ID_INTEL_IOAT_HSW7 0x2f27
#define PCI_DEVICE_ID_INTEL_IOAT_HSW8 0x2f2e
#define PCI_DEVICE_ID_INTEL_IOAT_HSW9 0x2f2f
#define PCI_DEVICE_ID_INTEL_IOAT_BWD0 0x0C50
#define PCI_DEVICE_ID_INTEL_IOAT_BWD1 0x0C51
#define PCI_DEVICE_ID_INTEL_IOAT_BWD2 0x0C52
#define PCI_DEVICE_ID_INTEL_IOAT_BWD3 0x0C53
#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE0 0x6f50
#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE1 0x6f51
#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE2 0x6f52
#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE3 0x6f53
#define PCI_DEVICE_ID_INTEL_IOAT_BDX0 0x6f20
#define PCI_DEVICE_ID_INTEL_IOAT_BDX1 0x6f21
#define PCI_DEVICE_ID_INTEL_IOAT_BDX2 0x6f22
#define PCI_DEVICE_ID_INTEL_IOAT_BDX3 0x6f23
#define PCI_DEVICE_ID_INTEL_IOAT_BDX4 0x6f24
#define PCI_DEVICE_ID_INTEL_IOAT_BDX5 0x6f25
#define PCI_DEVICE_ID_INTEL_IOAT_BDX6 0x6f26
#define PCI_DEVICE_ID_INTEL_IOAT_BDX7 0x6f27
#define PCI_DEVICE_ID_INTEL_IOAT_BDX8 0x6f2e
#define PCI_DEVICE_ID_INTEL_IOAT_BDX9 0x6f2f
#define PCI_VENDOR_ID_INTEL 0x8086
#endif /* __IOAT_PCI_H__ */

58
mk/ioat.unittest.mk Normal file
View File

@ -0,0 +1,58 @@
#
# BSD LICENSE
#
# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/..
IOAT_DIR := $(SPDK_ROOT_DIR)/lib/ioat
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
C_SRCS = $(TEST_FILE) $(OTHER_FILES)
CFLAGS += -I$(SPDK_ROOT_DIR)/lib -include $(SPDK_ROOT_DIR)/test/lib/ioat/unit/ioat_impl.h
LIBS += -lcunit -lpthread
APP = $(TEST_FILE:.c=)
all: $(APP)
$(APP) : $(OBJS)
$(LINK_C)
clean:
$(CLEAN_C) $(APP)
%.o: $(IOAT_DIR)/%.c %.d $(MAKEFILE_LIST)
$(COMPILE_C)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

5
scripts/unbind_nvme.sh → scripts/unbind.sh
View File

@ -4,14 +4,15 @@ set -e
function configure_linux {
rmmod nvme || true
rmmod ioatdma || true
}
function configure_freebsd {
TMP=`mktemp`
AWK_PROG="{if (count > 0) printf \",\"; printf \"%s:%s:%s\",\$2,\$3,\$4; count++}"
echo $AWK_PROG > $TMP
NVME_PCICONF=`pciconf -l | grep class=0x010802`
BDFS=`echo $NVME_PCICONF | awk -F: -f $TMP`
PCICONF=`pciconf -l | grep 'class=0x010802\|^ioat'`
BDFS=`echo $PCICONF | awk -F: -f $TMP`
kldunload nic_uio.ko || true
kenv hw.nic_uio.bdfs=$BDFS
kldload nic_uio.ko

2
test/lib/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(CURDIR)/../..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = nvme memory
DIRS-y = nvme memory ioat
.PHONY: all clean $(DIRS-y)

44
test/lib/ioat/Makefile Normal file
View File

@ -0,0 +1,44 @@
#
# BSD LICENSE
#
# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(CURDIR)/../../..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = unit
.PHONY: all clean $(DIRS-y)
all: $(DIRS-y)
clean: $(DIRS-y)
include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

23
test/lib/ioat/ioat.sh Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -xe
testdir=$(readlink -f $(dirname $0))
rootdir="$testdir/../../.."
source $rootdir/scripts/autotest_common.sh
timing_enter ioat
timing_enter unit
$testdir/unit/ioat_ut
timing_exit unit
timing_enter perf
$rootdir/examples/ioat/perf/perf
timing_exit perf
timing_enter verify
$rootdir/examples/ioat/verify/verify
timing_exit verify
timing_exit ioat

1
test/lib/ioat/unit/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
ioat_ut

39
test/lib/ioat/unit/Makefile Normal file
View File

@ -0,0 +1,39 @@
#
# BSD LICENSE
#
# Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(CURDIR)/../../../..
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
TEST_FILE = ioat_ut.c
include $(SPDK_ROOT_DIR)/mk/ioat.unittest.mk

50
test/lib/ioat/unit/ioat_impl.h Normal file
View File

@ -0,0 +1,50 @@
/* Unit test stubbed version of ioat_impl.h */
#ifndef __IOAT_IMPL_H__
#define __IOAT_IMPL_H__
#include <assert.h>
#include <pthread.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
static inline void *
ioat_zmalloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr)
{
return calloc(1, size);
}
#define ioat_noop() do { } while (0)
#define ioat_calloc(tag, num, size, align) calloc(num, size)
#define ioat_malloc(tag, size, align) malloc(size)
#define ioat_free(buf) free(buf)
#define ioat_vtophys(buf) (uint64_t)(buf)
#define ioat_delay_us(us) ioat_noop()
#define ioat_assert(check) assert(check)
#define ioat_printf(chan, fmt, args...) printf(fmt, ##args)
#define ioat_pcicfg_read32(handle, var, offset) do { *(var) = 0xFFFFFFFFu; } while (0)
#define ioat_pcicfg_write32(handle, var, offset) do { (void)(var); } while (0)
static inline int
ioat_pcicfg_map_bar(void *devhandle, uint32_t bar, uint32_t read_only, void **mapped_addr)
{
/* TODO */
*mapped_addr = NULL;
return -1;
}
static inline int
ioat_pcicfg_unmap_bar(void *devhandle, uint32_t bar, void *addr)
{
return 0;
}
typedef pthread_mutex_t ioat_mutex_t;
#define ioat_mutex_lock pthread_mutex_lock
#define ioat_mutex_unlock pthread_mutex_unlock
#define IOAT_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
#endif /* __IOAT_IMPL_H__ */

112
test/lib/ioat/unit/ioat_ut.c Normal file
View File

@ -0,0 +1,112 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "CUnit/Basic.h"
#include "ioat/ioat.c"
static void ioat_state_check(void)
{
/*
* CHANSTS's STATUS field is 3 bits (8 possible values), but only has 5 valid states:
* ACTIVE 0x0
* IDLE 0x1
* SUSPENDED 0x2
* HALTED 0x3
* ARMED 0x4
*/
CU_ASSERT(is_ioat_active(0) == 1); /* ACTIVE */
CU_ASSERT(is_ioat_active(1) == 0); /* IDLE */
CU_ASSERT(is_ioat_active(2) == 0); /* SUSPENDED */
CU_ASSERT(is_ioat_active(3) == 0); /* HALTED */
CU_ASSERT(is_ioat_active(4) == 0); /* ARMED */
CU_ASSERT(is_ioat_active(5) == 0); /* reserved */
CU_ASSERT(is_ioat_active(6) == 0); /* reserved */
CU_ASSERT(is_ioat_active(7) == 0); /* reserved */
CU_ASSERT(is_ioat_idle(0) == 0); /* ACTIVE */
CU_ASSERT(is_ioat_idle(1) == 1); /* IDLE */
CU_ASSERT(is_ioat_idle(2) == 0); /* SUSPENDED */
CU_ASSERT(is_ioat_idle(3) == 0); /* HALTED */
CU_ASSERT(is_ioat_idle(4) == 0); /* ARMED */
CU_ASSERT(is_ioat_idle(5) == 0); /* reserved */
CU_ASSERT(is_ioat_idle(6) == 0); /* reserved */
CU_ASSERT(is_ioat_idle(7) == 0); /* reserved */
CU_ASSERT(is_ioat_suspended(0) == 0); /* ACTIVE */
CU_ASSERT(is_ioat_suspended(1) == 0); /* IDLE */
CU_ASSERT(is_ioat_suspended(2) == 1); /* SUSPENDED */
CU_ASSERT(is_ioat_suspended(3) == 0); /* HALTED */
CU_ASSERT(is_ioat_suspended(4) == 0); /* ARMED */
CU_ASSERT(is_ioat_suspended(5) == 0); /* reserved */
CU_ASSERT(is_ioat_suspended(6) == 0); /* reserved */
CU_ASSERT(is_ioat_suspended(7) == 0); /* reserved */
CU_ASSERT(is_ioat_halted(0) == 0); /* ACTIVE */
CU_ASSERT(is_ioat_halted(1) == 0); /* IDLE */
CU_ASSERT(is_ioat_halted(2) == 0); /* SUSPENDED */
CU_ASSERT(is_ioat_halted(3) == 1); /* HALTED */
CU_ASSERT(is_ioat_halted(4) == 0); /* ARMED */
CU_ASSERT(is_ioat_halted(5) == 0); /* reserved */
CU_ASSERT(is_ioat_halted(6) == 0); /* reserved */
CU_ASSERT(is_ioat_halted(7) == 0); /* reserved */
}
int main(int argc, char **argv)
{
CU_pSuite suite = NULL;
unsigned int num_failures;
if (CU_initialize_registry() != CUE_SUCCESS) {
return CU_get_error();
}
suite = CU_add_suite("ioat", NULL, NULL);
if (suite == NULL) {
CU_cleanup_registry();
return CU_get_error();
}
if (
CU_add_test(suite, "ioat_state_check", ioat_state_check) == NULL) {
CU_cleanup_registry();
return CU_get_error();
}
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();
num_failures = CU_get_number_of_failures();
CU_cleanup_registry();
return num_failures;
}

4
unittest.sh
View File

@ -9,3 +9,7 @@ test/lib/nvme/unit/nvme_ctrlr_c/nvme_ctrlr_ut
test/lib/nvme/unit/nvme_ctrlr_cmd_c/nvme_ctrlr_cmd_ut
test/lib/nvme/unit/nvme_ns_cmd_c/nvme_ns_cmd_ut
test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut
make -C test/lib/ioat/unit
test/lib/ioat/unit/ioat_ut