ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ioat: make channel allocation explicit

Browse Source 
Add a parameter to each I/OAT library function that requires a channel
instead of implicitly using the thread-local channel registration model.

I/OAT channels are already reported by the spdk_ioat_probe() attach
callback, so no infrastructure for channel allocation is necessary.

Change-Id: I8731126fcaea9fe2bafc41a3f75c969a100ef8f0
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2016-03-10 11:07:32 -07:00 committed by Benjamin Walker
parent 1dd7473078
commit cf0871a57e
6 changed files with 75 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/ioat.index.txt
View File

@ -42,7 +42,6 @@
\section key_functions Key Functions
- spdk_ioat_probe() \copybrief spdk_ioat_probe()
- spdk_ioat_register_thread() \copybrief spdk_ioat_register_thread()
- spdk_ioat_get_dma_capabilities() \copybrief spdk_ioat_get_dma_capabilities()
- spdk_ioat_submit_copy() \copybrief spdk_ioat_submit_copy()
- spdk_ioat_submit_fill() \copybrief spdk_ioat_submit_fill()

40
examples/ioat/perf/perf.c
View File

@ -61,10 +61,12 @@ struct ioat_device {
};
static TAILQ_HEAD(, ioat_device) g_devices;
static struct ioat_device *g_next_device;
static struct user_config g_user_config;
struct thread_entry {
struct spdk_ioat_chan *chan;
uint64_t xfer_completed;
uint64_t xfer_failed;
uint64_t current_queue_depth;
@ -244,7 +246,7 @@ static void
drain_io(struct thread_entry *thread_entry)
{
while (thread_entry->current_queue_depth > 0) {
spdk_ioat_process_events();
spdk_ioat_process_events(thread_entry->chan);
}
}
@ -256,7 +258,8 @@ submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_tas
ioat_task->src = src;
ioat_task->dst = dst;
spdk_ioat_submit_copy(ioat_task, ioat_done, dst, src, g_user_config.xfer_size_bytes);
spdk_ioat_submit_copy(thread_entry->chan, ioat_task, ioat_done, dst, src,
g_user_config.xfer_size_bytes);
thread_entry->current_queue_depth++;
}
@ -283,6 +286,10 @@ work_fn(void *arg)
uint64_t tsc_end;
struct thread_entry *t = (struct thread_entry *)arg;
if (!t->chan) {
return 0;
}
t->lcore_id = rte_lcore_id();
snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
@ -297,24 +304,17 @@ work_fn(void *arg)
return 1;
}
if (spdk_ioat_register_thread() != 0) {
fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",
rte_lcore_id());
return 0;
}
tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();
// begin to submit transfers
submit_xfers(t, g_user_config.queue_depth);
while (rte_get_timer_cycles() < tsc_end) {
spdk_ioat_process_events();
spdk_ioat_process_events(t->chan);
}
// begin to drain io
t->is_draining = true;
drain_io(t);
spdk_ioat_unregister_thread();
return 0;
}
@ -383,6 +383,23 @@ dump_result(struct thread_entry *threads, int len)
return total_failed ? 1 : 0;
}
static struct spdk_ioat_chan *
get_next_chan(void)
{
struct spdk_ioat_chan *chan;
if (g_next_device == NULL) {
fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n");
return NULL;
}
chan = g_next_device->ioat;
g_next_device = TAILQ_NEXT(g_next_device, tailq);
return chan;
}
int
main(int argc, char **argv)
{
@ -400,10 +417,13 @@ main(int argc, char **argv)
dump_user_config(&g_user_config);
g_next_device = TAILQ_FIRST(&g_devices);
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
threads[lcore_id].chan = get_next_chan();
rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);
}
threads[rte_get_master_lcore()].chan = get_next_chan();
if (work_fn(&threads[rte_get_master_lcore()]) != 0) {
rc = 1;
goto cleanup;

47
examples/ioat/verify/verify.c
View File

@ -66,10 +66,12 @@ struct ioat_device {
};
static TAILQ_HEAD(, ioat_device) g_devices;
static struct ioat_device *g_next_device;
static struct user_config g_user_config;
struct thread_entry {
struct spdk_ioat_chan *chan;
uint64_t xfer_completed;
uint64_t xfer_failed;
uint64_t fill_completed;
@ -292,7 +294,7 @@ static void
drain_xfers(struct thread_entry *thread_entry)
{
while (thread_entry->current_queue_depth > 0) {
spdk_ioat_process_events();
spdk_ioat_process_events(thread_entry->chan);
}
}
@ -300,10 +302,11 @@ static void
submit_single_xfer(struct ioat_task *ioat_task)
{
if (ioat_task->type == IOAT_FILL_TYPE)
spdk_ioat_submit_fill(ioat_task, ioat_done, ioat_task->dst, ioat_task->fill_pattern,
ioat_task->len);
spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done,
ioat_task->dst, ioat_task->fill_pattern, ioat_task->len);
else
spdk_ioat_submit_copy(ioat_task, ioat_done, ioat_task->dst, ioat_task->src, ioat_task->len);
spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done,
ioat_task->dst, ioat_task->src, ioat_task->len);
ioat_task->thread_entry->current_queue_depth++;
}
@ -316,7 +319,7 @@ submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth)
rte_mempool_get(thread_entry->data_pool, &(ioat_task->buffer));
ioat_task->type = IOAT_COPY_TYPE;
if (spdk_ioat_get_dma_capabilities() & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {
if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {
if (queue_depth % 2)
ioat_task->type = IOAT_FILL_TYPE;
}
@ -332,6 +335,10 @@ work_fn(void *arg)
char buf_pool_name[20], task_pool_name[20];
struct thread_entry *t = (struct thread_entry *)arg;
if (!t->chan) {
return 0;
}
t->lcore_id = rte_lcore_id();
snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
@ -347,24 +354,16 @@ work_fn(void *arg)
return 1;
}
if (spdk_ioat_register_thread() != 0) {
fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",
rte_lcore_id());
return 0;
}
tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();
submit_xfers(t, g_user_config.queue_depth);
while (rte_get_timer_cycles() < tsc_end) {
spdk_ioat_process_events();
spdk_ioat_process_events(t->chan);
}
t->is_draining = true;
drain_xfers(t);
spdk_ioat_unregister_thread();
return 0;
}
@ -437,6 +436,23 @@ dump_result(struct thread_entry *threads, int len)
return total_failed ? 1 : 0;
}
static struct spdk_ioat_chan *
get_next_chan(void)
{
struct spdk_ioat_chan *chan;
if (g_next_device == NULL) {
fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n");
return NULL;
}
chan = g_next_device->ioat;
g_next_device = TAILQ_NEXT(g_next_device, tailq);
return chan;
}
int
main(int argc, char **argv)
{
@ -454,10 +470,13 @@ main(int argc, char **argv)
dump_user_config(&g_user_config);
g_next_device = TAILQ_FIRST(&g_devices);
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
threads[lcore_id].chan = get_next_chan();
rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);
}
threads[rte_get_master_lcore()].chan = get_next_chan();
if (work_fn(&threads[rte_get_master_lcore()]) != 0) {
rc = 1;
goto cleanup;

41
include/spdk/ioat.h
View File

@ -91,50 +91,26 @@ int spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_
*/
int spdk_ioat_detach(struct spdk_ioat_chan *ioat);
/**
* Request a DMA engine channel for the calling thread.
*
* Must be called before submitting any requests from a thread.
*
* The \ref spdk_ioat_unregister_thread() function can be called to release the channel.
*/
int spdk_ioat_register_thread(void);
/**
* Unregister the current thread's I/OAT channel.
*
* This function can be called after \ref spdk_ioat_register_thread() to release the thread's
* DMA engine channel for use by other threads.
*/
void spdk_ioat_unregister_thread(void);
/**
* Submit a DMA engine memory copy request.
*
* Before submitting any requests on a thread, the thread must be registered
* using the \ref spdk_ioat_register_thread() function.
*/
int64_t spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
int64_t spdk_ioat_submit_copy(struct spdk_ioat_chan *chan,
void *cb_arg, spdk_ioat_req_cb cb_fn,
void *dst, const void *src, uint64_t nbytes);
/**
* Submit a DMA engine memory fill request.
*
* Before submitting any requests on a thread, the thread must be registered
* using the \ref spdk_ioat_register_thread() function.
*/
int64_t spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,
int64_t spdk_ioat_submit_fill(struct spdk_ioat_chan *chan,
void *cb_arg, spdk_ioat_req_cb cb_fn,
void *dst, uint64_t fill_pattern, uint64_t nbytes);
/**
* Check for completed requests on the current thread.
*
* Before submitting any requests on a thread, the thread must be registered
* using the \ref spdk_ioat_register_thread() function.
* Check for completed requests on an I/OAT channel.
*
* \returns 0 on success or negative if something went wrong.
*/
int spdk_ioat_process_events(void);
int spdk_ioat_process_events(struct spdk_ioat_chan *chan);
/**
* DMA engine capability flags
@ -146,11 +122,8 @@ enum spdk_ioat_dma_capability_flags {
/**
* Get the DMA engine capabilities.
*
* Before submitting any requests on a thread, the thread must be registered
* using the \ref spdk_ioat_register_thread() function.
*/
uint32_t spdk_ioat_get_dma_capabilities(void);
uint32_t spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *chan);
#ifdef __cplusplus
}

72
lib/ioat/ioat.c
View File

@ -34,15 +34,6 @@
#include "ioat_internal.h"
#include "ioat_pci.h"
/** List of channels that have been attached but are not yet assigned to a thread.
*
* Must hold g_ioat_driver.lock while manipulating this list.
*/
static SLIST_HEAD(, spdk_ioat_chan) ioat_free_channels;
/** IOAT channel assigned to this thread (or NULL if not assigned yet). */
static __thread struct spdk_ioat_chan *ioat_thread_channel;
struct ioat_driver {
ioat_mutex_t lock;
TAILQ_HEAD(, spdk_ioat_chan) attached_chans;
@ -488,8 +479,6 @@ ioat_attach(void *device)
return NULL;
}
SLIST_INSERT_HEAD(&ioat_free_channels, ioat, next);
return ioat;
}
@ -565,7 +554,6 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat)
* when calling ioat_detach().
*/
ioat_mutex_lock(&driver->lock);
SLIST_REMOVE(&ioat_free_channels, ioat, spdk_ioat_chan, next);
TAILQ_REMOVE(&driver->attached_chans, ioat, tailq);
ioat_mutex_unlock(&driver->lock);
@ -575,55 +563,15 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat)
return 0;
}
int
spdk_ioat_register_thread(void)
{
struct ioat_driver *driver = &g_ioat_driver;
if (ioat_thread_channel) {
ioat_printf(NULL, "%s: thread already registered\n", __func__);
return -1;
}
ioat_mutex_lock(&driver->lock);
ioat_thread_channel = SLIST_FIRST(&ioat_free_channels);
if (ioat_thread_channel) {
SLIST_REMOVE_HEAD(&ioat_free_channels, next);
}
ioat_mutex_unlock(&driver->lock);
return ioat_thread_channel ? 0 : -1;
}
void
spdk_ioat_unregister_thread(void)
{
struct ioat_driver *driver = &g_ioat_driver;
if (!ioat_thread_channel) {
return;
}
ioat_mutex_lock(&driver->lock);
SLIST_INSERT_HEAD(&ioat_free_channels, ioat_thread_channel, next);
ioat_thread_channel = NULL;
ioat_mutex_unlock(&driver->lock);
}
#define min(a, b) (((a)<(b))?(a):(b))
#define _2MB_PAGE(ptr) ((ptr) & ~(0x200000 - 1))
#define _2MB_OFFSET(ptr) ((ptr) & (0x200000 - 1))
int64_t
spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
void *dst, const void *src, uint64_t nbytes)
{
struct spdk_ioat_chan *ioat;
struct ioat_descriptor *last_desc;
uint64_t remaining, op_size;
uint64_t vdst, vsrc;
@ -631,7 +579,6 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
uint64_t pdst_page, psrc_page;
uint32_t orig_head;
ioat = ioat_thread_channel;
if (!ioat) {
return -1;
}
@ -698,16 +645,14 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
}
int64_t
spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,
spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
void *dst, uint64_t fill_pattern, uint64_t nbytes)
{
struct spdk_ioat_chan *ioat;
struct ioat_descriptor *last_desc = NULL;
uint64_t remaining, op_size;
uint64_t vdst;
uint32_t orig_head;
ioat = ioat_thread_channel;
if (!ioat) {
return -1;
}
@ -756,11 +701,8 @@ spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,
}
uint32_t
spdk_ioat_get_dma_capabilities(void)
spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *ioat)
{
struct spdk_ioat_chan *ioat;
ioat = ioat_thread_channel;
if (!ioat) {
return 0;
}
@ -768,11 +710,7 @@ spdk_ioat_get_dma_capabilities(void)
}
int
spdk_ioat_process_events(void)
spdk_ioat_process_events(struct spdk_ioat_chan *ioat)
{
if (!ioat_thread_channel) {
return -1;
}
return ioat_process_channel_events(ioat_thread_channel);
return ioat_process_channel_events(ioat);
}

2
lib/ioat/ioat_internal.h
View File

@ -56,8 +56,6 @@ struct ioat_descriptor {
/* One of these per allocated PCI device. */
struct spdk_ioat_chan {
SLIST_ENTRY(spdk_ioat_chan) next;
/* Opaque handle to upper layer */
void *device;
uint64_t max_xfer_size;