From cf0871a57e34af4b05601f62cf627173071657b7 Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Thu, 10 Mar 2016 11:07:32 -0700 Subject: [PATCH] ioat: make channel allocation explicit Add a parameter to each I/OAT library function that requires a channel instead of implicitly using the thread-local channel registration model. I/OAT channels are already reported by the spdk_ioat_probe() attach callback, so no infrastructure for channel allocation is necessary. Change-Id: I8731126fcaea9fe2bafc41a3f75c969a100ef8f0 Signed-off-by: Daniel Verkamp --- doc/ioat.index.txt | 1 - examples/ioat/perf/perf.c | 40 ++++++++++++++----- examples/ioat/verify/verify.c | 47 ++++++++++++++++------- include/spdk/ioat.h | 41 ++++---------------- lib/ioat/ioat.c | 72 +++-------------------------------- lib/ioat/ioat_internal.h | 2 - 6 files changed, 75 insertions(+), 128 deletions(-) diff --git a/doc/ioat.index.txt b/doc/ioat.index.txt index 46cc68cec..9fbf5a712 100644 --- a/doc/ioat.index.txt +++ b/doc/ioat.index.txt @@ -42,7 +42,6 @@ \section key_functions Key Functions - spdk_ioat_probe() \copybrief spdk_ioat_probe() -- spdk_ioat_register_thread() \copybrief spdk_ioat_register_thread() - spdk_ioat_get_dma_capabilities() \copybrief spdk_ioat_get_dma_capabilities() - spdk_ioat_submit_copy() \copybrief spdk_ioat_submit_copy() - spdk_ioat_submit_fill() \copybrief spdk_ioat_submit_fill() diff --git a/examples/ioat/perf/perf.c b/examples/ioat/perf/perf.c index b81e99573..fd772bac3 100644 --- a/examples/ioat/perf/perf.c +++ b/examples/ioat/perf/perf.c @@ -61,10 +61,12 @@ struct ioat_device { }; static TAILQ_HEAD(, ioat_device) g_devices; +static struct ioat_device *g_next_device; static struct user_config g_user_config; struct thread_entry { + struct spdk_ioat_chan *chan; uint64_t xfer_completed; uint64_t xfer_failed; uint64_t current_queue_depth; @@ -244,7 +246,7 @@ static void drain_io(struct thread_entry *thread_entry) { while (thread_entry->current_queue_depth > 0) { - spdk_ioat_process_events(); + spdk_ioat_process_events(thread_entry->chan); } } @@ -256,7 +258,8 @@ submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_tas ioat_task->src = src; ioat_task->dst = dst; - spdk_ioat_submit_copy(ioat_task, ioat_done, dst, src, g_user_config.xfer_size_bytes); + spdk_ioat_submit_copy(thread_entry->chan, ioat_task, ioat_done, dst, src, + g_user_config.xfer_size_bytes); thread_entry->current_queue_depth++; } @@ -283,6 +286,10 @@ work_fn(void *arg) uint64_t tsc_end; struct thread_entry *t = (struct thread_entry *)arg; + if (!t->chan) { + return 0; + } + t->lcore_id = rte_lcore_id(); snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id()); @@ -297,24 +304,17 @@ work_fn(void *arg) return 1; } - if (spdk_ioat_register_thread() != 0) { - fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n", - rte_lcore_id()); - return 0; - } - tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz(); // begin to submit transfers submit_xfers(t, g_user_config.queue_depth); while (rte_get_timer_cycles() < tsc_end) { - spdk_ioat_process_events(); + spdk_ioat_process_events(t->chan); } // begin to drain io t->is_draining = true; drain_io(t); - spdk_ioat_unregister_thread(); return 0; } @@ -383,6 +383,23 @@ dump_result(struct thread_entry *threads, int len) return total_failed ? 1 : 0; } +static struct spdk_ioat_chan * +get_next_chan(void) +{ + struct spdk_ioat_chan *chan; + + if (g_next_device == NULL) { + fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n"); + return NULL; + } + + chan = g_next_device->ioat; + + g_next_device = TAILQ_NEXT(g_next_device, tailq); + + return chan; +} + int main(int argc, char **argv) { @@ -400,10 +417,13 @@ main(int argc, char **argv) dump_user_config(&g_user_config); + g_next_device = TAILQ_FIRST(&g_devices); RTE_LCORE_FOREACH_SLAVE(lcore_id) { + threads[lcore_id].chan = get_next_chan(); rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id); } + threads[rte_get_master_lcore()].chan = get_next_chan(); if (work_fn(&threads[rte_get_master_lcore()]) != 0) { rc = 1; goto cleanup; diff --git a/examples/ioat/verify/verify.c b/examples/ioat/verify/verify.c index 4294bdce9..9645f740e 100644 --- a/examples/ioat/verify/verify.c +++ b/examples/ioat/verify/verify.c @@ -66,10 +66,12 @@ struct ioat_device { }; static TAILQ_HEAD(, ioat_device) g_devices; +static struct ioat_device *g_next_device; static struct user_config g_user_config; struct thread_entry { + struct spdk_ioat_chan *chan; uint64_t xfer_completed; uint64_t xfer_failed; uint64_t fill_completed; @@ -292,7 +294,7 @@ static void drain_xfers(struct thread_entry *thread_entry) { while (thread_entry->current_queue_depth > 0) { - spdk_ioat_process_events(); + spdk_ioat_process_events(thread_entry->chan); } } @@ -300,10 +302,11 @@ static void submit_single_xfer(struct ioat_task *ioat_task) { if (ioat_task->type == IOAT_FILL_TYPE) - spdk_ioat_submit_fill(ioat_task, ioat_done, ioat_task->dst, ioat_task->fill_pattern, - ioat_task->len); + spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done, + ioat_task->dst, ioat_task->fill_pattern, ioat_task->len); else - spdk_ioat_submit_copy(ioat_task, ioat_done, ioat_task->dst, ioat_task->src, ioat_task->len); + spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done, + ioat_task->dst, ioat_task->src, ioat_task->len); ioat_task->thread_entry->current_queue_depth++; } @@ -316,7 +319,7 @@ submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth) rte_mempool_get(thread_entry->data_pool, &(ioat_task->buffer)); ioat_task->type = IOAT_COPY_TYPE; - if (spdk_ioat_get_dma_capabilities() & SPDK_IOAT_ENGINE_FILL_SUPPORTED) { + if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) { if (queue_depth % 2) ioat_task->type = IOAT_FILL_TYPE; } @@ -332,6 +335,10 @@ work_fn(void *arg) char buf_pool_name[20], task_pool_name[20]; struct thread_entry *t = (struct thread_entry *)arg; + if (!t->chan) { + return 0; + } + t->lcore_id = rte_lcore_id(); snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id()); @@ -347,24 +354,16 @@ work_fn(void *arg) return 1; } - if (spdk_ioat_register_thread() != 0) { - fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n", - rte_lcore_id()); - return 0; - } - tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz(); submit_xfers(t, g_user_config.queue_depth); while (rte_get_timer_cycles() < tsc_end) { - spdk_ioat_process_events(); + spdk_ioat_process_events(t->chan); } t->is_draining = true; drain_xfers(t); - spdk_ioat_unregister_thread(); - return 0; } @@ -437,6 +436,23 @@ dump_result(struct thread_entry *threads, int len) return total_failed ? 1 : 0; } +static struct spdk_ioat_chan * +get_next_chan(void) +{ + struct spdk_ioat_chan *chan; + + if (g_next_device == NULL) { + fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n"); + return NULL; + } + + chan = g_next_device->ioat; + + g_next_device = TAILQ_NEXT(g_next_device, tailq); + + return chan; +} + int main(int argc, char **argv) { @@ -454,10 +470,13 @@ main(int argc, char **argv) dump_user_config(&g_user_config); + g_next_device = TAILQ_FIRST(&g_devices); RTE_LCORE_FOREACH_SLAVE(lcore_id) { + threads[lcore_id].chan = get_next_chan(); rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id); } + threads[rte_get_master_lcore()].chan = get_next_chan(); if (work_fn(&threads[rte_get_master_lcore()]) != 0) { rc = 1; goto cleanup; diff --git a/include/spdk/ioat.h b/include/spdk/ioat.h index c2eaa2c9b..db5e9a007 100644 --- a/include/spdk/ioat.h +++ b/include/spdk/ioat.h @@ -91,50 +91,26 @@ int spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_ */ int spdk_ioat_detach(struct spdk_ioat_chan *ioat); -/** - * Request a DMA engine channel for the calling thread. - * - * Must be called before submitting any requests from a thread. - * - * The \ref spdk_ioat_unregister_thread() function can be called to release the channel. - */ -int spdk_ioat_register_thread(void); - -/** - * Unregister the current thread's I/OAT channel. - * - * This function can be called after \ref spdk_ioat_register_thread() to release the thread's - * DMA engine channel for use by other threads. - */ -void spdk_ioat_unregister_thread(void); - /** * Submit a DMA engine memory copy request. - * - * Before submitting any requests on a thread, the thread must be registered - * using the \ref spdk_ioat_register_thread() function. */ -int64_t spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn, +int64_t spdk_ioat_submit_copy(struct spdk_ioat_chan *chan, + void *cb_arg, spdk_ioat_req_cb cb_fn, void *dst, const void *src, uint64_t nbytes); /** * Submit a DMA engine memory fill request. - * - * Before submitting any requests on a thread, the thread must be registered - * using the \ref spdk_ioat_register_thread() function. */ -int64_t spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn, +int64_t spdk_ioat_submit_fill(struct spdk_ioat_chan *chan, + void *cb_arg, spdk_ioat_req_cb cb_fn, void *dst, uint64_t fill_pattern, uint64_t nbytes); /** - * Check for completed requests on the current thread. - * - * Before submitting any requests on a thread, the thread must be registered - * using the \ref spdk_ioat_register_thread() function. + * Check for completed requests on an I/OAT channel. * * \returns 0 on success or negative if something went wrong. */ -int spdk_ioat_process_events(void); +int spdk_ioat_process_events(struct spdk_ioat_chan *chan); /** * DMA engine capability flags @@ -146,11 +122,8 @@ enum spdk_ioat_dma_capability_flags { /** * Get the DMA engine capabilities. - * - * Before submitting any requests on a thread, the thread must be registered - * using the \ref spdk_ioat_register_thread() function. */ -uint32_t spdk_ioat_get_dma_capabilities(void); +uint32_t spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *chan); #ifdef __cplusplus } diff --git a/lib/ioat/ioat.c b/lib/ioat/ioat.c index bbbf81988..a4ad7e3a0 100644 --- a/lib/ioat/ioat.c +++ b/lib/ioat/ioat.c @@ -34,15 +34,6 @@ #include "ioat_internal.h" #include "ioat_pci.h" -/** List of channels that have been attached but are not yet assigned to a thread. - * - * Must hold g_ioat_driver.lock while manipulating this list. - */ -static SLIST_HEAD(, spdk_ioat_chan) ioat_free_channels; - -/** IOAT channel assigned to this thread (or NULL if not assigned yet). */ -static __thread struct spdk_ioat_chan *ioat_thread_channel; - struct ioat_driver { ioat_mutex_t lock; TAILQ_HEAD(, spdk_ioat_chan) attached_chans; @@ -488,8 +479,6 @@ ioat_attach(void *device) return NULL; } - SLIST_INSERT_HEAD(&ioat_free_channels, ioat, next); - return ioat; } @@ -565,7 +554,6 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat) * when calling ioat_detach(). */ ioat_mutex_lock(&driver->lock); - SLIST_REMOVE(&ioat_free_channels, ioat, spdk_ioat_chan, next); TAILQ_REMOVE(&driver->attached_chans, ioat, tailq); ioat_mutex_unlock(&driver->lock); @@ -575,55 +563,15 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat) return 0; } -int -spdk_ioat_register_thread(void) -{ - struct ioat_driver *driver = &g_ioat_driver; - - if (ioat_thread_channel) { - ioat_printf(NULL, "%s: thread already registered\n", __func__); - return -1; - } - - ioat_mutex_lock(&driver->lock); - - ioat_thread_channel = SLIST_FIRST(&ioat_free_channels); - if (ioat_thread_channel) { - SLIST_REMOVE_HEAD(&ioat_free_channels, next); - } - - ioat_mutex_unlock(&driver->lock); - - return ioat_thread_channel ? 0 : -1; -} - -void -spdk_ioat_unregister_thread(void) -{ - struct ioat_driver *driver = &g_ioat_driver; - - if (!ioat_thread_channel) { - return; - } - - ioat_mutex_lock(&driver->lock); - - SLIST_INSERT_HEAD(&ioat_free_channels, ioat_thread_channel, next); - ioat_thread_channel = NULL; - - ioat_mutex_unlock(&driver->lock); -} - #define min(a, b) (((a)<(b))?(a):(b)) #define _2MB_PAGE(ptr) ((ptr) & ~(0x200000 - 1)) #define _2MB_OFFSET(ptr) ((ptr) & (0x200000 - 1)) int64_t -spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn, +spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, void *dst, const void *src, uint64_t nbytes) { - struct spdk_ioat_chan *ioat; struct ioat_descriptor *last_desc; uint64_t remaining, op_size; uint64_t vdst, vsrc; @@ -631,7 +579,6 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn, uint64_t pdst_page, psrc_page; uint32_t orig_head; - ioat = ioat_thread_channel; if (!ioat) { return -1; } @@ -698,16 +645,14 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn, } int64_t -spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn, +spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, void *dst, uint64_t fill_pattern, uint64_t nbytes) { - struct spdk_ioat_chan *ioat; struct ioat_descriptor *last_desc = NULL; uint64_t remaining, op_size; uint64_t vdst; uint32_t orig_head; - ioat = ioat_thread_channel; if (!ioat) { return -1; } @@ -756,11 +701,8 @@ spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn, } uint32_t -spdk_ioat_get_dma_capabilities(void) +spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *ioat) { - struct spdk_ioat_chan *ioat; - - ioat = ioat_thread_channel; if (!ioat) { return 0; } @@ -768,11 +710,7 @@ spdk_ioat_get_dma_capabilities(void) } int -spdk_ioat_process_events(void) +spdk_ioat_process_events(struct spdk_ioat_chan *ioat) { - if (!ioat_thread_channel) { - return -1; - } - - return ioat_process_channel_events(ioat_thread_channel); + return ioat_process_channel_events(ioat); } diff --git a/lib/ioat/ioat_internal.h b/lib/ioat/ioat_internal.h index 9b23eef09..e76c4fc95 100644 --- a/lib/ioat/ioat_internal.h +++ b/lib/ioat/ioat_internal.h @@ -56,8 +56,6 @@ struct ioat_descriptor { /* One of these per allocated PCI device. */ struct spdk_ioat_chan { - SLIST_ENTRY(spdk_ioat_chan) next; - /* Opaque handle to upper layer */ void *device; uint64_t max_xfer_size;