ioat: make channel allocation explicit

Add a parameter to each I/OAT library function that requires a channel
instead of implicitly using the thread-local channel registration model.

I/OAT channels are already reported by the spdk_ioat_probe() attach
callback, so no infrastructure for channel allocation is necessary.

Change-Id: I8731126fcaea9fe2bafc41a3f75c969a100ef8f0
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>

doc/ioat.index.txt

@@ -42,7 +42,6 @@
 \section key_functions Key Functions
 
 - spdk_ioat_probe() \copybrief spdk_ioat_probe()
-- spdk_ioat_register_thread() \copybrief spdk_ioat_register_thread()
 - spdk_ioat_get_dma_capabilities() \copybrief spdk_ioat_get_dma_capabilities()
 - spdk_ioat_submit_copy() \copybrief spdk_ioat_submit_copy()
 - spdk_ioat_submit_fill() \copybrief spdk_ioat_submit_fill()

examples/ioat/perf/perf.c

@@ -61,10 +61,12 @@ struct ioat_device {
 };
 
 static TAILQ_HEAD(, ioat_device) g_devices;
+static struct ioat_device *g_next_device;
 
 static struct user_config g_user_config;
 
 struct thread_entry {
+	struct spdk_ioat_chan *chan;
 	uint64_t xfer_completed;
 	uint64_t xfer_failed;
 	uint64_t current_queue_depth;
@@ -244,7 +246,7 @@ static void
 drain_io(struct thread_entry *thread_entry)
 {
 	while (thread_entry->current_queue_depth > 0) {
-		spdk_ioat_process_events();
+		spdk_ioat_process_events(thread_entry->chan);
 	}
 }
 
@@ -256,7 +258,8 @@ submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_tas
 	ioat_task->src = src;
 	ioat_task->dst = dst;
 
-	spdk_ioat_submit_copy(ioat_task, ioat_done, dst, src, g_user_config.xfer_size_bytes);
+	spdk_ioat_submit_copy(thread_entry->chan, ioat_task, ioat_done, dst, src,
+			      g_user_config.xfer_size_bytes);
 
 	thread_entry->current_queue_depth++;
 }
@@ -283,6 +286,10 @@ work_fn(void *arg)
 	uint64_t tsc_end;
 	struct thread_entry *t = (struct thread_entry *)arg;
 
+	if (!t->chan) {
+		return 0;
+	}
+
 	t->lcore_id = rte_lcore_id();
 
 	snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
@@ -297,24 +304,17 @@ work_fn(void *arg)
 		return 1;
 	}
 
-	if (spdk_ioat_register_thread() != 0) {
-		fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",
-			rte_lcore_id());
-		return 0;
-	}
-
 	tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();
 
 	// begin to submit transfers
 	submit_xfers(t, g_user_config.queue_depth);
 	while (rte_get_timer_cycles() < tsc_end) {
-		spdk_ioat_process_events();
+		spdk_ioat_process_events(t->chan);
 	}
 
 	// begin to drain io
 	t->is_draining = true;
 	drain_io(t);
-	spdk_ioat_unregister_thread();
 
 	return 0;
 }
@@ -383,6 +383,23 @@ dump_result(struct thread_entry *threads, int len)
 	return total_failed ? 1 : 0;
 }
 
+static struct spdk_ioat_chan *
+get_next_chan(void)
+{
+	struct spdk_ioat_chan *chan;
+
+	if (g_next_device == NULL) {
+		fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n");
+		return NULL;
+	}
+
+	chan = g_next_device->ioat;
+
+	g_next_device = TAILQ_NEXT(g_next_device, tailq);
+
+	return chan;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -400,10 +417,13 @@ main(int argc, char **argv)
 
 	dump_user_config(&g_user_config);
 
+	g_next_device = TAILQ_FIRST(&g_devices);
 	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+		threads[lcore_id].chan = get_next_chan();
 		rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);
 	}
 
+	threads[rte_get_master_lcore()].chan = get_next_chan();
 	if (work_fn(&threads[rte_get_master_lcore()]) != 0) {
 		rc = 1;
 		goto cleanup;

examples/ioat/verify/verify.c

@@ -66,10 +66,12 @@ struct ioat_device {
 };
 
 static TAILQ_HEAD(, ioat_device) g_devices;
+static struct ioat_device *g_next_device;
 
 static struct user_config g_user_config;
 
 struct thread_entry {
+	struct spdk_ioat_chan *chan;
 	uint64_t xfer_completed;
 	uint64_t xfer_failed;
 	uint64_t fill_completed;
@@ -292,7 +294,7 @@ static void
 drain_xfers(struct thread_entry *thread_entry)
 {
 	while (thread_entry->current_queue_depth > 0) {
-		spdk_ioat_process_events();
+		spdk_ioat_process_events(thread_entry->chan);
 	}
 }
 
@@ -300,10 +302,11 @@ static void
 submit_single_xfer(struct ioat_task *ioat_task)
 {
 	if (ioat_task->type == IOAT_FILL_TYPE)
-		spdk_ioat_submit_fill(ioat_task, ioat_done, ioat_task->dst, ioat_task->fill_pattern,
+		spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done,
-				      ioat_task->len);
+				      ioat_task->dst, ioat_task->fill_pattern, ioat_task->len);
 	else
-		spdk_ioat_submit_copy(ioat_task, ioat_done, ioat_task->dst, ioat_task->src, ioat_task->len);
+		spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done,
+				      ioat_task->dst, ioat_task->src, ioat_task->len);
 	ioat_task->thread_entry->current_queue_depth++;
 }
 
@@ -316,7 +319,7 @@ submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth)
 		rte_mempool_get(thread_entry->data_pool, &(ioat_task->buffer));
 
 		ioat_task->type = IOAT_COPY_TYPE;
-		if (spdk_ioat_get_dma_capabilities() & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {
+		if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {
 			if (queue_depth % 2)
 				ioat_task->type = IOAT_FILL_TYPE;
 		}
@@ -332,6 +335,10 @@ work_fn(void *arg)
 	char buf_pool_name[20], task_pool_name[20];
 	struct thread_entry *t = (struct thread_entry *)arg;
 
+	if (!t->chan) {
+		return 0;
+	}
+
 	t->lcore_id = rte_lcore_id();
 
 	snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
@@ -347,24 +354,16 @@ work_fn(void *arg)
 		return 1;
 	}
 
-	if (spdk_ioat_register_thread() != 0) {
-		fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",
-			rte_lcore_id());
-		return 0;
-	}
-
 	tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();
 
 	submit_xfers(t, g_user_config.queue_depth);
 	while (rte_get_timer_cycles() < tsc_end) {
-		spdk_ioat_process_events();
+		spdk_ioat_process_events(t->chan);
 	}
 
 	t->is_draining = true;
 	drain_xfers(t);
 
-	spdk_ioat_unregister_thread();
-
 	return 0;
 }
 
@@ -437,6 +436,23 @@ dump_result(struct thread_entry *threads, int len)
 	return total_failed ? 1 : 0;
 }
 
+static struct spdk_ioat_chan *
+get_next_chan(void)
+{
+	struct spdk_ioat_chan *chan;
+
+	if (g_next_device == NULL) {
+		fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n");
+		return NULL;
+	}
+
+	chan = g_next_device->ioat;
+
+	g_next_device = TAILQ_NEXT(g_next_device, tailq);
+
+	return chan;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -454,10 +470,13 @@ main(int argc, char **argv)
 
 	dump_user_config(&g_user_config);
 
+	g_next_device = TAILQ_FIRST(&g_devices);
 	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+		threads[lcore_id].chan = get_next_chan();
 		rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);
 	}
 
+	threads[rte_get_master_lcore()].chan = get_next_chan();
 	if (work_fn(&threads[rte_get_master_lcore()]) != 0) {
 		rc = 1;
 		goto cleanup;

include/spdk/ioat.h

@@ -91,50 +91,26 @@ int spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_
  */
 int spdk_ioat_detach(struct spdk_ioat_chan *ioat);
 
-/**
- * Request a DMA engine channel for the calling thread.
- *
- * Must be called before submitting any requests from a thread.
- *
- * The \ref spdk_ioat_unregister_thread() function can be called to release the channel.
- */
-int spdk_ioat_register_thread(void);
-
-/**
- * Unregister the current thread's I/OAT channel.
- *
- * This function can be called after \ref spdk_ioat_register_thread() to release the thread's
- * DMA engine channel for use by other threads.
- */
-void spdk_ioat_unregister_thread(void);
-
 /**
  * Submit a DMA engine memory copy request.
- *
- * Before submitting any requests on a thread, the thread must be registered
- * using the \ref spdk_ioat_register_thread() function.
  */
-int64_t spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
+int64_t spdk_ioat_submit_copy(struct spdk_ioat_chan *chan,
+			      void *cb_arg, spdk_ioat_req_cb cb_fn,
 			      void *dst, const void *src, uint64_t nbytes);
 
 /**
  * Submit a DMA engine memory fill request.
- *
- * Before submitting any requests on a thread, the thread must be registered
- * using the \ref spdk_ioat_register_thread() function.
  */
-int64_t spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,
+int64_t spdk_ioat_submit_fill(struct spdk_ioat_chan *chan,
+			      void *cb_arg, spdk_ioat_req_cb cb_fn,
 			      void *dst, uint64_t fill_pattern, uint64_t nbytes);
 
 /**
- * Check for completed requests on the current thread.
+ * Check for completed requests on an I/OAT channel.
- *
- * Before submitting any requests on a thread, the thread must be registered
- * using the \ref spdk_ioat_register_thread() function.
  *
  * \returns 0 on success or negative if something went wrong.
  */
-int spdk_ioat_process_events(void);
+int spdk_ioat_process_events(struct spdk_ioat_chan *chan);
 
 /**
  * DMA engine capability flags
@@ -146,11 +122,8 @@ enum spdk_ioat_dma_capability_flags {
 
 /**
  * Get the DMA engine capabilities.
- *
- * Before submitting any requests on a thread, the thread must be registered
- * using the \ref spdk_ioat_register_thread() function.
  */
-uint32_t spdk_ioat_get_dma_capabilities(void);
+uint32_t spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *chan);
 
 #ifdef __cplusplus
 }

lib/ioat/ioat.c

@@ -34,15 +34,6 @@
 #include "ioat_internal.h"
 #include "ioat_pci.h"
 
-/** List of channels that have been attached but are not yet assigned to a thread.
- *
- * Must hold g_ioat_driver.lock while manipulating this list.
- */
-static SLIST_HEAD(, spdk_ioat_chan) ioat_free_channels;
-
-/** IOAT channel assigned to this thread (or NULL if not assigned yet). */
-static __thread struct spdk_ioat_chan *ioat_thread_channel;
-
 struct ioat_driver {
 	ioat_mutex_t	lock;
 	TAILQ_HEAD(, spdk_ioat_chan)	attached_chans;
@@ -488,8 +479,6 @@ ioat_attach(void *device)
 		return NULL;
 	}
 
-	SLIST_INSERT_HEAD(&ioat_free_channels, ioat, next);
-
 	return ioat;
 }
 
@@ -565,7 +554,6 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat)
 	 * when calling ioat_detach().
 	 */
 	ioat_mutex_lock(&driver->lock);
-	SLIST_REMOVE(&ioat_free_channels, ioat, spdk_ioat_chan, next);
 	TAILQ_REMOVE(&driver->attached_chans, ioat, tailq);
 	ioat_mutex_unlock(&driver->lock);
 
@@ -575,55 +563,15 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat)
 	return 0;
 }
 
-int
-spdk_ioat_register_thread(void)
-{
-	struct ioat_driver	*driver = &g_ioat_driver;
-
-	if (ioat_thread_channel) {
-		ioat_printf(NULL, "%s: thread already registered\n", __func__);
-		return -1;
-	}
-
-	ioat_mutex_lock(&driver->lock);
-
-	ioat_thread_channel = SLIST_FIRST(&ioat_free_channels);
-	if (ioat_thread_channel) {
-		SLIST_REMOVE_HEAD(&ioat_free_channels, next);
-	}
-
-	ioat_mutex_unlock(&driver->lock);
-
-	return ioat_thread_channel ? 0 : -1;
-}
-
-void
-spdk_ioat_unregister_thread(void)
-{
-	struct ioat_driver	*driver = &g_ioat_driver;
-
-	if (!ioat_thread_channel) {
-		return;
-	}
-
-	ioat_mutex_lock(&driver->lock);
-
-	SLIST_INSERT_HEAD(&ioat_free_channels, ioat_thread_channel, next);
-	ioat_thread_channel = NULL;
-
-	ioat_mutex_unlock(&driver->lock);
-}
-
 #define min(a, b) (((a)<(b))?(a):(b))
 
 #define _2MB_PAGE(ptr)		((ptr) & ~(0x200000 - 1))
 #define _2MB_OFFSET(ptr)	((ptr) &  (0x200000 - 1))
 
 int64_t
-spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
+spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
 		      void *dst, const void *src, uint64_t nbytes)
 {
-	struct spdk_ioat_chan	*ioat;
 	struct ioat_descriptor	*last_desc;
 	uint64_t	remaining, op_size;
 	uint64_t	vdst, vsrc;
@@ -631,7 +579,6 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
 	uint64_t	pdst_page, psrc_page;
 	uint32_t	orig_head;
 
-	ioat = ioat_thread_channel;
 	if (!ioat) {
 		return -1;
 	}
@@ -698,16 +645,14 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,
 }
 
 int64_t
-spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,
+spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,
 		      void *dst, uint64_t fill_pattern, uint64_t nbytes)
 {
-	struct spdk_ioat_chan	*ioat;
 	struct ioat_descriptor	*last_desc = NULL;
 	uint64_t	remaining, op_size;
 	uint64_t	vdst;
 	uint32_t	orig_head;
 
-	ioat = ioat_thread_channel;
 	if (!ioat) {
 		return -1;
 	}
@@ -756,11 +701,8 @@ spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,
 }
 
 uint32_t
-spdk_ioat_get_dma_capabilities(void)
+spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *ioat)
 {
-	struct spdk_ioat_chan	*ioat;
-
-	ioat = ioat_thread_channel;
 	if (!ioat) {
 		return 0;
 	}
@@ -768,11 +710,7 @@ spdk_ioat_get_dma_capabilities(void)
 }
 
 int
-spdk_ioat_process_events(void)
+spdk_ioat_process_events(struct spdk_ioat_chan *ioat)
 {
-	if (!ioat_thread_channel) {
+	return ioat_process_channel_events(ioat);
-		return -1;
-	}
-
-	return ioat_process_channel_events(ioat_thread_channel);
 }

lib/ioat/ioat_internal.h

@@ -56,8 +56,6 @@ struct ioat_descriptor {
 
 /* One of these per allocated PCI device. */
 struct spdk_ioat_chan {
-	SLIST_ENTRY(spdk_ioat_chan) next;
-
 	/* Opaque handle to upper layer */
 	void                *device;
 	uint64_t            max_xfer_size;