copy_engine, ioat: add spdk_io_channel support

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I40db1b39882b68147aee4520cbd056f99901cfe4

include/spdk/copy_engine.h

@@ -56,6 +56,7 @@ struct spdk_copy_engine {
 	int64_t	(*fill)(void *cb_arg, void *dst, uint8_t fill,
 			uint64_t nbytes, copy_completion_cb cb);
 	void	(*check_io)(void);
+	struct spdk_io_channel *(*get_io_channel)(uint32_t priority);
 };
 
 struct spdk_copy_module_if {
@@ -84,6 +85,7 @@ struct spdk_copy_module_if {
 };
 
 void spdk_copy_engine_register(struct spdk_copy_engine *copy_engine);
+struct spdk_io_channel *spdk_copy_engine_get_io_channel(uint32_t priority);
 int64_t spdk_copy_submit(struct copy_task *copy_req, void *dst, void *src,
 			 uint64_t nbytes, copy_completion_cb cb);
 int64_t spdk_copy_submit_fill(struct copy_task *copy_req, void *dst, uint8_t fill,

lib/copy/copy_engine.c

@@ -35,14 +35,14 @@
 
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 #include <rte_config.h>
 #include <rte_debug.h>
-#include <rte_malloc.h>
 #include <rte_memcpy.h>
-#include <rte_lcore.h>
 
 #include "spdk/log.h"
 #include "spdk/event.h"
+#include "spdk/io_channel.h"
 
 static struct spdk_copy_engine *hw_copy_engine = NULL;
 /* Memcpy engine always exist */
@@ -51,6 +51,11 @@ static struct spdk_copy_engine *mem_copy_engine = NULL;
 TAILQ_HEAD(, spdk_copy_module_if) spdk_copy_module_list =
 	TAILQ_HEAD_INITIALIZER(spdk_copy_module_list);
 
+struct copy_io_channel {
+	struct spdk_copy_engine	*engine;
+	struct spdk_io_channel	*ch;
+};
+
 void
 spdk_copy_engine_register(struct spdk_copy_engine *copy_engine)
 {
@@ -149,11 +154,30 @@ mem_copy_fill(void *cb_arg, void *dst, uint8_t fill, uint64_t nbytes,
 	return nbytes;
 }
 
+static struct spdk_io_channel *mem_get_io_channel(uint32_t priority);
+
 static struct spdk_copy_engine memcpy_copy_engine = {
 	.copy		= mem_copy_submit,
 	.fill		= mem_copy_fill,
+	.get_io_channel	= mem_get_io_channel,
 };
 
+static int
+memcpy_create_cb(void *io_device, uint32_t priority, void *ctx_buf)
+{
+	return 0;
+}
+
+static void
+memcpy_destroy_cb(void *io_device, void *ctx_buf)
+{
+}
+
+static struct spdk_io_channel *mem_get_io_channel(uint32_t priority)
+{
+	return spdk_get_io_channel(&memcpy_copy_engine, priority);
+}
+
 static int
 copy_engine_mem_get_ctx_size(void)
 {
@@ -178,10 +202,44 @@ void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module)
 	TAILQ_INSERT_TAIL(&spdk_copy_module_list, copy_module, tailq);
 }
 
+static int
+copy_create_cb(void *io_device, uint32_t priority, void *ctx_buf)
+{
+	struct copy_io_channel	*copy_ch = ctx_buf;
+
+	if (hw_copy_engine != NULL) {
+		copy_ch->ch = hw_copy_engine->get_io_channel(priority);
+		if (copy_ch->ch != NULL) {
+			copy_ch->engine = hw_copy_engine;
+			return 0;
+		}
+	}
+
+	copy_ch->ch = mem_copy_engine->get_io_channel(priority);
+	assert(copy_ch->ch != NULL);
+	copy_ch->engine = mem_copy_engine;
+	return 0;
+}
+
+static void
+copy_destroy_cb(void *io_device, void *ctx_buf)
+{
+	struct copy_io_channel	*copy_ch = ctx_buf;
+
+	spdk_put_io_channel(copy_ch->ch);
+}
+
+struct spdk_io_channel *
+spdk_copy_engine_get_io_channel(uint32_t priority)
+{
+	return spdk_get_io_channel(&spdk_copy_module_list, priority);
+}
+
 static int
 copy_engine_mem_init(void)
 {
 	spdk_memcpy_register(&memcpy_copy_engine);
+	spdk_io_device_register(&memcpy_copy_engine, memcpy_create_cb, memcpy_destroy_cb, 0);
 
 	return 0;
 }
@@ -211,6 +269,12 @@ static int
 spdk_copy_engine_initialize(void)
 {
 	spdk_copy_engine_module_initialize();
+	/*
+	 * We need a unique identifier for the copy engine framework, so use the
+	 *  spdk_copy_module_list address for this purpose.
+	 */
+	spdk_io_device_register(&spdk_copy_module_list, copy_create_cb, copy_destroy_cb,
+				sizeof(struct copy_io_channel));
 	return 0;
 }
 

lib/copy/ioat/copy_engine_ioat.c

@@ -46,6 +46,7 @@
 #include "spdk/log.h"
 #include "spdk/event.h"
 #include "spdk/pci.h"
+#include "spdk/io_channel.h"
 
 #include "spdk/ioat.h"
 
@@ -70,6 +71,12 @@ struct ioat_whitelist {
 	uint32_t func;
 };
 
+struct ioat_io_channel {
+	struct spdk_ioat_chan	*ioat_ch;
+	struct ioat_device	*ioat_dev;
+	struct spdk_poller	*poller;
+};
+
 static int
 ioat_find_dev_by_whitelist_bdf(struct spdk_pci_device *dev,
 			       struct ioat_whitelist *whitelist,
@@ -202,12 +209,48 @@ ioat_check_io(void)
 	ioat_poll(chan);
 }
 
+static struct spdk_io_channel *ioat_get_io_channel(uint32_t priority);
+
 static struct spdk_copy_engine ioat_copy_engine = {
 	.copy		= ioat_copy_submit,
 	.fill		= ioat_copy_submit_fill,
 	.check_io	= ioat_check_io,
+	.get_io_channel	= ioat_get_io_channel,
 };
 
+static int
+ioat_create_cb(void *io_device, uint32_t priority, void *ctx_buf)
+{
+	struct ioat_io_channel *ch = ctx_buf;
+	struct ioat_device *ioat_dev;
+
+	ioat_dev = ioat_allocate_device();
+	if (ioat_dev == NULL) {
+		return -1;
+	}
+
+	ch->ioat_dev = ioat_dev;
+	ch->ioat_ch = ioat_dev->ioat;
+	spdk_poller_register(&ch->poller, ioat_poll, ch->ioat_ch,
+			     spdk_app_get_current_core(), NULL, 0);
+	return 0;
+}
+
+static void
+ioat_destroy_cb(void *io_device, void *ctx_buf)
+{
+	struct ioat_io_channel *ch = ctx_buf;
+
+	ioat_free_device(ch->ioat_dev);
+	spdk_poller_unregister(&ch->poller, NULL);
+}
+
+static struct spdk_io_channel *
+ioat_get_io_channel(uint32_t priority)
+{
+	return spdk_get_io_channel(&ioat_copy_engine, priority);
+}
+
 struct ioat_probe_ctx {
 	int num_whitelist_devices;
 	struct ioat_whitelist whitelist[IOAT_MAX_CHANNELS];
@@ -334,6 +377,8 @@ copy_engine_ioat_init(void)
 
 	SPDK_NOTICELOG("Ioat Copy Engine Offload Enabled\n");
 	spdk_copy_engine_register(&ioat_copy_engine);
+	spdk_io_device_register(&ioat_copy_engine, ioat_create_cb, ioat_destroy_cb,
+				sizeof(struct ioat_io_channel));
 
 	return 0;
 }