FTL: Add io channel logic

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: Ibf6bfbabc03c43e7938531c4fe08fde01ce02a3f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13307
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/ftl/Makefile

@@ -19,7 +19,7 @@ FTL_SUBDIRS := mngt utils
 
 C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c
 C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c
-C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c
+C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c
 C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c
 
 SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ftl.map)

lib/ftl/ftl_core.c

@@ -46,6 +46,76 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at
 	attrs->optimum_io_size = dev->xfer_size;
 }
 
+static void
+start_io(struct ftl_io *io)
+{
+	struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch);
+
+	io->map = ftl_mempool_get(ioch->map_pool);
+	if (spdk_unlikely(!io->map)) {
+		io->status = -ENOMEM;
+		ftl_io_complete(io);
+		return;
+	}
+
+	switch (io->type) {
+	case FTL_IO_READ:
+	case FTL_IO_WRITE:
+	case FTL_IO_UNMAP:
+	default:
+		io->status = -EOPNOTSUPP;
+		ftl_io_complete(io);
+	}
+}
+
+#define FTL_IO_QUEUE_BATCH 16
+int
+ftl_io_channel_poll(void *arg)
+{
+	struct ftl_io_channel *ch = arg;
+	void *ios[FTL_IO_QUEUE_BATCH];
+	uint64_t i, count;
+
+	count = spdk_ring_dequeue(ch->cq, ios, FTL_IO_QUEUE_BATCH);
+	if (count == 0) {
+		return SPDK_POLLER_IDLE;
+	}
+
+	for (i = 0; i < count; i++) {
+		struct ftl_io *io = ios[i];
+		io->user_fn(io->cb_ctx, io->status);
+	}
+
+	return SPDK_POLLER_BUSY;
+}
+
+static void
+ftl_process_io_channel(struct spdk_ftl_dev *dev, struct ftl_io_channel *ioch)
+{
+	void *ios[FTL_IO_QUEUE_BATCH];
+	size_t count, i;
+
+	count = spdk_ring_dequeue(ioch->sq, ios, FTL_IO_QUEUE_BATCH);
+	if (count == 0) {
+		return;
+	}
+
+	for (i = 0; i < count; i++) {
+		struct ftl_io *io = ios[i];
+		start_io(io);
+	}
+}
+
+static void
+ftl_process_io_queue(struct spdk_ftl_dev *dev)
+{
+	struct ftl_io_channel *ioch;
+
+	TAILQ_FOREACH(ioch, &dev->ioch_queue, entry) {
+		ftl_process_io_channel(dev, ioch);
+	}
+}
+
 int
 ftl_core_poller(void *ctx)
 {
@@ -57,6 +127,8 @@ ftl_core_poller(void *ctx)
 		return SPDK_POLLER_IDLE;
 	}
 
+	ftl_process_io_queue(dev);
+
 	if (io_activity_total_old != dev->io_activity_total) {
 		return SPDK_POLLER_BUSY;
 	}

lib/ftl/ftl_core.h

@@ -28,6 +28,9 @@ struct spdk_ftl_dev {
 	/* FTL device layout */
 	struct ftl_layout		layout;
 
+	/* Queue of registered IO channels */
+	TAILQ_HEAD(, ftl_io_channel)	ioch_queue;
+
 	/* Underlying device */
 	struct spdk_bdev_desc		*base_bdev_desc;
 
@@ -49,6 +52,12 @@ struct spdk_ftl_dev {
 	/* Indicates the device is about to be stopped */
 	bool				halt;
 
+	/* Indicates if the device is registered as an IO device */
+	bool				io_device_registered;
+
+	/* Management process to be continued after IO device unregistration completes */
+	struct ftl_mngt_process		*unregister_process;
+
 	/* counters for poller busy, include
 	   1. nv cache read/write
 	   2. metadata read/write
@@ -99,6 +108,10 @@ struct spdk_ftl_dev {
 
 int ftl_core_poller(void *ctx);
 
+int ftl_io_channel_poll(void *arg);
+
+struct ftl_io_channel *ftl_io_channel_get_ctx(struct spdk_io_channel *ioch);
+
 static inline uint64_t
 ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev)
 {
@@ -128,6 +141,12 @@ ftl_get_write_unit_size(struct spdk_bdev *bdev)
 	return 32;
 }
 
+static inline struct spdk_thread *
+ftl_get_core_thread(const struct spdk_ftl_dev *dev)
+{
+	return dev->core_thread;
+}
+
 static inline size_t
 ftl_get_num_bands(const struct spdk_ftl_dev *dev)
 {

lib/ftl/ftl_init.c

@@ -115,6 +115,7 @@ allocate_dev(const struct spdk_ftl_conf *conf, int *error)
 
 	TAILQ_INIT(&dev->rd_sq);
 	TAILQ_INIT(&dev->wr_sq);
+	TAILQ_INIT(&dev->ioch_queue);
 
 	return dev;
 error:

lib/ftl/ftl_io.c

@@ -117,6 +117,9 @@ ftl_io_iovec_len_left(struct ftl_io *io)
 static void
 ftl_io_cb(struct ftl_io *io, void *arg, int status)
 {
+	struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch);
+	size_t result;
+
 	if (spdk_unlikely(status)) {
 		io->status = status;
 
@@ -145,15 +148,20 @@ ftl_io_cb(struct ftl_io *io, void *arg, int status)
 		}
 	}
 
-	/* User completion added in next patch */
+	if (io->map) {
+		ftl_mempool_put(ioch->map_pool, io->map);
+	}
+
+	result = spdk_ring_enqueue(ioch->cq, (void **)&io, 1, NULL);
+	assert(result != 0);
 }
 
 int
 ftl_io_init(struct spdk_io_channel *_ioch, struct ftl_io *io, uint64_t lba, size_t num_blocks,
 	    struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type)
 {
-	/* dev initialized from io channel context in next patch */
+	struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(_ioch);
-	struct spdk_ftl_dev *dev = NULL;
+	struct spdk_ftl_dev *dev = ioch->dev;
 
 	memset(io, 0, sizeof(struct ftl_io));
 	io->ioch = _ioch;

lib/ftl/ftl_io.h

@@ -34,6 +34,21 @@ enum ftl_io_type {
 
 #define FTL_IO_MAX_IOVEC 4
 
+struct ftl_io_channel {
+	/*  Device */
+	struct spdk_ftl_dev		*dev;
+	/*  Entry of IO channels queue/list */
+	TAILQ_ENTRY(ftl_io_channel)	entry;
+	/*  IO map pool */
+	struct ftl_mempool		*map_pool;
+	/*  Poller used for completing user requests and retrying IO */
+	struct spdk_poller		*poller;
+	/*  Submission queue */
+	struct spdk_ring		*sq;
+	/*  Completion queue */
+	struct spdk_ring		*cq;
+};
+
 /* General IO descriptor for user requests */
 struct ftl_io {
 	/* Device */

lib/ftl/mngt/ftl_mngt_ioch.c

@@ -0,0 +1,202 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/thread.h"
+
+#include "ftl_core.h"
+#include "ftl_mngt.h"
+#include "ftl_mngt_steps.h"
+#include "utils/ftl_mempool.h"
+
+struct ftl_io_channel_ctx {
+	struct ftl_io_channel *ioch;
+};
+
+struct ftl_io_channel *
+ftl_io_channel_get_ctx(struct spdk_io_channel *ioch)
+{
+	struct ftl_io_channel_ctx *ctx = spdk_io_channel_get_ctx(ioch);
+	return ctx->ioch;
+}
+
+static void
+ftl_dev_register_channel(void *ctx)
+{
+	struct ftl_io_channel *ioch = ctx;
+	struct spdk_ftl_dev *dev = ioch->dev;
+
+	/* This only runs on the core thread, so it's safe to do this lockless */
+	TAILQ_INSERT_TAIL(&dev->ioch_queue, ioch, entry);
+}
+
+static void
+io_channel_unregister(void *ctx)
+{
+	struct ftl_io_channel *ioch = ctx;
+	struct spdk_ftl_dev *dev = ioch->dev;
+
+	TAILQ_REMOVE(&dev->ioch_queue, ioch, entry);
+
+	spdk_ring_free(ioch->cq);
+	spdk_ring_free(ioch->sq);
+	ftl_mempool_destroy(ioch->map_pool);
+	free(ioch);
+}
+
+static int
+io_channel_create_cb(void *io_device, void *ctx)
+{
+	struct spdk_ftl_dev *dev = io_device;
+	struct ftl_io_channel_ctx *_ioch = ctx;
+	struct ftl_io_channel *ioch;
+	char mempool_name[32];
+	int rc;
+
+	FTL_NOTICELOG(dev, "FTL IO channel created on %s\n",
+		      spdk_thread_get_name(spdk_get_thread()));
+
+	/* This gets unregistered asynchronously with the device -
+	 * we can't just use the ctx buffer passed by the thread library
+	 */
+	ioch = calloc(1, sizeof(*ioch));
+	if (ioch == NULL) {
+		FTL_ERRLOG(dev, "Failed to allocate IO channel\n");
+		return -1;
+	}
+
+	rc = snprintf(mempool_name, sizeof(mempool_name), "ftl_io_%p", ioch);
+	if (rc < 0 || rc >= (int)sizeof(mempool_name)) {
+		FTL_ERRLOG(dev, "Failed to create IO channel pool name\n");
+		free(ioch);
+		return -1;
+	}
+
+	ioch->dev = dev;
+
+	ioch->map_pool = ftl_mempool_create(
+				 dev->conf.user_io_pool_size,
+				 sizeof(ftl_addr) * dev->xfer_size,
+				 64,
+				 SPDK_ENV_SOCKET_ID_ANY);
+	if (!ioch->map_pool) {
+		FTL_ERRLOG(dev, "Failed to create IO channel's  map IO pool\n");
+		goto fail_io_pool;
+	}
+
+	ioch->cq = spdk_ring_create(SPDK_RING_TYPE_SP_SC, spdk_align64pow2(dev->conf.user_io_pool_size + 1),
+				    SPDK_ENV_SOCKET_ID_ANY);
+	if (!ioch->cq) {
+		FTL_ERRLOG(dev, "Failed to create IO channel completion queue\n");
+		goto fail_io_pool;
+	}
+
+	ioch->sq = spdk_ring_create(SPDK_RING_TYPE_SP_SC, spdk_align64pow2(dev->conf.user_io_pool_size + 1),
+				    SPDK_ENV_SOCKET_ID_ANY);
+	if (!ioch->sq) {
+		FTL_ERRLOG(dev, "Failed to create IO channel submission queue\n");
+		goto fail_cq;
+	}
+
+	ioch->poller = SPDK_POLLER_REGISTER(ftl_io_channel_poll, ioch, 0);
+	if (!ioch->poller) {
+		FTL_ERRLOG(dev, "Failed to register IO channel poller\n");
+		goto fail_sq;
+	}
+
+	if (spdk_thread_send_msg(dev->core_thread, ftl_dev_register_channel, ioch)) {
+		FTL_ERRLOG(dev, "Failed to register IO channel\n");
+		goto fail_poller;
+	}
+
+	_ioch->ioch = ioch;
+	return 0;
+
+fail_poller:
+	spdk_poller_unregister(&ioch->poller);
+fail_cq:
+	spdk_ring_free(ioch->cq);
+fail_sq:
+	spdk_ring_free(ioch->sq);
+fail_io_pool:
+	ftl_mempool_destroy(ioch->map_pool);
+	free(ioch);
+
+	return -1;
+}
+
+static void
+io_channel_destroy_cb(void *io_device, void *ctx)
+{
+	struct ftl_io_channel_ctx *_ioch = ctx;
+	struct ftl_io_channel *ioch = _ioch->ioch;
+	struct spdk_ftl_dev *dev = ioch->dev;
+
+	FTL_NOTICELOG(dev, "FTL IO channel destroy on %s\n",
+		      spdk_thread_get_name(spdk_get_thread()));
+
+	spdk_poller_unregister(&ioch->poller);
+	spdk_thread_send_msg(ftl_get_core_thread(dev),
+			     io_channel_unregister, ioch);
+}
+
+void
+ftl_mngt_register_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	dev->io_device_registered = true;
+
+	spdk_io_device_register(dev, io_channel_create_cb,
+				io_channel_destroy_cb,
+				sizeof(struct ftl_io_channel_ctx),
+				NULL);
+
+	ftl_mngt_next_step(mngt);
+}
+
+static void
+unregister_cb(void *io_device)
+{
+	struct spdk_ftl_dev *dev = io_device;
+	struct ftl_mngt_process *mngt = dev->unregister_process;
+
+	dev->io_device_registered = false;
+	dev->unregister_process = NULL;
+
+	ftl_mngt_next_step(mngt);
+}
+
+void
+ftl_mngt_unregister_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	if (dev->io_device_registered) {
+		dev->unregister_process = mngt;
+		spdk_io_device_unregister(dev, unregister_cb);
+	} else {
+		ftl_mngt_skip_step(mngt);
+	}
+}
+
+void
+ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	dev->ioch = spdk_get_io_channel(dev);
+	if (!dev->ioch) {
+		FTL_ERRLOG(dev, "Unable to get IO channel for core thread");
+		ftl_mngt_fail_step(mngt);
+		return;
+	}
+
+	ftl_mngt_next_step(mngt);
+}
+
+void
+ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	if (dev->ioch) {
+		spdk_put_io_channel(dev->ioch);
+		dev->ioch = NULL;
+	}
+
+	ftl_mngt_next_step(mngt);
+}

lib/ftl/mngt/ftl_mngt_shutdown.c

@@ -15,6 +15,14 @@ static const struct ftl_mngt_process_desc desc_shutdown = {
 	.name = "FTL shutdown",
 	.error_handler = ftl_mngt_rollback_device,
 	.steps = {
+		{
+			.name = "Deinit core IO channel",
+			.action = ftl_mngt_deinit_io_channel
+		},
+		{
+			.name = "Unregister IO device",
+			.action = ftl_mngt_unregister_io_device
+		},
 		{
 			.name = "Stop core poller",
 			.action = ftl_mngt_stop_core_poller

lib/ftl/mngt/ftl_mngt_startup.c

@@ -42,6 +42,16 @@ static const struct ftl_mngt_process_desc desc_startup = {
 			.action = ftl_mngt_open_cache_bdev,
 			.cleanup = ftl_mngt_close_cache_bdev
 		},
+		{
+			.name = "Register IO device",
+			.action = ftl_mngt_register_io_device,
+			.cleanup = ftl_mngt_unregister_io_device
+		},
+		{
+			.name = "Initialize core IO channel",
+			.action = ftl_mngt_init_io_channel,
+			.cleanup = ftl_mngt_deinit_io_channel
+		},
 		{
 			.name = "Initialize layout",
 			.action = ftl_mngt_init_layout

lib/ftl/mngt/ftl_mngt_steps.h

@@ -18,6 +18,14 @@ void ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process
 
 void ftl_mngt_close_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_register_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
+void ftl_mngt_unregister_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
+void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
+void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);