FTL: Add ftl_io helper structure

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I608b500c6fb14efe289932955f508484f2ecf1b6
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13305
Community-CI: Mellanox Build Bot
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

include/spdk/ftl.h

@@ -134,6 +134,13 @@ void spdk_ftl_conf_deinit(struct spdk_ftl_conf *conf);
  */
 void spdk_ftl_get_default_conf(struct spdk_ftl_conf *conf);
 
+/**
+ * Returns the size of ftl_io struct that needs to be passed to spdk_ftl_read/write
+ *
+ * \return The size of struct
+ */
+size_t spdk_ftl_io_size(void);
+
 #ifdef __cplusplus
 }
 #endif

lib/ftl/Makefile

@@ -17,7 +17,7 @@ CFLAGS += -I.
 
 FTL_SUBDIRS := mngt utils
 
-C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c
+C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c
 C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c
 C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c
 C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c

lib/ftl/ftl_core.c

@@ -13,11 +13,19 @@
 #include "spdk/crc32.h"
 
 #include "ftl_core.h"
+#include "ftl_io.h"
 #include "ftl_debug.h"
 #include "ftl_internal.h"
 #include "mngt/ftl_mngt.h"
 #include "utils/ftl_mempool.h"
 
+
+size_t
+spdk_ftl_io_size(void)
+{
+	return sizeof(struct ftl_io);
+}
+
 static int
 ftl_shutdown_complete(struct spdk_ftl_dev *dev)
 {

lib/ftl/ftl_core.h

@@ -17,6 +17,7 @@
 #include "spdk/bdev_zone.h"
 
 #include "ftl_internal.h"
+#include "ftl_io.h"
 #include "ftl_layout.h"
 #include "utils/ftl_log.h"
 
@@ -139,4 +140,10 @@ ftl_get_num_zones(const struct spdk_ftl_dev *dev)
 	return ftl_get_num_bands(dev) * ftl_get_num_zones_in_band(dev);
 }
 
+static inline bool
+ftl_check_core_thread(const struct spdk_ftl_dev *dev)
+{
+	return dev->core_thread == spdk_get_thread();
+}
+
 #endif /* FTL_CORE_H */

lib/ftl/ftl_init.c

@@ -16,6 +16,7 @@
 #include "spdk/config.h"
 
 #include "ftl_core.h"
+#include "ftl_io.h"
 #include "ftl_debug.h"
 #include "ftl_utils.h"
 #include "mngt/ftl_mngt.h"

lib/ftl/ftl_internal.h

@@ -13,6 +13,8 @@
 
 /* Marks address as invalid */
 #define FTL_ADDR_INVALID	((ftl_addr)-1)
+/* Marks LBA as invalid */
+#define FTL_LBA_INVALID		((uint64_t)-1)
 /* Smallest data unit size */
 #define FTL_BLOCK_SIZE		4096ULL
 

lib/ftl/ftl_io.c

@@ -0,0 +1,199 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/ftl.h"
+#include "spdk/likely.h"
+#include "spdk/util.h"
+
+#include "ftl_io.h"
+#include "ftl_core.h"
+#include "ftl_debug.h"
+#include "utils/ftl_mempool.h"
+
+void
+ftl_io_inc_req(struct ftl_io *io)
+{
+	io->dev->num_inflight++;
+	io->req_cnt++;
+}
+
+void
+ftl_io_dec_req(struct ftl_io *io)
+{
+	assert(io->dev->num_inflight > 0);
+	assert(io->req_cnt > 0);
+
+	io->dev->num_inflight--;
+	io->req_cnt--;
+}
+
+struct iovec *
+ftl_io_iovec(struct ftl_io *io)
+{
+	return &io->iov[0];
+}
+
+uint64_t
+ftl_io_get_lba(const struct ftl_io *io, size_t offset)
+{
+	assert(offset < io->num_blocks);
+	return io->lba + offset;
+}
+
+uint64_t
+ftl_io_current_lba(const struct ftl_io *io)
+{
+	return ftl_io_get_lba(io, io->pos);
+}
+
+void
+ftl_io_advance(struct ftl_io *io, size_t num_blocks)
+{
+	struct iovec *iov = ftl_io_iovec(io);
+	size_t iov_blocks, block_left = num_blocks;
+
+	io->pos += num_blocks;
+
+	if (io->iov_cnt == 0) {
+		return;
+	}
+
+	while (block_left > 0) {
+		assert(io->iov_pos < io->iov_cnt);
+		iov_blocks = iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE;
+
+		if (io->iov_off + block_left < iov_blocks) {
+			io->iov_off += block_left;
+			break;
+		}
+
+		assert(iov_blocks > io->iov_off);
+		block_left -= (iov_blocks - io->iov_off);
+		io->iov_off = 0;
+		io->iov_pos++;
+	}
+}
+
+size_t
+ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt)
+{
+	size_t num_blocks = 0, i = 0;
+
+	for (; i < iov_cnt; ++i) {
+		if (iov[i].iov_len & (FTL_BLOCK_SIZE - 1)) {
+			return 0;
+		}
+
+		num_blocks += iov[i].iov_len / FTL_BLOCK_SIZE;
+	}
+
+	return num_blocks;
+}
+
+void *
+ftl_io_iovec_addr(struct ftl_io *io)
+{
+	assert(io->iov_pos < io->iov_cnt);
+	assert(io->iov_off * FTL_BLOCK_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len);
+
+	return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base +
+	       io->iov_off * FTL_BLOCK_SIZE;
+}
+
+size_t
+ftl_io_iovec_len_left(struct ftl_io *io)
+{
+	if (io->iov_pos < io->iov_cnt) {
+		struct iovec *iov = ftl_io_iovec(io);
+		return iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE - io->iov_off;
+	} else {
+		return 0;
+	}
+}
+
+static void
+ftl_io_cb(struct ftl_io *io, void *arg, int status)
+{
+	if (spdk_unlikely(status)) {
+		io->status = status;
+
+		if (-EAGAIN == status) {
+			/* IO has to be rescheduled again */
+			switch (io->type) {
+			case FTL_IO_READ:
+				ftl_io_clear(io);
+				TAILQ_INSERT_HEAD(&io->dev->rd_sq, io, queue_entry);
+				break;
+			case FTL_IO_WRITE:
+				ftl_io_clear(io);
+				TAILQ_INSERT_HEAD(&io->dev->wr_sq, io, queue_entry);
+				break;
+			default:
+				/* Unknown IO type, complete to the user */
+				assert(0);
+				break;
+			}
+
+		}
+
+		if (!io->status) {
+			/* IO rescheduled, return from the function */
+			return;
+		}
+	}
+
+	/* User completion added in next patch */
+}
+
+int
+ftl_io_init(struct spdk_io_channel *_ioch, struct ftl_io *io, uint64_t lba, size_t num_blocks,
+	    struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type)
+{
+	/* dev initialized from io channel context in next patch */
+	struct spdk_ftl_dev *dev = NULL;
+
+	memset(io, 0, sizeof(struct ftl_io));
+	io->ioch = _ioch;
+
+	io->flags |= FTL_IO_INITIALIZED;
+	io->type = type;
+	io->dev = dev;
+	io->lba = FTL_LBA_INVALID;
+	io->addr = FTL_ADDR_INVALID;
+	io->cb_ctx = cb_ctx;
+	io->lba = lba;
+	io->user_fn = cb_fn;
+	io->iov = iov;
+	io->iov_cnt = iov_cnt;
+	io->num_blocks = num_blocks;
+
+	return 0;
+}
+
+void
+ftl_io_complete(struct ftl_io *io)
+{
+	io->flags &= ~FTL_IO_INITIALIZED;
+	io->done = true;
+
+	ftl_io_cb(io, io->cb_ctx, io->status);
+}
+
+void
+ftl_io_fail(struct ftl_io *io, int status)
+{
+	io->status = status;
+	ftl_io_advance(io, io->num_blocks - io->pos);
+}
+
+void
+ftl_io_clear(struct ftl_io *io)
+{
+	io->req_cnt = io->pos = io->iov_pos = io->iov_off = 0;
+	io->done = false;
+	io->status = 0;
+	io->flags = 0;
+}

lib/ftl/ftl_io.h

@@ -0,0 +1,128 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#ifndef FTL_IO_H
+#define FTL_IO_H
+
+#include "spdk/stdinc.h"
+#include "spdk/nvme.h"
+#include "spdk/ftl.h"
+#include "spdk/bdev.h"
+#include "spdk/util.h"
+
+#include "ftl_internal.h"
+#include "utils/ftl_md.h"
+
+struct spdk_ftl_dev;
+struct ftl_io;
+
+typedef void (*ftl_io_fn)(struct ftl_io *, void *, int);
+
+/* IO flags */
+enum ftl_io_flags {
+	/* Indicates whether IO is already initialized */
+	FTL_IO_INITIALIZED	= (1 << 0),
+};
+
+enum ftl_io_type {
+	FTL_IO_READ,
+	FTL_IO_WRITE,
+	FTL_IO_UNMAP,
+};
+
+#define FTL_IO_MAX_IOVEC 4
+
+/* General IO descriptor */
+struct ftl_io {
+	/* Device */
+	struct spdk_ftl_dev		*dev;
+
+	/* IO channel */
+	struct spdk_io_channel		*ioch;
+
+	/* LBA address */
+	uint64_t			lba;
+
+	/* First address of write when sent to cache device */
+	ftl_addr			addr;
+
+	/* Number of processed blocks */
+	size_t				pos;
+
+	/* Number of blocks */
+	size_t				num_blocks;
+
+	/* IO vector pointer */
+	struct iovec			*iov;
+
+	/* Metadata */
+	void				*md;
+
+	/* Number of IO vectors */
+	size_t				iov_cnt;
+
+	/* Position within the io vector array */
+	size_t				iov_pos;
+
+	/* Offset within the iovec (in blocks) */
+	size_t				iov_off;
+
+	/* Request status */
+	int				status;
+
+	/* Number of split requests */
+	size_t				req_cnt;
+
+	/* Callback's context */
+	void				*cb_ctx;
+
+	/* User callback function */
+	spdk_ftl_fn			user_fn;
+
+	/* Flags */
+	int				flags;
+
+	/* IO type */
+	enum ftl_io_type		type;
+
+	/* Done flag */
+	bool				done;
+
+	/* Used by retry and write completion queues */
+	TAILQ_ENTRY(ftl_io)		queue_entry;
+
+	/* Reference to the chunk within NV cache */
+	struct ftl_nv_cache_chunk	*nv_cache_chunk;
+
+	/* Logical to physical mapping for this IO, number of entries equals to
+	 * number of transfer blocks */
+	ftl_addr			*map;
+
+	struct spdk_bdev_io_wait_entry	bdev_io_wait;
+};
+
+void ftl_io_fail(struct ftl_io *io, int status);
+void ftl_io_clear(struct ftl_io *io);
+void ftl_io_inc_req(struct ftl_io *io);
+void ftl_io_dec_req(struct ftl_io *io);
+struct iovec *ftl_io_iovec(struct ftl_io *io);
+uint64_t ftl_io_current_lba(const struct ftl_io *io);
+uint64_t ftl_io_get_lba(const struct ftl_io *io, size_t offset);
+void ftl_io_advance(struct ftl_io *io, size_t num_blocks);
+size_t ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt);
+void *ftl_io_iovec_addr(struct ftl_io *io);
+size_t ftl_io_iovec_len_left(struct ftl_io *io);
+int ftl_io_init(struct spdk_io_channel *ioch, struct ftl_io *io, uint64_t lba,
+		size_t num_blocks, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn,
+		void *cb_arg, int type);
+void ftl_io_complete(struct ftl_io *io);
+
+static inline bool
+ftl_io_done(const struct ftl_io *io)
+{
+	return io->req_cnt == 0 && io->pos == io->num_blocks;
+}
+
+#endif /* FTL_IO_H */

lib/ftl/mngt/ftl_mngt_bdev.c

@@ -11,7 +11,6 @@
 #include "ftl_internal.h"
 #include "ftl_core.h"
 #include "utils/ftl_defs.h"
-#include "utils/ftl_md.h"
 
 #define MINIMUM_CACHE_SIZE_GIB 5
 #define MINIMUM_BASE_SIZE_GIB 20

lib/ftl/spdk_ftl.map

@@ -10,6 +10,7 @@
 	spdk_ftl_conf_copy;
 	spdk_ftl_conf_deinit;
 	spdk_ftl_get_io_channel;
+	spdk_ftl_io_size;
 
 	local: *;
 };

module/bdev/ftl/bdev_ftl.c

@@ -41,7 +41,7 @@ static void bdev_ftl_examine(struct spdk_bdev *bdev);
 static int
 bdev_ftl_get_ctx_size(void)
 {
-	return 0;
+	return spdk_ftl_io_size();
 }
 
 static struct spdk_bdev_module g_ftl_if = {