diff --git a/include/spdk/ftl.h b/include/spdk/ftl.h index 28ecf7f3a..a2ac2952e 100644 --- a/include/spdk/ftl.h +++ b/include/spdk/ftl.h @@ -134,6 +134,13 @@ void spdk_ftl_conf_deinit(struct spdk_ftl_conf *conf); */ void spdk_ftl_get_default_conf(struct spdk_ftl_conf *conf); +/** + * Returns the size of ftl_io struct that needs to be passed to spdk_ftl_read/write + * + * \return The size of struct + */ +size_t spdk_ftl_io_size(void); + #ifdef __cplusplus } #endif diff --git a/lib/ftl/Makefile b/lib/ftl/Makefile index 31b001f61..9df1202fd 100644 --- a/lib/ftl/Makefile +++ b/lib/ftl/Makefile @@ -17,7 +17,7 @@ CFLAGS += -I. FTL_SUBDIRS := mngt utils -C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c +C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 72da507c5..a77f692d5 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -13,11 +13,19 @@ #include "spdk/crc32.h" #include "ftl_core.h" +#include "ftl_io.h" #include "ftl_debug.h" #include "ftl_internal.h" #include "mngt/ftl_mngt.h" #include "utils/ftl_mempool.h" + +size_t +spdk_ftl_io_size(void) +{ + return sizeof(struct ftl_io); +} + static int ftl_shutdown_complete(struct spdk_ftl_dev *dev) { diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 934005516..bf1b1e476 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -17,6 +17,7 @@ #include "spdk/bdev_zone.h" #include "ftl_internal.h" +#include "ftl_io.h" #include "ftl_layout.h" #include "utils/ftl_log.h" @@ -139,4 +140,10 @@ ftl_get_num_zones(const struct spdk_ftl_dev *dev) return ftl_get_num_bands(dev) * ftl_get_num_zones_in_band(dev); } +static inline bool +ftl_check_core_thread(const struct spdk_ftl_dev *dev) +{ + return dev->core_thread == spdk_get_thread(); +} + #endif /* FTL_CORE_H */ diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index 4695eefa7..74755acc4 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -16,6 +16,7 @@ #include "spdk/config.h" #include "ftl_core.h" +#include "ftl_io.h" #include "ftl_debug.h" #include "ftl_utils.h" #include "mngt/ftl_mngt.h" diff --git a/lib/ftl/ftl_internal.h b/lib/ftl/ftl_internal.h index 2e978e38e..8caa62fbd 100644 --- a/lib/ftl/ftl_internal.h +++ b/lib/ftl/ftl_internal.h @@ -13,6 +13,8 @@ /* Marks address as invalid */ #define FTL_ADDR_INVALID ((ftl_addr)-1) +/* Marks LBA as invalid */ +#define FTL_LBA_INVALID ((uint64_t)-1) /* Smallest data unit size */ #define FTL_BLOCK_SIZE 4096ULL diff --git a/lib/ftl/ftl_io.c b/lib/ftl/ftl_io.c new file mode 100644 index 000000000..9290968ed --- /dev/null +++ b/lib/ftl/ftl_io.c @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "spdk/stdinc.h" +#include "spdk/ftl.h" +#include "spdk/likely.h" +#include "spdk/util.h" + +#include "ftl_io.h" +#include "ftl_core.h" +#include "ftl_debug.h" +#include "utils/ftl_mempool.h" + +void +ftl_io_inc_req(struct ftl_io *io) +{ + io->dev->num_inflight++; + io->req_cnt++; +} + +void +ftl_io_dec_req(struct ftl_io *io) +{ + assert(io->dev->num_inflight > 0); + assert(io->req_cnt > 0); + + io->dev->num_inflight--; + io->req_cnt--; +} + +struct iovec * +ftl_io_iovec(struct ftl_io *io) +{ + return &io->iov[0]; +} + +uint64_t +ftl_io_get_lba(const struct ftl_io *io, size_t offset) +{ + assert(offset < io->num_blocks); + return io->lba + offset; +} + +uint64_t +ftl_io_current_lba(const struct ftl_io *io) +{ + return ftl_io_get_lba(io, io->pos); +} + +void +ftl_io_advance(struct ftl_io *io, size_t num_blocks) +{ + struct iovec *iov = ftl_io_iovec(io); + size_t iov_blocks, block_left = num_blocks; + + io->pos += num_blocks; + + if (io->iov_cnt == 0) { + return; + } + + while (block_left > 0) { + assert(io->iov_pos < io->iov_cnt); + iov_blocks = iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE; + + if (io->iov_off + block_left < iov_blocks) { + io->iov_off += block_left; + break; + } + + assert(iov_blocks > io->iov_off); + block_left -= (iov_blocks - io->iov_off); + io->iov_off = 0; + io->iov_pos++; + } +} + +size_t +ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt) +{ + size_t num_blocks = 0, i = 0; + + for (; i < iov_cnt; ++i) { + if (iov[i].iov_len & (FTL_BLOCK_SIZE - 1)) { + return 0; + } + + num_blocks += iov[i].iov_len / FTL_BLOCK_SIZE; + } + + return num_blocks; +} + +void * +ftl_io_iovec_addr(struct ftl_io *io) +{ + assert(io->iov_pos < io->iov_cnt); + assert(io->iov_off * FTL_BLOCK_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len); + + return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base + + io->iov_off * FTL_BLOCK_SIZE; +} + +size_t +ftl_io_iovec_len_left(struct ftl_io *io) +{ + if (io->iov_pos < io->iov_cnt) { + struct iovec *iov = ftl_io_iovec(io); + return iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE - io->iov_off; + } else { + return 0; + } +} + +static void +ftl_io_cb(struct ftl_io *io, void *arg, int status) +{ + if (spdk_unlikely(status)) { + io->status = status; + + if (-EAGAIN == status) { + /* IO has to be rescheduled again */ + switch (io->type) { + case FTL_IO_READ: + ftl_io_clear(io); + TAILQ_INSERT_HEAD(&io->dev->rd_sq, io, queue_entry); + break; + case FTL_IO_WRITE: + ftl_io_clear(io); + TAILQ_INSERT_HEAD(&io->dev->wr_sq, io, queue_entry); + break; + default: + /* Unknown IO type, complete to the user */ + assert(0); + break; + } + + } + + if (!io->status) { + /* IO rescheduled, return from the function */ + return; + } + } + + /* User completion added in next patch */ +} + +int +ftl_io_init(struct spdk_io_channel *_ioch, struct ftl_io *io, uint64_t lba, size_t num_blocks, + struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type) +{ + /* dev initialized from io channel context in next patch */ + struct spdk_ftl_dev *dev = NULL; + + memset(io, 0, sizeof(struct ftl_io)); + io->ioch = _ioch; + + io->flags |= FTL_IO_INITIALIZED; + io->type = type; + io->dev = dev; + io->lba = FTL_LBA_INVALID; + io->addr = FTL_ADDR_INVALID; + io->cb_ctx = cb_ctx; + io->lba = lba; + io->user_fn = cb_fn; + io->iov = iov; + io->iov_cnt = iov_cnt; + io->num_blocks = num_blocks; + + return 0; +} + +void +ftl_io_complete(struct ftl_io *io) +{ + io->flags &= ~FTL_IO_INITIALIZED; + io->done = true; + + ftl_io_cb(io, io->cb_ctx, io->status); +} + +void +ftl_io_fail(struct ftl_io *io, int status) +{ + io->status = status; + ftl_io_advance(io, io->num_blocks - io->pos); +} + +void +ftl_io_clear(struct ftl_io *io) +{ + io->req_cnt = io->pos = io->iov_pos = io->iov_off = 0; + io->done = false; + io->status = 0; + io->flags = 0; +} diff --git a/lib/ftl/ftl_io.h b/lib/ftl/ftl_io.h new file mode 100644 index 000000000..23e7e7767 --- /dev/null +++ b/lib/ftl/ftl_io.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#ifndef FTL_IO_H +#define FTL_IO_H + +#include "spdk/stdinc.h" +#include "spdk/nvme.h" +#include "spdk/ftl.h" +#include "spdk/bdev.h" +#include "spdk/util.h" + +#include "ftl_internal.h" +#include "utils/ftl_md.h" + +struct spdk_ftl_dev; +struct ftl_io; + +typedef void (*ftl_io_fn)(struct ftl_io *, void *, int); + +/* IO flags */ +enum ftl_io_flags { + /* Indicates whether IO is already initialized */ + FTL_IO_INITIALIZED = (1 << 0), +}; + +enum ftl_io_type { + FTL_IO_READ, + FTL_IO_WRITE, + FTL_IO_UNMAP, +}; + +#define FTL_IO_MAX_IOVEC 4 + +/* General IO descriptor */ +struct ftl_io { + /* Device */ + struct spdk_ftl_dev *dev; + + /* IO channel */ + struct spdk_io_channel *ioch; + + /* LBA address */ + uint64_t lba; + + /* First address of write when sent to cache device */ + ftl_addr addr; + + /* Number of processed blocks */ + size_t pos; + + /* Number of blocks */ + size_t num_blocks; + + /* IO vector pointer */ + struct iovec *iov; + + /* Metadata */ + void *md; + + /* Number of IO vectors */ + size_t iov_cnt; + + /* Position within the io vector array */ + size_t iov_pos; + + /* Offset within the iovec (in blocks) */ + size_t iov_off; + + /* Request status */ + int status; + + /* Number of split requests */ + size_t req_cnt; + + /* Callback's context */ + void *cb_ctx; + + /* User callback function */ + spdk_ftl_fn user_fn; + + /* Flags */ + int flags; + + /* IO type */ + enum ftl_io_type type; + + /* Done flag */ + bool done; + + /* Used by retry and write completion queues */ + TAILQ_ENTRY(ftl_io) queue_entry; + + /* Reference to the chunk within NV cache */ + struct ftl_nv_cache_chunk *nv_cache_chunk; + + /* Logical to physical mapping for this IO, number of entries equals to + * number of transfer blocks */ + ftl_addr *map; + + struct spdk_bdev_io_wait_entry bdev_io_wait; +}; + +void ftl_io_fail(struct ftl_io *io, int status); +void ftl_io_clear(struct ftl_io *io); +void ftl_io_inc_req(struct ftl_io *io); +void ftl_io_dec_req(struct ftl_io *io); +struct iovec *ftl_io_iovec(struct ftl_io *io); +uint64_t ftl_io_current_lba(const struct ftl_io *io); +uint64_t ftl_io_get_lba(const struct ftl_io *io, size_t offset); +void ftl_io_advance(struct ftl_io *io, size_t num_blocks); +size_t ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt); +void *ftl_io_iovec_addr(struct ftl_io *io); +size_t ftl_io_iovec_len_left(struct ftl_io *io); +int ftl_io_init(struct spdk_io_channel *ioch, struct ftl_io *io, uint64_t lba, + size_t num_blocks, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, + void *cb_arg, int type); +void ftl_io_complete(struct ftl_io *io); + +static inline bool +ftl_io_done(const struct ftl_io *io) +{ + return io->req_cnt == 0 && io->pos == io->num_blocks; +} + +#endif /* FTL_IO_H */ diff --git a/lib/ftl/mngt/ftl_mngt_bdev.c b/lib/ftl/mngt/ftl_mngt_bdev.c index b9ff48b44..e07dc6839 100644 --- a/lib/ftl/mngt/ftl_mngt_bdev.c +++ b/lib/ftl/mngt/ftl_mngt_bdev.c @@ -11,7 +11,6 @@ #include "ftl_internal.h" #include "ftl_core.h" #include "utils/ftl_defs.h" -#include "utils/ftl_md.h" #define MINIMUM_CACHE_SIZE_GIB 5 #define MINIMUM_BASE_SIZE_GIB 20 diff --git a/lib/ftl/spdk_ftl.map b/lib/ftl/spdk_ftl.map index 55d78b92a..e3ba837e5 100644 --- a/lib/ftl/spdk_ftl.map +++ b/lib/ftl/spdk_ftl.map @@ -10,6 +10,7 @@ spdk_ftl_conf_copy; spdk_ftl_conf_deinit; spdk_ftl_get_io_channel; + spdk_ftl_io_size; local: *; }; diff --git a/module/bdev/ftl/bdev_ftl.c b/module/bdev/ftl/bdev_ftl.c index 61f912167..837e101f1 100644 --- a/module/bdev/ftl/bdev_ftl.c +++ b/module/bdev/ftl/bdev_ftl.c @@ -41,7 +41,7 @@ static void bdev_ftl_examine(struct spdk_bdev *bdev); static int bdev_ftl_get_ctx_size(void) { - return 0; + return spdk_ftl_io_size(); } static struct spdk_bdev_module g_ftl_if = {