diff --git a/lib/ftl/Makefile b/lib/ftl/Makefile index 48213a415..f1711b520 100644 --- a/lib/ftl/Makefile +++ b/lib/ftl/Makefile @@ -22,7 +22,7 @@ CFLAGS += -I. FTL_SUBDIRS := mngt utils C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c ftl_sb.c ftl_l2p.c ftl_l2p_flat.c -C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c +C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c ftl_writer.c C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c mngt/ftl_mngt_l2p.c C_SRCS += mngt/ftl_mngt_band.c diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 1e52c6e6a..0ddf3bd3e 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -73,6 +73,16 @@ ftl_shutdown_complete(struct spdk_ftl_dev *dev) return false; } + if (!ftl_writer_is_halted(&dev->writer_user)) { + ftl_writer_halt(&dev->writer_user); + return false; + } + + if (!ftl_writer_is_halted(&dev->writer_gc)) { + ftl_writer_halt(&dev->writer_gc); + return false; + } + if (!ftl_nv_cache_chunks_busy(&dev->nv_cache)) { return false; } @@ -481,6 +491,8 @@ ftl_core_poller(void *ctx) } ftl_process_io_queue(dev); + ftl_writer_run(&dev->writer_user); + ftl_writer_run(&dev->writer_gc); ftl_nv_cache_process(dev); ftl_l2p_process(dev); diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 4b9a71800..e6112fecd 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -18,6 +18,7 @@ #include "ftl_internal.h" #include "ftl_io.h" #include "ftl_nv_cache.h" +#include "ftl_writer.h" #include "ftl_layout.h" #include "ftl_sb.h" #include "ftl_l2p.h" @@ -138,6 +139,12 @@ struct spdk_ftl_dev { /* Write submission queue */ TAILQ_HEAD(, ftl_io) wr_sq; + + /* Writer for user IOs */ + struct ftl_writer writer_user; + + /* Writer for GC IOs */ + struct ftl_writer writer_gc; }; void ftl_apply_limits(struct spdk_ftl_dev *dev); diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index 6e9dc51ab..96d2a0087 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -19,6 +19,7 @@ #include "ftl_band.h" #include "ftl_debug.h" #include "ftl_nv_cache.h" +#include "ftl_writer.h" #include "ftl_utils.h" #include "mngt/ftl_mngt.h" @@ -118,6 +119,9 @@ allocate_dev(const struct spdk_ftl_conf *conf, int *error) TAILQ_INIT(&dev->wr_sq); TAILQ_INIT(&dev->ioch_queue); + ftl_writer_init(dev, &dev->writer_user, SPDK_FTL_LIMIT_HIGH, FTL_BAND_TYPE_COMPACTION); + ftl_writer_init(dev, &dev->writer_gc, SPDK_FTL_LIMIT_CRIT, FTL_BAND_TYPE_GC); + return dev; error: free_dev(dev); diff --git a/lib/ftl/ftl_writer.c b/lib/ftl/ftl_writer.c new file mode 100644 index 000000000..831768ac2 --- /dev/null +++ b/lib/ftl/ftl_writer.c @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "spdk/likely.h" + +#include "ftl_writer.h" +#include "ftl_band.h" + +void +ftl_writer_init(struct spdk_ftl_dev *dev, struct ftl_writer *writer, + uint64_t limit, enum ftl_band_type type) +{ + memset(writer, 0, sizeof(*writer)); + writer->dev = dev; + TAILQ_INIT(&writer->rq_queue); + TAILQ_INIT(&writer->full_bands); + writer->limit = limit; + writer->halt = true; + writer->writer_type = type; +} + +static bool +can_write(struct ftl_writer *writer) +{ + if (spdk_unlikely(writer->halt)) { + return false; + } + + return writer->band->md->state == FTL_BAND_STATE_OPEN; +} + +void +ftl_writer_band_state_change(struct ftl_band *band) +{ + struct ftl_writer *writer = band->owner.priv; + + switch (band->md->state) { + case FTL_BAND_STATE_FULL: + assert(writer->band == band); + TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry); + writer->band = NULL; + break; + + case FTL_BAND_STATE_CLOSED: + assert(writer->num_bands > 0); + writer->num_bands--; + ftl_band_clear_owner(band, ftl_writer_band_state_change, writer); + break; + + default: + break; + } +} + +static void +close_full_bands(struct ftl_writer *writer) +{ + struct ftl_band *band, *next; + + TAILQ_FOREACH_SAFE(band, &writer->full_bands, queue_entry, next) { + if (band->queue_depth) { + continue; + } + + TAILQ_REMOVE(&writer->full_bands, band, queue_entry); + ftl_band_close(band); + } +} + +static bool +is_active(struct ftl_writer *writer) +{ + if (writer->dev->limit < writer->limit) { + return false; + } + + return true; +} + +static struct ftl_band * +get_band(struct ftl_writer *writer) +{ + if (spdk_unlikely(!writer->band)) { + if (!is_active(writer)) { + return NULL; + } + + if (spdk_unlikely(NULL != writer->next_band)) { + if (FTL_BAND_STATE_OPEN == writer->next_band->md->state) { + writer->band = writer->next_band; + writer->next_band = NULL; + + return writer->band; + } else { + assert(FTL_BAND_STATE_OPEN == writer->next_band->md->state); + ftl_abort(); + } + } + + writer->band = ftl_band_get_next_free(writer->dev); + if (writer->band) { + writer->num_bands++; + ftl_band_set_owner(writer->band, + ftl_writer_band_state_change, writer); + + if (ftl_band_write_prep(writer->band)) { + /* + * This error might happen due to allocation failure. However number + * of open bands is controlled and it should have enough resources + * to do it. So here is better to perform a crash and recover from + * shared memory to bring back stable state. + * */ + ftl_abort(); + } + } else { + return NULL; + } + } + + if (spdk_likely(writer->band->md->state == FTL_BAND_STATE_OPEN)) { + return writer->band; + } else { + if (spdk_unlikely(writer->band->md->state == FTL_BAND_STATE_PREP)) { + ftl_band_open(writer->band, writer->writer_type); + } + return NULL; + } +} + +void +ftl_writer_run(struct ftl_writer *writer) +{ + struct ftl_band *band; + struct ftl_rq *rq; + + close_full_bands(writer); + + if (!TAILQ_EMPTY(&writer->rq_queue)) { + band = get_band(writer); + if (spdk_unlikely(!band)) { + return; + } + + if (!can_write(writer)) { + return; + } + + /* Finally we can write to band */ + rq = TAILQ_FIRST(&writer->rq_queue); + TAILQ_REMOVE(&writer->rq_queue, rq, qentry); + ftl_band_rq_write(writer->band, rq); + } +} + +bool +ftl_writer_is_halted(struct ftl_writer *writer) +{ + if (spdk_unlikely(!TAILQ_EMPTY(&writer->full_bands))) { + return false; + } + + if (writer->band) { + if (writer->band->md->state != FTL_BAND_STATE_OPEN) { + return false; + } + + if (writer->band->queue_depth) { + return false; + } + } + + return writer->halt; +} + +uint64_t +ftl_writer_get_free_blocks(struct ftl_writer *writer) +{ + uint64_t free_blocks = 0; + + if (writer->band) { + free_blocks += ftl_band_user_blocks_left(writer->band, + writer->band->md->iter.offset); + } + + if (writer->next_band) { + free_blocks += ftl_band_user_blocks_left(writer->next_band, + writer->next_band->md->iter.offset); + } + + return free_blocks; +} diff --git a/lib/ftl/ftl_writer.h b/lib/ftl/ftl_writer.h new file mode 100644 index 000000000..fe6f11e71 --- /dev/null +++ b/lib/ftl/ftl_writer.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#ifndef FTL_WRITER_H +#define FTL_WRITER_H + +#include "spdk/stdinc.h" +#include "spdk/queue.h" + +#include "ftl_io.h" + +struct ftl_writer { + struct spdk_ftl_dev *dev; + + TAILQ_HEAD(, ftl_rq) rq_queue; + + /* Band currently being written to */ + struct ftl_band *band; + + /* Number of bands associated with writer */ + uint64_t num_bands; + + /* Band next being written to */ + struct ftl_band *next_band; + + /* List of full bands */ + TAILQ_HEAD(, ftl_band) full_bands; + + /* FTL band limit which blocks writes */ + int limit; + + /* Flag indicating halt has been requested */ + bool halt; + + /* Which type of band the writer uses */ + enum ftl_band_type writer_type; +}; + +bool ftl_writer_is_halted(struct ftl_writer *writer); + +void ftl_writer_init(struct spdk_ftl_dev *dev, struct ftl_writer *writer, + uint64_t limit, enum ftl_band_type type); + +void ftl_writer_run(struct ftl_writer *writer); + +void ftl_writer_band_state_change(struct ftl_band *band); + +static inline void +ftl_writer_halt(struct ftl_writer *writer) +{ + writer->halt = true; +} + +static inline void +ftl_writer_resume(struct ftl_writer *writer) +{ + writer->halt = false; +} + +bool ftl_writer_is_halted(struct ftl_writer *writer); + +void ftl_writer_run(struct ftl_writer *writer); + +static inline void +ftl_writer_queue_rq(struct ftl_writer *writer, struct ftl_rq *rq) +{ + TAILQ_INSERT_TAIL(&writer->rq_queue, rq, qentry); +} + +/** + * @brief Returns free space in currently processing band + */ +uint64_t ftl_writer_get_free_blocks(struct ftl_writer *writer); + +#endif /* FTL_WRITER_H */ diff --git a/lib/ftl/mngt/ftl_mngt_band.c b/lib/ftl/mngt/ftl_mngt_band.c index 6eaae625e..4af9ee913 100644 --- a/lib/ftl/mngt/ftl_mngt_band.c +++ b/lib/ftl/mngt/ftl_mngt_band.c @@ -158,14 +158,16 @@ ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) void ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { - struct ftl_band *band, *temp_band; - uint64_t num_open = 0, num_shut = 0; + struct ftl_band *band, *temp_band, *open_bands[FTL_MAX_OPEN_BANDS]; + struct ftl_writer *writer; + uint64_t i, num_open = 0, num_shut = 0; + uint64_t offset; TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) { if (band->md->state == FTL_BAND_STATE_OPEN || band->md->state == FTL_BAND_STATE_FULL) { TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); - num_open++; + open_bands[num_open++] = band; assert(num_open <= FTL_MAX_OPEN_BANDS); continue; } @@ -180,6 +182,43 @@ ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process * } } + /* Assign open bands to writers and alloc necessary resources */ + for (i = 0; i < num_open; ++i) { + band = open_bands[i]; + + if (band->md->type == FTL_BAND_TYPE_COMPACTION) { + writer = &dev->writer_user; + } else if (band->md->type == FTL_BAND_TYPE_GC) { + writer = &dev->writer_gc; + } else { + assert(false); + } + + if (band->md->state == FTL_BAND_STATE_FULL) { + TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry); + } else { + if (writer->band == NULL) { + writer->band = band; + } else { + writer->next_band = band; + } + } + + writer->num_bands++; + ftl_band_set_owner(band, ftl_writer_band_state_change, writer); + + if (dev->sb->clean) { + if (ftl_band_alloc_p2l_map(band)) { + ftl_mngt_fail_step(mngt); + return; + } + + offset = band->md->iter.offset; + ftl_band_iter_init(band); + ftl_band_iter_set(band, offset); + } + } + /* Recalculate number of free bands */ dev->num_free = 0; TAILQ_FOREACH(band, &dev->free_bands, queue_entry) { diff --git a/lib/ftl/mngt/ftl_mngt_misc.c b/lib/ftl/mngt/ftl_mngt_misc.c index 81c421b13..55e8fc5d3 100644 --- a/lib/ftl/mngt/ftl_mngt_misc.c +++ b/lib/ftl/mngt/ftl_mngt_misc.c @@ -140,6 +140,8 @@ ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mng { dev->initialized = 1; + ftl_writer_resume(&dev->writer_user); + ftl_writer_resume(&dev->writer_gc); ftl_nv_cache_resume(&dev->nv_cache); ftl_mngt_next_step(mngt); diff --git a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c b/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c index 20e56bfd2..d1b055476 100644 --- a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c +++ b/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c @@ -108,6 +108,8 @@ DEFINE_STUB(ftl_nv_cache_read, int, (struct ftl_io *io, ftl_addr addr, uint32_t DEFINE_STUB(spdk_bdev_queue_io_wait, int, (struct spdk_bdev *bdev, struct spdk_io_channel *ch, struct spdk_bdev_io_wait_entry *entry), 0); DEFINE_STUB(ftl_l2p_get, ftl_addr, (struct spdk_ftl_dev *dev, uint64_t lba), 0); +DEFINE_STUB_V(ftl_writer_run, (struct ftl_writer *writer)); +DEFINE_STUB(ftl_writer_is_halted, bool, (struct ftl_writer *writer), true); static void setup_band(void)