FTL: Add writer logic

Add writer - tracks and manages band state transitions and write pointer as IO
is issued to it.

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I5f878dc15bc1c1ac84835f75fe440672fad541d5
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13335
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/ftl/Makefile

@@ -22,7 +22,7 @@ CFLAGS += -I.
 FTL_SUBDIRS := mngt utils
 
 C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c ftl_sb.c ftl_l2p.c ftl_l2p_flat.c
-C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c
+C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c ftl_writer.c
 C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c
 C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c mngt/ftl_mngt_l2p.c
 C_SRCS += mngt/ftl_mngt_band.c

lib/ftl/ftl_core.c

@@ -73,6 +73,16 @@ ftl_shutdown_complete(struct spdk_ftl_dev *dev)
 		return false;
 	}
 
+	if (!ftl_writer_is_halted(&dev->writer_user)) {
+		ftl_writer_halt(&dev->writer_user);
+		return false;
+	}
+
+	if (!ftl_writer_is_halted(&dev->writer_gc)) {
+		ftl_writer_halt(&dev->writer_gc);
+		return false;
+	}
+
 	if (!ftl_nv_cache_chunks_busy(&dev->nv_cache)) {
 		return false;
 	}
@@ -481,6 +491,8 @@ ftl_core_poller(void *ctx)
 	}
 
 	ftl_process_io_queue(dev);
+	ftl_writer_run(&dev->writer_user);
+	ftl_writer_run(&dev->writer_gc);
 	ftl_nv_cache_process(dev);
 	ftl_l2p_process(dev);
 

lib/ftl/ftl_core.h

@@ -18,6 +18,7 @@
 #include "ftl_internal.h"
 #include "ftl_io.h"
 #include "ftl_nv_cache.h"
+#include "ftl_writer.h"
 #include "ftl_layout.h"
 #include "ftl_sb.h"
 #include "ftl_l2p.h"
@@ -138,6 +139,12 @@ struct spdk_ftl_dev {
 
 	/* Write submission queue */
 	TAILQ_HEAD(, ftl_io)		wr_sq;
+
+	/* Writer for user IOs */
+	struct ftl_writer		writer_user;
+
+	/* Writer for GC IOs */
+	struct ftl_writer		writer_gc;
 };
 
 void ftl_apply_limits(struct spdk_ftl_dev *dev);

lib/ftl/ftl_init.c

@@ -19,6 +19,7 @@
 #include "ftl_band.h"
 #include "ftl_debug.h"
 #include "ftl_nv_cache.h"
+#include "ftl_writer.h"
 #include "ftl_utils.h"
 #include "mngt/ftl_mngt.h"
 
@@ -118,6 +119,9 @@ allocate_dev(const struct spdk_ftl_conf *conf, int *error)
 	TAILQ_INIT(&dev->wr_sq);
 	TAILQ_INIT(&dev->ioch_queue);
 
+	ftl_writer_init(dev, &dev->writer_user, SPDK_FTL_LIMIT_HIGH, FTL_BAND_TYPE_COMPACTION);
+	ftl_writer_init(dev, &dev->writer_gc, SPDK_FTL_LIMIT_CRIT, FTL_BAND_TYPE_GC);
+
 	return dev;
 error:
 	free_dev(dev);

lib/ftl/ftl_writer.c

@@ -0,0 +1,193 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#include "spdk/likely.h"
+
+#include "ftl_writer.h"
+#include "ftl_band.h"
+
+void
+ftl_writer_init(struct spdk_ftl_dev *dev, struct ftl_writer *writer,
+		uint64_t limit, enum ftl_band_type type)
+{
+	memset(writer, 0, sizeof(*writer));
+	writer->dev = dev;
+	TAILQ_INIT(&writer->rq_queue);
+	TAILQ_INIT(&writer->full_bands);
+	writer->limit = limit;
+	writer->halt = true;
+	writer->writer_type = type;
+}
+
+static bool
+can_write(struct ftl_writer *writer)
+{
+	if (spdk_unlikely(writer->halt)) {
+		return false;
+	}
+
+	return writer->band->md->state == FTL_BAND_STATE_OPEN;
+}
+
+void
+ftl_writer_band_state_change(struct ftl_band *band)
+{
+	struct ftl_writer *writer = band->owner.priv;
+
+	switch (band->md->state) {
+	case FTL_BAND_STATE_FULL:
+		assert(writer->band == band);
+		TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry);
+		writer->band = NULL;
+		break;
+
+	case FTL_BAND_STATE_CLOSED:
+		assert(writer->num_bands > 0);
+		writer->num_bands--;
+		ftl_band_clear_owner(band, ftl_writer_band_state_change, writer);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void
+close_full_bands(struct ftl_writer *writer)
+{
+	struct ftl_band *band, *next;
+
+	TAILQ_FOREACH_SAFE(band, &writer->full_bands, queue_entry, next) {
+		if (band->queue_depth) {
+			continue;
+		}
+
+		TAILQ_REMOVE(&writer->full_bands, band, queue_entry);
+		ftl_band_close(band);
+	}
+}
+
+static bool
+is_active(struct ftl_writer *writer)
+{
+	if (writer->dev->limit < writer->limit) {
+		return false;
+	}
+
+	return true;
+}
+
+static struct ftl_band *
+get_band(struct ftl_writer *writer)
+{
+	if (spdk_unlikely(!writer->band)) {
+		if (!is_active(writer)) {
+			return NULL;
+		}
+
+		if (spdk_unlikely(NULL != writer->next_band)) {
+			if (FTL_BAND_STATE_OPEN == writer->next_band->md->state) {
+				writer->band = writer->next_band;
+				writer->next_band = NULL;
+
+				return writer->band;
+			} else {
+				assert(FTL_BAND_STATE_OPEN == writer->next_band->md->state);
+				ftl_abort();
+			}
+		}
+
+		writer->band = ftl_band_get_next_free(writer->dev);
+		if (writer->band) {
+			writer->num_bands++;
+			ftl_band_set_owner(writer->band,
+					   ftl_writer_band_state_change, writer);
+
+			if (ftl_band_write_prep(writer->band)) {
+				/*
+				 * This error might happen due to allocation failure. However number
+				 * of open bands is controlled and it should have enough resources
+				 * to do it. So here is better to perform a crash and recover from
+				 * shared memory to bring back stable state.
+				 *  */
+				ftl_abort();
+			}
+		} else {
+			return NULL;
+		}
+	}
+
+	if (spdk_likely(writer->band->md->state == FTL_BAND_STATE_OPEN)) {
+		return writer->band;
+	} else {
+		if (spdk_unlikely(writer->band->md->state == FTL_BAND_STATE_PREP)) {
+			ftl_band_open(writer->band, writer->writer_type);
+		}
+		return NULL;
+	}
+}
+
+void
+ftl_writer_run(struct ftl_writer *writer)
+{
+	struct ftl_band *band;
+	struct ftl_rq *rq;
+
+	close_full_bands(writer);
+
+	if (!TAILQ_EMPTY(&writer->rq_queue)) {
+		band = get_band(writer);
+		if (spdk_unlikely(!band)) {
+			return;
+		}
+
+		if (!can_write(writer)) {
+			return;
+		}
+
+		/* Finally we can write to band */
+		rq = TAILQ_FIRST(&writer->rq_queue);
+		TAILQ_REMOVE(&writer->rq_queue, rq, qentry);
+		ftl_band_rq_write(writer->band, rq);
+	}
+}
+
+bool
+ftl_writer_is_halted(struct ftl_writer *writer)
+{
+	if (spdk_unlikely(!TAILQ_EMPTY(&writer->full_bands))) {
+		return false;
+	}
+
+	if (writer->band) {
+		if (writer->band->md->state != FTL_BAND_STATE_OPEN) {
+			return false;
+		}
+
+		if (writer->band->queue_depth) {
+			return false;
+		}
+	}
+
+	return writer->halt;
+}
+
+uint64_t
+ftl_writer_get_free_blocks(struct ftl_writer *writer)
+{
+	uint64_t free_blocks = 0;
+
+	if (writer->band) {
+		free_blocks += ftl_band_user_blocks_left(writer->band,
+				writer->band->md->iter.offset);
+	}
+
+	if (writer->next_band) {
+		free_blocks += ftl_band_user_blocks_left(writer->next_band,
+				writer->next_band->md->iter.offset);
+	}
+
+	return free_blocks;
+}

lib/ftl/ftl_writer.h

@@ -0,0 +1,77 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ */
+
+#ifndef FTL_WRITER_H
+#define FTL_WRITER_H
+
+#include "spdk/stdinc.h"
+#include "spdk/queue.h"
+
+#include "ftl_io.h"
+
+struct ftl_writer {
+	struct spdk_ftl_dev *dev;
+
+	TAILQ_HEAD(, ftl_rq) rq_queue;
+
+	/* Band currently being written to */
+	struct ftl_band	*band;
+
+	/* Number of bands associated with writer */
+	uint64_t num_bands;
+
+	/* Band next being written to */
+	struct ftl_band *next_band;
+
+	/* List of full bands */
+	TAILQ_HEAD(, ftl_band) full_bands;
+
+	/* FTL band limit which blocks writes */
+	int limit;
+
+	/* Flag indicating halt has been requested */
+	bool halt;
+
+	/* Which type of band the writer uses */
+	enum ftl_band_type writer_type;
+};
+
+bool ftl_writer_is_halted(struct ftl_writer *writer);
+
+void ftl_writer_init(struct spdk_ftl_dev *dev, struct ftl_writer *writer,
+		     uint64_t limit, enum ftl_band_type type);
+
+void ftl_writer_run(struct ftl_writer *writer);
+
+void ftl_writer_band_state_change(struct ftl_band *band);
+
+static inline void
+ftl_writer_halt(struct ftl_writer *writer)
+{
+	writer->halt = true;
+}
+
+static inline void
+ftl_writer_resume(struct ftl_writer *writer)
+{
+	writer->halt = false;
+}
+
+bool ftl_writer_is_halted(struct ftl_writer *writer);
+
+void ftl_writer_run(struct ftl_writer *writer);
+
+static inline void
+ftl_writer_queue_rq(struct ftl_writer *writer, struct ftl_rq *rq)
+{
+	TAILQ_INSERT_TAIL(&writer->rq_queue, rq, qentry);
+}
+
+/**
+ * @brief Returns free space in currently processing band
+ */
+uint64_t ftl_writer_get_free_blocks(struct ftl_writer *writer);
+
+#endif  /* FTL_WRITER_H */

lib/ftl/mngt/ftl_mngt_band.c

@@ -158,14 +158,16 @@ ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 void
 ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {
-	struct ftl_band *band, *temp_band;
-	uint64_t num_open = 0, num_shut = 0;
+	struct ftl_band *band, *temp_band, *open_bands[FTL_MAX_OPEN_BANDS];
+	struct ftl_writer *writer;
+	uint64_t i, num_open = 0, num_shut = 0;
+	uint64_t offset;
 
 	TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) {
 		if (band->md->state == FTL_BAND_STATE_OPEN ||
 		    band->md->state == FTL_BAND_STATE_FULL) {
 			TAILQ_REMOVE(&dev->shut_bands, band, queue_entry);
-			num_open++;
+			open_bands[num_open++] = band;
 			assert(num_open <= FTL_MAX_OPEN_BANDS);
 			continue;
 		}
@@ -180,6 +182,43 @@ ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *
 		}
 	}
 
+	/* Assign open bands to writers and alloc necessary resources */
+	for (i = 0; i < num_open; ++i) {
+		band = open_bands[i];
+
+		if (band->md->type == FTL_BAND_TYPE_COMPACTION) {
+			writer = &dev->writer_user;
+		} else if (band->md->type == FTL_BAND_TYPE_GC) {
+			writer = &dev->writer_gc;
+		} else {
+			assert(false);
+		}
+
+		if (band->md->state == FTL_BAND_STATE_FULL) {
+			TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry);
+		} else {
+			if (writer->band == NULL) {
+				writer->band = band;
+			} else {
+				writer->next_band = band;
+			}
+		}
+
+		writer->num_bands++;
+		ftl_band_set_owner(band, ftl_writer_band_state_change, writer);
+
+		if (dev->sb->clean) {
+			if (ftl_band_alloc_p2l_map(band)) {
+				ftl_mngt_fail_step(mngt);
+				return;
+			}
+
+			offset = band->md->iter.offset;
+			ftl_band_iter_init(band);
+			ftl_band_iter_set(band, offset);
+		}
+	}
+
 	/* Recalculate number of free bands */
 	dev->num_free = 0;
 	TAILQ_FOREACH(band, &dev->free_bands, queue_entry) {

lib/ftl/mngt/ftl_mngt_misc.c

@@ -140,6 +140,8 @@ ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mng
 {
 	dev->initialized = 1;
 
+	ftl_writer_resume(&dev->writer_user);
+	ftl_writer_resume(&dev->writer_gc);
 	ftl_nv_cache_resume(&dev->nv_cache);
 
 	ftl_mngt_next_step(mngt);

test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c

@@ -108,6 +108,8 @@ DEFINE_STUB(ftl_nv_cache_read, int, (struct ftl_io *io, ftl_addr addr, uint32_t
 DEFINE_STUB(spdk_bdev_queue_io_wait, int, (struct spdk_bdev *bdev, struct spdk_io_channel *ch,
 		struct spdk_bdev_io_wait_entry *entry), 0);
 DEFINE_STUB(ftl_l2p_get, ftl_addr, (struct spdk_ftl_dev *dev, uint64_t lba), 0);
+DEFINE_STUB_V(ftl_writer_run, (struct ftl_writer *writer));
+DEFINE_STUB(ftl_writer_is_halted, bool, (struct ftl_writer *writer), true);
 
 static void
 setup_band(void)