FTL: Add reloc helper functions

Adds functions for reading end metadata and initializing band reloc state.

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I3d12c4a7edd36f0437bf10316114c83efe449f0f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13343
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/ftl/ftl_band.c

@@ -66,6 +66,7 @@ _ftl_band_set_free(struct ftl_band *band)
 
 	/* Add the band to the free band list */
 	TAILQ_INSERT_TAIL(&dev->free_bands, band, queue_entry);
+	band->reloc = false;
 
 	dev->num_free++;
 	ftl_apply_limits(dev);
@@ -364,3 +365,223 @@ ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev)
 	/* Map pool element holds the whole tail md */
 	return ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE;
 }
+
+static double
+_band_invalidity(struct ftl_band *band)
+{
+	double valid = band->p2l_map.num_valid;
+	double count = ftl_band_user_blocks(band);
+
+	return 1.0 - (valid / count);
+}
+
+static void
+dump_bands_under_relocation(struct spdk_ftl_dev *dev)
+{
+	uint64_t i = dev->sb_shm->gc_info.current_band_id;
+	uint64_t end = dev->sb_shm->gc_info.current_band_id + dev->num_logical_bands_in_physical;
+
+	for (; i < end; i++) {
+		struct ftl_band *band = &dev->bands[i];
+
+		FTL_DEBUGLOG(dev, "Band, id %u, phys_is %u, wr cnt = %u, invalidity = %u%%\n",
+			     band->id, band->phys_id, (uint32_t)band->md->wr_cnt,
+			     (uint32_t)(_band_invalidity(band) * 100));
+	}
+}
+
+static bool
+is_band_relocateable(struct ftl_band *band)
+{
+	/* Can only move data from closed bands */
+	if (FTL_BAND_STATE_CLOSED != band->md->state) {
+		return false;
+	}
+
+	/* Band is already under relocation, skip it */
+	if (band->reloc) {
+		return false;
+	}
+
+	return true;
+}
+
+static void
+get_band_phys_info(struct spdk_ftl_dev *dev, uint64_t phys_id,
+		   double *invalidity, double *wr_cnt)
+{
+	struct ftl_band *band;
+	uint64_t band_id = phys_id * dev->num_logical_bands_in_physical;
+
+	*wr_cnt = *invalidity = 0.0L;
+	for (; band_id < ftl_get_num_bands(dev); band_id++) {
+		band = &dev->bands[band_id];
+
+		if (phys_id != band->phys_id) {
+			break;
+		}
+
+		*wr_cnt += band->md->wr_cnt;
+
+		if (!is_band_relocateable(band)) {
+			continue;
+		}
+
+		*invalidity += _band_invalidity(band);
+	}
+
+	*invalidity /= dev->num_logical_bands_in_physical;
+	*wr_cnt /= dev->num_logical_bands_in_physical;
+}
+
+static bool
+band_cmp(double a_invalidity, double a_wr_cnt,
+	 double b_invalidity, double b_wr_cnt,
+	 uint64_t a_id, uint64_t b_id)
+{
+	assert(a_id != FTL_BAND_PHYS_ID_INVALID);
+	assert(b_id != FTL_BAND_PHYS_ID_INVALID);
+	double diff = a_invalidity - b_invalidity;
+	if (diff < 0.0L) {
+		diff *= -1.0L;
+	}
+
+	/* Use the following metrics for picking bands for GC (in order):
+	 * - relative invalidity
+	 * - if invalidity is similar (within 10% points), then their write counts (how many times band was written to)
+	 * - if write count is equal, then pick based on their placement on base device (lower LBAs win)
+	 */
+	if (diff > 0.1L) {
+		return a_invalidity > b_invalidity;
+	}
+
+	if (a_wr_cnt != b_wr_cnt) {
+		return a_wr_cnt < b_wr_cnt;
+	}
+
+	return a_id < b_id;
+}
+
+static void
+band_start_gc(struct spdk_ftl_dev *dev, struct ftl_band *band)
+{
+	ftl_bug(false == is_band_relocateable(band));
+
+	TAILQ_REMOVE(&dev->shut_bands, band, queue_entry);
+	band->reloc = true;
+
+	FTL_DEBUGLOG(dev, "Band to GC, id %u\n", band->id);
+}
+
+static struct ftl_band *
+gc_high_priority_band(struct spdk_ftl_dev *dev)
+{
+	struct ftl_band *band;
+	uint64_t high_prio_id = dev->sb_shm->gc_info.band_id_high_prio;
+
+	if (FTL_BAND_ID_INVALID != high_prio_id) {
+		ftl_bug(high_prio_id >= dev->num_bands);
+
+		band = &dev->bands[high_prio_id];
+		dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID;
+
+		band_start_gc(dev, band);
+		FTL_NOTICELOG(dev, "GC takes high priority band, id %u\n", band->id);
+		return band;
+	}
+
+	return 0;
+}
+
+static void
+ftl_band_reset_gc_iter(struct spdk_ftl_dev *dev)
+{
+	dev->sb->gc_info.is_valid = 0;
+	dev->sb->gc_info.current_band_id = FTL_BAND_ID_INVALID;
+	dev->sb->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID;
+	dev->sb->gc_info.band_phys_id = FTL_BAND_PHYS_ID_INVALID;
+
+	dev->sb_shm->gc_info = dev->sb->gc_info;
+}
+
+struct ftl_band *
+ftl_band_search_next_to_reloc(struct spdk_ftl_dev *dev)
+{
+	double invalidity, max_invalidity = 0.0L;
+	double wr_cnt, max_wr_cnt = 0.0L;
+	uint64_t phys_id = FTL_BAND_PHYS_ID_INVALID;
+	struct ftl_band *band;
+	uint64_t i, band_count;
+	uint64_t phys_count;
+
+	band = gc_high_priority_band(dev);
+	if (spdk_unlikely(NULL != band)) {
+		return band;
+	}
+
+	phys_count = dev->num_logical_bands_in_physical;
+	band_count = ftl_get_num_bands(dev);
+
+	for (; dev->sb_shm->gc_info.current_band_id < band_count;) {
+		band = &dev->bands[dev->sb_shm->gc_info.current_band_id];
+		if (band->phys_id != dev->sb_shm->gc_info.band_phys_id) {
+			break;
+		}
+
+		if (false == is_band_relocateable(band)) {
+			dev->sb_shm->gc_info.current_band_id++;
+			continue;
+		}
+
+		band_start_gc(dev, band);
+		return band;
+	}
+
+	for (i = 0; i < band_count; i += phys_count) {
+		band = &dev->bands[i];
+
+		/* Calculate entire band physical group invalidity */
+		get_band_phys_info(dev, band->phys_id, &invalidity, &wr_cnt);
+
+		if (invalidity != 0.0L) {
+			if (phys_id == FTL_BAND_PHYS_ID_INVALID ||
+			    band_cmp(invalidity, wr_cnt, max_invalidity, max_wr_cnt,
+				     band->phys_id, phys_id)) {
+				max_invalidity = invalidity;
+				max_wr_cnt = wr_cnt;
+				phys_id = band->phys_id;
+			}
+		}
+	}
+
+	if (FTL_BAND_PHYS_ID_INVALID != phys_id) {
+		FTL_DEBUGLOG(dev, "Band physical id %"PRIu64" to GC\n", phys_id);
+		dev->sb_shm->gc_info.is_valid = 0;
+		dev->sb_shm->gc_info.current_band_id = phys_id * phys_count;
+		dev->sb_shm->gc_info.band_phys_id = phys_id;
+		dev->sb_shm->gc_info.is_valid = 1;
+		dump_bands_under_relocation(dev);
+		return ftl_band_search_next_to_reloc(dev);
+	} else {
+		ftl_band_reset_gc_iter(dev);
+	}
+
+	return NULL;
+}
+
+void
+ftl_band_init_gc_iter(struct spdk_ftl_dev *dev)
+{
+	if (dev->conf.mode & SPDK_FTL_MODE_CREATE) {
+		ftl_band_reset_gc_iter(dev);
+		return;
+	}
+
+	if (dev->sb->clean) {
+		dev->sb_shm->gc_info = dev->sb->gc_info;
+		return;
+	}
+
+	/* We lost GC state due to dirty shutdown, reset GC state to start over */
+	ftl_band_reset_gc_iter(dev);
+}

lib/ftl/ftl_band.h

@@ -97,6 +97,9 @@ struct ftl_band {
 	/* P2L map */
 	struct ftl_p2l_map		p2l_map;
 
+	/* Band relocation is in progress */
+	bool				reloc;
+
 	/* Band's index */
 	uint32_t			id;
 
@@ -144,6 +147,8 @@ ftl_addr ftl_band_tail_md_addr(struct ftl_band *band);
 int ftl_band_filled(struct ftl_band *band, size_t offset);
 int ftl_band_write_prep(struct ftl_band *band);
 size_t ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev);
+struct ftl_band *ftl_band_search_next_to_reloc(struct spdk_ftl_dev *dev);
+void ftl_band_init_gc_iter(struct spdk_ftl_dev *dev);
 ftl_addr ftl_band_p2l_map_addr(struct ftl_band *band);
 void ftl_band_open(struct ftl_band *band, enum ftl_band_type type);
 void ftl_band_close(struct ftl_band *band);
@@ -152,6 +157,7 @@ void ftl_band_rq_write(struct ftl_band *band, struct ftl_rq *rq);
 void ftl_band_rq_read(struct ftl_band *band, struct ftl_rq *rq);
 void ftl_band_basic_rq_write(struct ftl_band *band, struct ftl_basic_rq *brq);
 void ftl_band_basic_rq_read(struct ftl_band *band, struct ftl_basic_rq *brq);
+void ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx);
 void ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx);
 
 static inline void

lib/ftl/ftl_band_ops.c

@@ -340,7 +340,7 @@ ftl_band_close(struct ftl_band *band)
 	void *metadata = band->p2l_map.band_map;
 	uint64_t num_blocks = ftl_tail_md_num_blocks(dev);
 
-	/* Write LBA map first, after completion, set the state to close on nvcache, then internally */
+	/* Write P2L map first, after completion, set the state to close on nvcache, then internally */
 	ftl_band_set_state(band, FTL_BAND_STATE_CLOSING);
 	ftl_basic_rq_init(dev, &band->metadata_rq, metadata, num_blocks);
 	ftl_basic_rq_set_owner(&band->metadata_rq, band_map_write_cb, band);
@@ -382,6 +382,68 @@ ftl_band_free(struct ftl_band *band)
 	/* TODO: The whole band erase code should probably be done here instead */
 }
 
+static void
+read_md_cb(struct ftl_basic_rq *brq)
+{
+	struct ftl_band *band = brq->owner.priv;
+	struct spdk_ftl_dev *dev = band->dev;
+	ftl_band_ops_cb cb;
+	uint32_t band_map_crc;
+	bool success = true;
+	void *priv;
+
+	cb = band->owner.ops_fn;
+	priv = band->owner.priv;
+
+	if (!brq->success) {
+		ftl_band_basic_rq_read(band, &band->metadata_rq);
+		return;
+	}
+
+	band_map_crc = spdk_crc32c_update(band->p2l_map.band_map,
+					  ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE, 0);
+	if (band->md->p2l_map_checksum && band->md->p2l_map_checksum != band_map_crc) {
+		FTL_ERRLOG(dev, "GC error, inconsistent P2L map CRC\n");
+		success = false;
+	}
+	band->owner.ops_fn = NULL;
+	band->owner.priv = NULL;
+	cb(band, priv, success);
+}
+
+static int
+_read_md(struct ftl_band *band)
+{
+	struct spdk_ftl_dev *dev = band->dev;
+	struct ftl_basic_rq *rq = &band->metadata_rq;
+
+	if (ftl_band_alloc_p2l_map(band)) {
+		return -ENOMEM;
+	}
+
+	/* Read P2L map */
+	ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_p2l_map_num_blocks(dev));
+	ftl_basic_rq_set_owner(rq, read_md_cb, band);
+
+	rq->io.band = band;
+	rq->io.addr = ftl_band_p2l_map_addr(band);
+
+	ftl_band_basic_rq_read(band, &band->metadata_rq);
+
+	return 0;
+}
+
+static void
+read_md(void *band)
+{
+	int rc;
+
+	rc = _read_md(band);
+	if (spdk_unlikely(rc)) {
+		spdk_thread_send_msg(spdk_get_thread(), read_md, band);
+	}
+}
+
 static void
 read_tail_md_cb(struct ftl_basic_rq *brq)
 {
@@ -426,3 +488,24 @@ ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx)
 
 	ftl_band_basic_rq_read(band, &band->metadata_rq);
 }
+
+void
+ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx)
+{
+	struct ftl_band *band = ftl_band_search_next_to_reloc(dev);
+
+	/* if disk is very small, GC start very early that no band is ready for it */
+	if (spdk_unlikely(!band)) {
+		cb(NULL, cntx, false);
+		return;
+	}
+
+	/* Only one owner is allowed */
+	assert(!band->queue_depth);
+	assert(!band->owner.ops_fn);
+	assert(!band->owner.priv);
+	band->owner.ops_fn = cb;
+	band->owner.priv = cntx;
+
+	read_md(band);
+}

lib/ftl/ftl_core.h

@@ -150,6 +150,8 @@ struct spdk_ftl_dev {
 	/* Writer for GC IOs */
 	struct ftl_writer		writer_gc;
 
+	uint32_t			num_logical_bands_in_physical;
+
 	/* Retry init sequence */
 	bool				init_retry;
 };

lib/ftl/mngt/ftl_mngt_band.c

@@ -146,6 +146,8 @@ decorate_bands(struct spdk_ftl_dev *dev)
 		TAILQ_REMOVE(&dev->shut_bands, band, queue_entry);
 		i++;
 	}
+
+	dev->num_logical_bands_in_physical = num_logical_in_phys;
 }
 
 void
@@ -155,6 +157,65 @@ ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 	ftl_mngt_next_step(mngt);
 }
 
+static struct ftl_band *
+next_high_prio_band(struct spdk_ftl_dev *dev)
+{
+	struct ftl_band *result = NULL, *band;
+	uint64_t validity = UINT64_MAX;
+
+	TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) {
+		if (band->p2l_map.num_valid < validity) {
+			result = band;
+			validity = result->p2l_map.num_valid;
+		}
+	}
+
+	return result;
+}
+
+static int
+finalize_init_gc(struct spdk_ftl_dev *dev)
+{
+	struct ftl_band *band;
+	uint64_t free_blocks, blocks_to_move;
+
+	ftl_band_init_gc_iter(dev);
+	dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID;
+
+	if (0 == dev->num_free) {
+		/* Get number of available blocks in writer */
+		free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc);
+
+		/*
+		 * First, check a band candidate to GC
+		 */
+		band = ftl_band_search_next_to_reloc(dev);
+		ftl_bug(NULL == band);
+		blocks_to_move = band->p2l_map.num_valid;
+		if (blocks_to_move <= free_blocks) {
+			/* This GC band can be moved */
+			return 0;
+		}
+
+		/*
+		 * The GC candidate cannot be moved because no enough space. We need to find
+		 * another band.
+		 */
+		band = next_high_prio_band(dev);
+		ftl_bug(NULL == band);
+
+		if (band->p2l_map.num_valid > free_blocks) {
+			FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n");
+			return -1;
+		} else {
+			/* GC needs to start using this band */
+			dev->sb_shm->gc_info.band_id_high_prio = band->id;
+		}
+	}
+
+	return 0;
+}
+
 void
 ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {
@@ -233,5 +294,9 @@ ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *
 		return;
 	}
 
-	ftl_mngt_next_step(mngt);
+	if (finalize_init_gc(dev)) {
+		ftl_mngt_fail_step(mngt);
+	} else {
+		ftl_mngt_next_step(mngt);
+	}
 }

lib/ftl/mngt/ftl_mngt_md.c

@@ -177,6 +177,8 @@ ftl_mngt_init_default_sb(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt
 
 	sb->overprovisioning = dev->conf.overprovisioning;
 
+	ftl_band_init_gc_iter(dev);
+
 	/* md layout isn't initialized yet.
 	 * empty region list => all regions in the default location */
 	sb->md_layout_head.type = FTL_LAYOUT_REGION_TYPE_INVALID;