ftl: valid map

Adds P2L validity map tracking - a bitmap marking all physical LBAs
as containing valid (current) user data or not. A clear bit denotes
the location has no valid data and may be skipped during relocation
or compaction. A set bit means it may have valid data (it's still
necessary to do the necessary comparision against L2P).

Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Change-Id: I6a831a97b3080eb7c880d9c4feab41b523467885
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13350
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

lib/ftl/ftl_band.c

@@ -184,6 +184,7 @@ ftl_band_set_addr(struct ftl_band *band, uint64_t lba, ftl_addr addr)
 
 	p2l_map->band_map[offset] = lba;
 	p2l_map->num_valid++;
+	ftl_bitmap_set(band->dev->valid_map, addr);
 }
 
 size_t

lib/ftl/ftl_band.h

@@ -202,6 +202,14 @@ ftl_band_qd(const struct ftl_band *band)
 	return band->queue_depth;
 }
 
+static inline bool
+ftl_band_block_offset_valid(struct ftl_band *band, size_t block_off)
+{
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
+
+	return ftl_bitmap_get(p2l_map->valid, block_off);
+}
+
 static inline void
 ftl_band_iter_init(struct ftl_band *band)
 {

lib/ftl/ftl_core.c

@@ -133,15 +133,20 @@ ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr)
 	struct ftl_p2l_map *p2l_map;
 
 	if (ftl_addr_in_nvc(dev, addr)) {
+		ftl_bitmap_clear(dev->valid_map, addr);
 		return;
 	}
 
 	band = ftl_band_from_addr(dev, addr);
 	p2l_map = &band->p2l_map;
 
-	/* TODO: fix case when the same address is invalidated from multiple sources */
+	/* The bit might be already cleared if two writes are scheduled to the */
+	/* same LBA at the same time */
+	if (ftl_bitmap_get(dev->valid_map, addr)) {
 		assert(p2l_map->num_valid > 0);
+		ftl_bitmap_clear(dev->valid_map, addr);
 		p2l_map->num_valid--;
+	}
 
 	/* Invalidate open/full band p2l_map entry to keep p2l and l2p
 	 * consistency when band is going to close state */

lib/ftl/ftl_core.h

@@ -22,6 +22,7 @@
 #include "ftl_layout.h"
 #include "ftl_sb.h"
 #include "ftl_l2p.h"
+#include "utils/ftl_bitmap.h"
 #include "utils/ftl_log.h"
 
 /*
@@ -115,6 +116,9 @@ struct spdk_ftl_dev {
 	/* Size of the l2p table */
 	uint64_t			num_lbas;
 
+	/* P2L valid map */
+	struct ftl_bitmap		*valid_map;
+
 	/* Metadata size */
 	uint64_t			md_size;
 

lib/ftl/ftl_internal.h

@@ -11,6 +11,8 @@
 #include "spdk/util.h"
 #include "spdk/uuid.h"
 
+#include "utils/ftl_bitmap.h"
+
 /* Marks address as invalid */
 #define FTL_ADDR_INVALID	((ftl_addr)-1)
 /* Marks LBA as invalid */
@@ -68,6 +70,9 @@ struct ftl_p2l_map {
 	/* P2L map's reference count, prevents premature release of resources during dirty shutdown recovery for open bands */
 	size_t					ref_cnt;
 
+	/* Bitmap of valid LBAs */
+	struct ftl_bitmap			*valid;
+
 	/* P2L map (only valid for open/relocating bands) */
 	union {
 		uint64_t	*band_map;

lib/ftl/ftl_layout.c

@@ -306,6 +306,17 @@ setup_layout_base(struct spdk_ftl_dev *dev)
 	/* Move offset after base superblock */
 	offset += layout->region[FTL_LAYOUT_REGION_TYPE_SB_BASE].current.blocks;
 
+	/* Setup validity map */
+	region = &layout->region[FTL_LAYOUT_REGION_TYPE_VALID_MAP];
+	region->type = FTL_LAYOUT_REGION_TYPE_VALID_MAP;
+	region->name = "vmap";
+	region->current.version = region->prev.version = 0;
+	region->current.offset = offset;
+	region->current.blocks = blocks_region(spdk_divide_round_up(
+			layout->base.total_blocks + layout->nvc.total_blocks, 8));
+	set_region_bdev_btm(region, dev);
+	offset += region->current.blocks;
+
 	/* Checking for underflow */
 	left = layout->base.total_blocks - offset;
 	if (left > layout->base.total_blocks) {

lib/ftl/ftl_layout.h

@@ -28,6 +28,9 @@ enum ftl_layout_region_type {
 	/* Mirrored instance of bands state */
 	FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR,
 
+	/* Map of valid physical addresses, used for more efficient garbage collection */
+	FTL_LAYOUT_REGION_TYPE_VALID_MAP,
+
 	/* State of chunks */
 	FTL_LAYOUT_REGION_TYPE_NVC_MD,
 	/* Mirrored instance of the state of chunks */
@@ -44,7 +47,7 @@ enum ftl_layout_region_type {
 
 /* last nvc/base region in terms of lba address space */
 #define FTL_LAYOUT_REGION_LAST_NVC FTL_LAYOUT_REGION_TYPE_DATA_NVC
-#define FTL_LAYOUT_REGION_LAST_BASE FTL_LAYOUT_REGION_TYPE_DATA_BASE
+#define FTL_LAYOUT_REGION_LAST_BASE FTL_LAYOUT_REGION_TYPE_VALID_MAP
 #define FTL_LAYOUT_REGION_TYPE_FREE_BASE (UINT32_MAX - 2)
 #define FTL_LAYOUT_REGION_TYPE_FREE_NVC (UINT32_MAX - 1)
 #define FTL_LAYOUT_REGION_TYPE_INVALID (UINT32_MAX)

lib/ftl/ftl_nv_cache.c

@@ -1170,6 +1170,7 @@ ftl_nv_cache_set_addr(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr)
 	assert(lba != FTL_LBA_INVALID);
 
 	ftl_chunk_set_addr(chunk, lba, addr);
+	ftl_bitmap_set(dev->valid_map, addr);
 }
 
 static void ftl_chunk_open(struct ftl_nv_cache_chunk *chunk);

lib/ftl/ftl_reloc.c

@@ -292,6 +292,7 @@ move_advance_rq(struct ftl_rq *rq)
 		offset = ftl_band_block_offset_from_addr(band, rq->io.addr);
 
 		assert(offset < ftl_get_num_blocks_in_band(band->dev));
+		assert(ftl_band_block_offset_valid(band, offset));
 
 		entry->lba = band->p2l_map.band_map[offset];
 		entry->addr = rq->io.addr;

lib/ftl/mngt/ftl_mngt_band.c

@@ -12,9 +12,25 @@ static int
 ftl_band_init_md(struct ftl_band *band)
 {
 	struct spdk_ftl_dev *dev = band->dev;
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
 	struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD];
+	struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP];
+	uint64_t band_num_blocks = ftl_get_num_blocks_in_band(band->dev);
+	size_t band_valid_map_bytes;
 	struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md);
 
+	if (band_num_blocks % (ftl_bitmap_buffer_alignment * 8)) {
+		FTL_ERRLOG(dev, "The number of blocks in band is not divisible by bitmap word bits\n");
+		return -EINVAL;
+	}
+	band_valid_map_bytes = band_num_blocks / 8;
+
+	p2l_map->valid = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md) +
+					   band_valid_map_bytes * band->id, band_valid_map_bytes);
+	if (!p2l_map->valid) {
+		return -ENOMEM;
+	}
+
 	band->md = &band_md[band->id];
 	if (!ftl_fast_startup(dev)) {
 		band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID;
@@ -73,6 +89,22 @@ ftl_dev_deinit_bands(struct spdk_ftl_dev *dev)
 	free(dev->bands);
 }
 
+static void
+ftl_dev_deinit_bands_md(struct spdk_ftl_dev *dev)
+{
+	if (dev->bands) {
+		uint64_t i;
+		for (i = 0; i < dev->num_bands; ++i) {
+			struct ftl_band *band = &dev->bands[i];
+
+			ftl_bitmap_destroy(band->p2l_map.valid);
+			band->p2l_map.valid = NULL;
+
+			band->md = NULL;
+		}
+	}
+}
+
 void
 ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {
@@ -100,6 +132,13 @@ ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 	ftl_mngt_next_step(mngt);
 }
 
+void
+ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	ftl_dev_deinit_bands_md(dev);
+	ftl_mngt_next_step(mngt);
+}
+
 /*
  * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's
  * perspective. Improves WAF.

lib/ftl/mngt/ftl_mngt_misc.c

@@ -218,3 +218,30 @@ ftl_mngt_dump_stats(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 	ftl_dev_dump_stats(dev);
 	ftl_mngt_next_step(mngt);
 }
+
+void
+ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP];
+
+	dev->valid_map = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md),
+					   ftl_md_get_buffer_size(valid_map_md));
+	if (!dev->valid_map) {
+		FTL_ERRLOG(dev, "Failed to create valid map\n");
+		ftl_mngt_fail_step(mngt);
+		return;
+	}
+
+	ftl_mngt_next_step(mngt);
+}
+
+void
+ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	if (dev->valid_map) {
+		ftl_bitmap_destroy(dev->valid_map);
+		dev->valid_map = NULL;
+	}
+
+	ftl_mngt_next_step(mngt);
+}

lib/ftl/mngt/ftl_mngt_startup.c

@@ -92,9 +92,15 @@ static const struct ftl_mngt_process_desc desc_startup = {
 			.action = ftl_mngt_init_nv_cache,
 			.cleanup = ftl_mngt_deinit_nv_cache
 		},
+		{
+			.name = "Initialize valid map",
+			.action = ftl_mngt_init_vld_map,
+			.cleanup = ftl_mngt_deinit_vld_map
+		},
 		{
 			.name = "Initialize bands metadata",
-			.action = ftl_mngt_init_bands_md
+			.action = ftl_mngt_init_bands_md,
+			.cleanup = ftl_mngt_deinit_bands_md
 		},
 		{
 			.name = "Initialize reloc",

lib/ftl/mngt/ftl_mngt_steps.h

@@ -42,6 +42,8 @@ void ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *m
 
 void ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
@@ -86,6 +88,10 @@ void ftl_mngt_init_default_sb(struct spdk_ftl_dev *dev, struct ftl_mngt_process
 
 void ftl_mngt_set_dirty(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
+void ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_persist_band_info_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_persist_nv_cache_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);

lib/ftl/utils/ftl_md.c

@@ -1125,6 +1125,11 @@ ftl_md_create_region_flags(struct spdk_ftl_dev *dev, int region_type)
 			flags |= FTL_MD_CREATE_SHM_NEW;
 		}
 		break;
+	case FTL_LAYOUT_REGION_TYPE_VALID_MAP:
+		if (!ftl_fast_startup(dev) && !ftl_fast_recovery(dev)) {
+			flags |= FTL_MD_CREATE_SHM_NEW;
+		}
+		break;
 	default:
 		return FTL_MD_CREATE_HEAP;
 	}
@@ -1138,6 +1143,7 @@ ftl_md_destroy_region_flags(struct spdk_ftl_dev *dev, int region_type)
 	switch (region_type) {
 	case FTL_LAYOUT_REGION_TYPE_SB:
 	case FTL_LAYOUT_REGION_TYPE_BAND_MD:
+	case FTL_LAYOUT_REGION_TYPE_VALID_MAP:
 	case FTL_LAYOUT_REGION_TYPE_NVC_MD:
 		if (dev->conf.fast_shutdown) {
 			return FTL_MD_DESTROY_SHM_KEEP;

test/unit/lib/ftl/common/utils.c

@@ -116,6 +116,9 @@ test_init_ftl_band(struct spdk_ftl_dev *dev, size_t id, size_t zone_size)
 	band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID;
 	TAILQ_INSERT_HEAD(&dev->shut_bands, band, queue_entry);
 
+	band->p2l_map.valid = (struct ftl_bitmap *)spdk_bit_array_create(ftl_get_num_blocks_in_band(dev));
+	SPDK_CU_ASSERT_FATAL(band->p2l_map.valid != NULL);
+
 	band->start_addr = zone_size * id;
 
 	return band;
@@ -149,6 +152,7 @@ void
 test_free_ftl_band(struct ftl_band *band)
 {
 	SPDK_CU_ASSERT_FATAL(band != NULL);
+	spdk_bit_array_free((struct spdk_bit_array **)&band->p2l_map.valid);
 	spdk_dma_free(band->p2l_map.band_dma_md);
 
 	band->p2l_map.band_dma_md = NULL;

test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c

@@ -120,6 +120,39 @@ DEFINE_STUB(ftl_writer_is_halted, bool, (struct ftl_writer *writer), true);
 DEFINE_STUB(ftl_mempool_claim_df, void *, (struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id),
 	    NULL);
 
+static void
+adjust_bitmap(struct ftl_bitmap **bitmap, uint64_t *bit)
+{
+	if (*bitmap == g_dev->valid_map) {
+		*bitmap = g_band->p2l_map.valid;
+		*bit = test_offset_from_addr(*bit, g_band);
+	}
+}
+
+bool
+ftl_bitmap_get(const struct ftl_bitmap *bitmap, uint64_t bit)
+{
+	adjust_bitmap((struct ftl_bitmap **)&bitmap, &bit);
+	return spdk_bit_array_get((struct spdk_bit_array *)bitmap, bit);
+}
+
+void
+ftl_bitmap_set(struct ftl_bitmap *bitmap, uint64_t bit)
+{
+	int ret;
+
+	adjust_bitmap(&bitmap, &bit);
+	ret = spdk_bit_array_set((struct spdk_bit_array *)bitmap, bit);
+	CU_ASSERT_EQUAL(ret, 0);
+}
+
+void
+ftl_bitmap_clear(struct ftl_bitmap *bitmap, uint64_t bit)
+{
+	adjust_bitmap(&bitmap, &bit);
+	spdk_bit_array_clear((struct spdk_bit_array *)bitmap, bit);
+}
+
 static void
 setup_band(void)
 {
@@ -220,14 +253,17 @@ test_band_set_addr(void)
 	ftl_band_set_addr(g_band, TEST_LBA, addr);
 	CU_ASSERT_EQUAL(p2l_map->num_valid, 1);
 	CU_ASSERT_EQUAL(p2l_map->band_map[offset], TEST_LBA);
+	CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset));
 
 	addr += g_geo.zone_size / 2;
 	offset = test_offset_from_addr(addr, g_band);
 	ftl_band_set_addr(g_band, TEST_LBA + 1, addr);
 	CU_ASSERT_EQUAL(p2l_map->num_valid, 2);
 	CU_ASSERT_EQUAL(p2l_map->band_map[offset], TEST_LBA + 1);
+	CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset));
 	addr -= g_geo.zone_size / 2;
 	offset = test_offset_from_addr(addr, g_band);
+	CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset));
 	cleanup_band();
 }
 
@@ -236,23 +272,33 @@ test_invalidate_addr(void)
 {
 	struct ftl_p2l_map *p2l_map;
 	ftl_addr addr;
+	uint64_t offset[2];
 
 	setup_band();
 	p2l_map = &g_band->p2l_map;
 	addr = addr_from_zone_id(0);
 	addr += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev);
+	offset[0] = test_offset_from_addr(addr, g_band);
 
 	ftl_band_set_addr(g_band, TEST_LBA, addr);
 	CU_ASSERT_EQUAL(p2l_map->num_valid, 1);
+	CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[0]));
 	ftl_invalidate_addr(g_band->dev, addr);
 	CU_ASSERT_EQUAL(p2l_map->num_valid, 0);
+	CU_ASSERT_FALSE(ftl_bitmap_get(p2l_map->valid, offset[0]));
 
+	offset[0] = test_offset_from_addr(addr, g_band);
 	ftl_band_set_addr(g_band, TEST_LBA, addr);
 	addr += g_geo.zone_size / 2;
+	offset[1] = test_offset_from_addr(addr, g_band);
 	ftl_band_set_addr(g_band, TEST_LBA + 1, addr);
 	CU_ASSERT_EQUAL(p2l_map->num_valid, 2);
+	CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[0]));
+	CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[1]));
 	ftl_invalidate_addr(g_band->dev, addr);
 	CU_ASSERT_EQUAL(p2l_map->num_valid, 1);
+	CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[0]));
+	CU_ASSERT_FALSE(ftl_bitmap_get(p2l_map->valid, offset[1]));
 	cleanup_band();
 }