From cea8dadecf1ba24be873cfb8f44b5a278a5d9b27 Mon Sep 17 00:00:00 2001 From: Artur Paszkiewicz Date: Fri, 10 Jun 2022 10:09:27 +0200 Subject: [PATCH] ftl: valid map Adds P2L validity map tracking - a bitmap marking all physical LBAs as containing valid (current) user data or not. A clear bit denotes the location has no valid data and may be skipped during relocation or compaction. A set bit means it may have valid data (it's still necessary to do the necessary comparision against L2P). Signed-off-by: Artur Paszkiewicz Signed-off-by: Kozlowski Mateusz Change-Id: I6a831a97b3080eb7c880d9c4feab41b523467885 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13350 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- lib/ftl/ftl_band.c | 1 + lib/ftl/ftl_band.h | 8 ++++ lib/ftl/ftl_core.c | 11 ++++-- lib/ftl/ftl_core.h | 4 ++ lib/ftl/ftl_internal.h | 5 +++ lib/ftl/ftl_layout.c | 11 ++++++ lib/ftl/ftl_layout.h | 5 ++- lib/ftl/ftl_nv_cache.c | 1 + lib/ftl/ftl_reloc.c | 1 + lib/ftl/mngt/ftl_mngt_band.c | 39 ++++++++++++++++++ lib/ftl/mngt/ftl_mngt_misc.c | 27 +++++++++++++ lib/ftl/mngt/ftl_mngt_startup.c | 8 +++- lib/ftl/mngt/ftl_mngt_steps.h | 6 +++ lib/ftl/utils/ftl_md.c | 6 +++ test/unit/lib/ftl/common/utils.c | 4 ++ test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c | 46 ++++++++++++++++++++++ 16 files changed, 178 insertions(+), 5 deletions(-) diff --git a/lib/ftl/ftl_band.c b/lib/ftl/ftl_band.c index 350a5d918..961930773 100644 --- a/lib/ftl/ftl_band.c +++ b/lib/ftl/ftl_band.c @@ -184,6 +184,7 @@ ftl_band_set_addr(struct ftl_band *band, uint64_t lba, ftl_addr addr) p2l_map->band_map[offset] = lba; p2l_map->num_valid++; + ftl_bitmap_set(band->dev->valid_map, addr); } size_t diff --git a/lib/ftl/ftl_band.h b/lib/ftl/ftl_band.h index 2da210396..bc66c8991 100644 --- a/lib/ftl/ftl_band.h +++ b/lib/ftl/ftl_band.h @@ -202,6 +202,14 @@ ftl_band_qd(const struct ftl_band *band) return band->queue_depth; } +static inline bool +ftl_band_block_offset_valid(struct ftl_band *band, size_t block_off) +{ + struct ftl_p2l_map *p2l_map = &band->p2l_map; + + return ftl_bitmap_get(p2l_map->valid, block_off); +} + static inline void ftl_band_iter_init(struct ftl_band *band) { diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 890b0fbb6..04cd8c343 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -133,15 +133,20 @@ ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr) struct ftl_p2l_map *p2l_map; if (ftl_addr_in_nvc(dev, addr)) { + ftl_bitmap_clear(dev->valid_map, addr); return; } band = ftl_band_from_addr(dev, addr); p2l_map = &band->p2l_map; - /* TODO: fix case when the same address is invalidated from multiple sources */ - assert(p2l_map->num_valid > 0); - p2l_map->num_valid--; + /* The bit might be already cleared if two writes are scheduled to the */ + /* same LBA at the same time */ + if (ftl_bitmap_get(dev->valid_map, addr)) { + assert(p2l_map->num_valid > 0); + ftl_bitmap_clear(dev->valid_map, addr); + p2l_map->num_valid--; + } /* Invalidate open/full band p2l_map entry to keep p2l and l2p * consistency when band is going to close state */ diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 0a478e5b5..8071d2808 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -22,6 +22,7 @@ #include "ftl_layout.h" #include "ftl_sb.h" #include "ftl_l2p.h" +#include "utils/ftl_bitmap.h" #include "utils/ftl_log.h" /* @@ -115,6 +116,9 @@ struct spdk_ftl_dev { /* Size of the l2p table */ uint64_t num_lbas; + /* P2L valid map */ + struct ftl_bitmap *valid_map; + /* Metadata size */ uint64_t md_size; diff --git a/lib/ftl/ftl_internal.h b/lib/ftl/ftl_internal.h index de4c78b71..d87ceba74 100644 --- a/lib/ftl/ftl_internal.h +++ b/lib/ftl/ftl_internal.h @@ -11,6 +11,8 @@ #include "spdk/util.h" #include "spdk/uuid.h" +#include "utils/ftl_bitmap.h" + /* Marks address as invalid */ #define FTL_ADDR_INVALID ((ftl_addr)-1) /* Marks LBA as invalid */ @@ -68,6 +70,9 @@ struct ftl_p2l_map { /* P2L map's reference count, prevents premature release of resources during dirty shutdown recovery for open bands */ size_t ref_cnt; + /* Bitmap of valid LBAs */ + struct ftl_bitmap *valid; + /* P2L map (only valid for open/relocating bands) */ union { uint64_t *band_map; diff --git a/lib/ftl/ftl_layout.c b/lib/ftl/ftl_layout.c index 96cf47b30..4dcd9af95 100644 --- a/lib/ftl/ftl_layout.c +++ b/lib/ftl/ftl_layout.c @@ -306,6 +306,17 @@ setup_layout_base(struct spdk_ftl_dev *dev) /* Move offset after base superblock */ offset += layout->region[FTL_LAYOUT_REGION_TYPE_SB_BASE].current.blocks; + /* Setup validity map */ + region = &layout->region[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; + region->type = FTL_LAYOUT_REGION_TYPE_VALID_MAP; + region->name = "vmap"; + region->current.version = region->prev.version = 0; + region->current.offset = offset; + region->current.blocks = blocks_region(spdk_divide_round_up( + layout->base.total_blocks + layout->nvc.total_blocks, 8)); + set_region_bdev_btm(region, dev); + offset += region->current.blocks; + /* Checking for underflow */ left = layout->base.total_blocks - offset; if (left > layout->base.total_blocks) { diff --git a/lib/ftl/ftl_layout.h b/lib/ftl/ftl_layout.h index b1d2b8f75..7d84f067c 100644 --- a/lib/ftl/ftl_layout.h +++ b/lib/ftl/ftl_layout.h @@ -28,6 +28,9 @@ enum ftl_layout_region_type { /* Mirrored instance of bands state */ FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR, + /* Map of valid physical addresses, used for more efficient garbage collection */ + FTL_LAYOUT_REGION_TYPE_VALID_MAP, + /* State of chunks */ FTL_LAYOUT_REGION_TYPE_NVC_MD, /* Mirrored instance of the state of chunks */ @@ -44,7 +47,7 @@ enum ftl_layout_region_type { /* last nvc/base region in terms of lba address space */ #define FTL_LAYOUT_REGION_LAST_NVC FTL_LAYOUT_REGION_TYPE_DATA_NVC -#define FTL_LAYOUT_REGION_LAST_BASE FTL_LAYOUT_REGION_TYPE_DATA_BASE +#define FTL_LAYOUT_REGION_LAST_BASE FTL_LAYOUT_REGION_TYPE_VALID_MAP #define FTL_LAYOUT_REGION_TYPE_FREE_BASE (UINT32_MAX - 2) #define FTL_LAYOUT_REGION_TYPE_FREE_NVC (UINT32_MAX - 1) #define FTL_LAYOUT_REGION_TYPE_INVALID (UINT32_MAX) diff --git a/lib/ftl/ftl_nv_cache.c b/lib/ftl/ftl_nv_cache.c index 5ae5b3e7e..1cb04f8d7 100644 --- a/lib/ftl/ftl_nv_cache.c +++ b/lib/ftl/ftl_nv_cache.c @@ -1170,6 +1170,7 @@ ftl_nv_cache_set_addr(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr) assert(lba != FTL_LBA_INVALID); ftl_chunk_set_addr(chunk, lba, addr); + ftl_bitmap_set(dev->valid_map, addr); } static void ftl_chunk_open(struct ftl_nv_cache_chunk *chunk); diff --git a/lib/ftl/ftl_reloc.c b/lib/ftl/ftl_reloc.c index ca08a91b1..04455f3cf 100644 --- a/lib/ftl/ftl_reloc.c +++ b/lib/ftl/ftl_reloc.c @@ -292,6 +292,7 @@ move_advance_rq(struct ftl_rq *rq) offset = ftl_band_block_offset_from_addr(band, rq->io.addr); assert(offset < ftl_get_num_blocks_in_band(band->dev)); + assert(ftl_band_block_offset_valid(band, offset)); entry->lba = band->p2l_map.band_map[offset]; entry->addr = rq->io.addr; diff --git a/lib/ftl/mngt/ftl_mngt_band.c b/lib/ftl/mngt/ftl_mngt_band.c index 1ddd4305a..daaa67df5 100644 --- a/lib/ftl/mngt/ftl_mngt_band.c +++ b/lib/ftl/mngt/ftl_mngt_band.c @@ -12,9 +12,25 @@ static int ftl_band_init_md(struct ftl_band *band) { struct spdk_ftl_dev *dev = band->dev; + struct ftl_p2l_map *p2l_map = &band->p2l_map; struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; + struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; + uint64_t band_num_blocks = ftl_get_num_blocks_in_band(band->dev); + size_t band_valid_map_bytes; struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md); + if (band_num_blocks % (ftl_bitmap_buffer_alignment * 8)) { + FTL_ERRLOG(dev, "The number of blocks in band is not divisible by bitmap word bits\n"); + return -EINVAL; + } + band_valid_map_bytes = band_num_blocks / 8; + + p2l_map->valid = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md) + + band_valid_map_bytes * band->id, band_valid_map_bytes); + if (!p2l_map->valid) { + return -ENOMEM; + } + band->md = &band_md[band->id]; if (!ftl_fast_startup(dev)) { band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; @@ -73,6 +89,22 @@ ftl_dev_deinit_bands(struct spdk_ftl_dev *dev) free(dev->bands); } +static void +ftl_dev_deinit_bands_md(struct spdk_ftl_dev *dev) +{ + if (dev->bands) { + uint64_t i; + for (i = 0; i < dev->num_bands; ++i) { + struct ftl_band *band = &dev->bands[i]; + + ftl_bitmap_destroy(band->p2l_map.valid); + band->p2l_map.valid = NULL; + + band->md = NULL; + } + } +} + void ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { @@ -100,6 +132,13 @@ ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) ftl_mngt_next_step(mngt); } +void +ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + ftl_dev_deinit_bands_md(dev); + ftl_mngt_next_step(mngt); +} + /* * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's * perspective. Improves WAF. diff --git a/lib/ftl/mngt/ftl_mngt_misc.c b/lib/ftl/mngt/ftl_mngt_misc.c index d21ce961e..9dcb52a08 100644 --- a/lib/ftl/mngt/ftl_mngt_misc.c +++ b/lib/ftl/mngt/ftl_mngt_misc.c @@ -218,3 +218,30 @@ ftl_mngt_dump_stats(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) ftl_dev_dump_stats(dev); ftl_mngt_next_step(mngt); } + +void +ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; + + dev->valid_map = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md), + ftl_md_get_buffer_size(valid_map_md)); + if (!dev->valid_map) { + FTL_ERRLOG(dev, "Failed to create valid map\n"); + ftl_mngt_fail_step(mngt); + return; + } + + ftl_mngt_next_step(mngt); +} + +void +ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + if (dev->valid_map) { + ftl_bitmap_destroy(dev->valid_map); + dev->valid_map = NULL; + } + + ftl_mngt_next_step(mngt); +} diff --git a/lib/ftl/mngt/ftl_mngt_startup.c b/lib/ftl/mngt/ftl_mngt_startup.c index 360113c05..9594d8ba4 100644 --- a/lib/ftl/mngt/ftl_mngt_startup.c +++ b/lib/ftl/mngt/ftl_mngt_startup.c @@ -92,9 +92,15 @@ static const struct ftl_mngt_process_desc desc_startup = { .action = ftl_mngt_init_nv_cache, .cleanup = ftl_mngt_deinit_nv_cache }, + { + .name = "Initialize valid map", + .action = ftl_mngt_init_vld_map, + .cleanup = ftl_mngt_deinit_vld_map + }, { .name = "Initialize bands metadata", - .action = ftl_mngt_init_bands_md + .action = ftl_mngt_init_bands_md, + .cleanup = ftl_mngt_deinit_bands_md }, { .name = "Initialize reloc", diff --git a/lib/ftl/mngt/ftl_mngt_steps.h b/lib/ftl/mngt/ftl_mngt_steps.h index b42c4af3a..411ef6c33 100644 --- a/lib/ftl/mngt/ftl_mngt_steps.h +++ b/lib/ftl/mngt/ftl_mngt_steps.h @@ -42,6 +42,8 @@ void ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *m void ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); @@ -86,6 +88,10 @@ void ftl_mngt_init_default_sb(struct spdk_ftl_dev *dev, struct ftl_mngt_process void ftl_mngt_set_dirty(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + +void ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_persist_band_info_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); void ftl_mngt_persist_nv_cache_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); diff --git a/lib/ftl/utils/ftl_md.c b/lib/ftl/utils/ftl_md.c index 856a42d6e..6c90a0be5 100644 --- a/lib/ftl/utils/ftl_md.c +++ b/lib/ftl/utils/ftl_md.c @@ -1125,6 +1125,11 @@ ftl_md_create_region_flags(struct spdk_ftl_dev *dev, int region_type) flags |= FTL_MD_CREATE_SHM_NEW; } break; + case FTL_LAYOUT_REGION_TYPE_VALID_MAP: + if (!ftl_fast_startup(dev) && !ftl_fast_recovery(dev)) { + flags |= FTL_MD_CREATE_SHM_NEW; + } + break; default: return FTL_MD_CREATE_HEAP; } @@ -1138,6 +1143,7 @@ ftl_md_destroy_region_flags(struct spdk_ftl_dev *dev, int region_type) switch (region_type) { case FTL_LAYOUT_REGION_TYPE_SB: case FTL_LAYOUT_REGION_TYPE_BAND_MD: + case FTL_LAYOUT_REGION_TYPE_VALID_MAP: case FTL_LAYOUT_REGION_TYPE_NVC_MD: if (dev->conf.fast_shutdown) { return FTL_MD_DESTROY_SHM_KEEP; diff --git a/test/unit/lib/ftl/common/utils.c b/test/unit/lib/ftl/common/utils.c index ebfad3a76..5a7dfc7e7 100644 --- a/test/unit/lib/ftl/common/utils.c +++ b/test/unit/lib/ftl/common/utils.c @@ -116,6 +116,9 @@ test_init_ftl_band(struct spdk_ftl_dev *dev, size_t id, size_t zone_size) band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; TAILQ_INSERT_HEAD(&dev->shut_bands, band, queue_entry); + band->p2l_map.valid = (struct ftl_bitmap *)spdk_bit_array_create(ftl_get_num_blocks_in_band(dev)); + SPDK_CU_ASSERT_FATAL(band->p2l_map.valid != NULL); + band->start_addr = zone_size * id; return band; @@ -149,6 +152,7 @@ void test_free_ftl_band(struct ftl_band *band) { SPDK_CU_ASSERT_FATAL(band != NULL); + spdk_bit_array_free((struct spdk_bit_array **)&band->p2l_map.valid); spdk_dma_free(band->p2l_map.band_dma_md); band->p2l_map.band_dma_md = NULL; diff --git a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c b/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c index 9815a3f22..ddc8ab667 100644 --- a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c +++ b/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c @@ -120,6 +120,39 @@ DEFINE_STUB(ftl_writer_is_halted, bool, (struct ftl_writer *writer), true); DEFINE_STUB(ftl_mempool_claim_df, void *, (struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id), NULL); +static void +adjust_bitmap(struct ftl_bitmap **bitmap, uint64_t *bit) +{ + if (*bitmap == g_dev->valid_map) { + *bitmap = g_band->p2l_map.valid; + *bit = test_offset_from_addr(*bit, g_band); + } +} + +bool +ftl_bitmap_get(const struct ftl_bitmap *bitmap, uint64_t bit) +{ + adjust_bitmap((struct ftl_bitmap **)&bitmap, &bit); + return spdk_bit_array_get((struct spdk_bit_array *)bitmap, bit); +} + +void +ftl_bitmap_set(struct ftl_bitmap *bitmap, uint64_t bit) +{ + int ret; + + adjust_bitmap(&bitmap, &bit); + ret = spdk_bit_array_set((struct spdk_bit_array *)bitmap, bit); + CU_ASSERT_EQUAL(ret, 0); +} + +void +ftl_bitmap_clear(struct ftl_bitmap *bitmap, uint64_t bit) +{ + adjust_bitmap(&bitmap, &bit); + spdk_bit_array_clear((struct spdk_bit_array *)bitmap, bit); +} + static void setup_band(void) { @@ -220,14 +253,17 @@ test_band_set_addr(void) ftl_band_set_addr(g_band, TEST_LBA, addr); CU_ASSERT_EQUAL(p2l_map->num_valid, 1); CU_ASSERT_EQUAL(p2l_map->band_map[offset], TEST_LBA); + CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset)); addr += g_geo.zone_size / 2; offset = test_offset_from_addr(addr, g_band); ftl_band_set_addr(g_band, TEST_LBA + 1, addr); CU_ASSERT_EQUAL(p2l_map->num_valid, 2); CU_ASSERT_EQUAL(p2l_map->band_map[offset], TEST_LBA + 1); + CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset)); addr -= g_geo.zone_size / 2; offset = test_offset_from_addr(addr, g_band); + CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset)); cleanup_band(); } @@ -236,23 +272,33 @@ test_invalidate_addr(void) { struct ftl_p2l_map *p2l_map; ftl_addr addr; + uint64_t offset[2]; setup_band(); p2l_map = &g_band->p2l_map; addr = addr_from_zone_id(0); addr += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); + offset[0] = test_offset_from_addr(addr, g_band); ftl_band_set_addr(g_band, TEST_LBA, addr); CU_ASSERT_EQUAL(p2l_map->num_valid, 1); + CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[0])); ftl_invalidate_addr(g_band->dev, addr); CU_ASSERT_EQUAL(p2l_map->num_valid, 0); + CU_ASSERT_FALSE(ftl_bitmap_get(p2l_map->valid, offset[0])); + offset[0] = test_offset_from_addr(addr, g_band); ftl_band_set_addr(g_band, TEST_LBA, addr); addr += g_geo.zone_size / 2; + offset[1] = test_offset_from_addr(addr, g_band); ftl_band_set_addr(g_band, TEST_LBA + 1, addr); CU_ASSERT_EQUAL(p2l_map->num_valid, 2); + CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[0])); + CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[1])); ftl_invalidate_addr(g_band->dev, addr); CU_ASSERT_EQUAL(p2l_map->num_valid, 1); + CU_ASSERT_TRUE(ftl_bitmap_get(p2l_map->valid, offset[0])); + CU_ASSERT_FALSE(ftl_bitmap_get(p2l_map->valid, offset[1])); cleanup_band(); }