From b3e5d8a7239adc79b3e91bec77e5d7fd39a33464 Mon Sep 17 00:00:00 2001 From: Kozlowski Mateusz Date: Fri, 27 May 2022 13:46:59 +0200 Subject: [PATCH] ftl: Add recovery and restart path for trim Restores necessary metadata and sets L2P during clean/dirty shutdown recovery process. Signed-off-by: Kozlowski Mateusz Signed-off-by: Artur Paszkiewicz Change-Id: Iaa44025250b44f424ac9de5859d1db82900ecaa9 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13380 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- lib/ftl/ftl_l2p.c | 6 ++ lib/ftl/ftl_l2p_cache.c | 115 +++++++++++++++++++++++++++ lib/ftl/mngt/ftl_mngt_l2p.c | 6 ++ lib/ftl/mngt/ftl_mngt_md.c | 29 +++++++ lib/ftl/mngt/ftl_mngt_recovery.c | 128 ++++++++++++++++++++++++++++++- lib/ftl/mngt/ftl_mngt_shutdown.c | 4 + 6 files changed, 286 insertions(+), 2 deletions(-) diff --git a/lib/ftl/ftl_l2p.c b/lib/ftl/ftl_l2p.c index b5403d120..1fee554aa 100644 --- a/lib/ftl/ftl_l2p.c +++ b/lib/ftl/ftl_l2p.c @@ -93,6 +93,12 @@ ftl_l2p_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx) FTL_L2P_OP(persist)(dev, cb, cb_ctx); } +void +ftl_l2p_unmap(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx) +{ + FTL_L2P_OP(unmap)(dev, cb, cb_ctx); +} + void ftl_l2p_process(struct spdk_ftl_dev *dev) { diff --git a/lib/ftl/ftl_l2p_cache.c b/lib/ftl/ftl_l2p_cache.c index 20df08f67..a6a7bac5b 100644 --- a/lib/ftl/ftl_l2p_cache.c +++ b/lib/ftl/ftl_l2p_cache.c @@ -567,6 +567,24 @@ process_finish(struct ftl_l2p_cache *cache) static void process_page_out_retry(void *_page); static void process_persist(struct ftl_l2p_cache *cache); +static void +process_page_in(struct ftl_l2p_page *page, spdk_bdev_io_completion_cb cb) +{ + struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache; + int rc; + + assert(page->page_buffer); + + rc = ftl_nv_cache_bdev_read_blocks_with_md(cache->dev, ftl_l2p_cache_get_bdev_desc(cache), + ftl_l2p_cache_get_bdev_iochannel(cache), + page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page), + 1, cb, page); + + if (rc) { + cb(NULL, false, page); + } +} + static void process_persist_page_out_cb(struct spdk_bdev_io *bdev_io, bool success, void *arg) { @@ -638,6 +656,103 @@ process_page_out_retry(void *_page) process_page_out(page, page->ctx.cb); } +static void process_unmap(struct ftl_l2p_cache *cache); + +static void +process_unmap_page_out_cb(struct spdk_bdev_io *bdev_io, bool success, void *ctx_page) +{ + struct ftl_l2p_page *page = (struct ftl_l2p_page *)ctx_page; + struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache; + struct spdk_ftl_dev *dev = cache->dev; + struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx; + + assert(bdev_io); + spdk_bdev_free_io(bdev_io); + + if (!success) { + ctx->status = -EIO; + } + + assert(!page->on_lru_list); + assert(ftl_bitmap_get(dev->unmap_map, page->page_no)); + ftl_bitmap_clear(dev->unmap_map, page->page_no); + ftl_l2p_cache_page_remove(cache, page); + + ctx->qd--; + process_unmap(cache); +} + +static void +process_unmap_page_in_cb(struct spdk_bdev_io *bdev_io, bool success, void *ctx_page) +{ + struct ftl_l2p_page *page = (struct ftl_l2p_page *)ctx_page; + struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache; + struct spdk_ftl_dev *dev = cache->dev; + struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx; + + if (bdev_io) { + spdk_bdev_free_io(bdev_io); + } + if (success) { + assert(ftl_bitmap_get(dev->unmap_map, page->page_no)); + ftl_l2p_page_set_invalid(dev, page); + process_page_out(page, process_unmap_page_out_cb); + } else { + ctx->status = -EIO; + ctx->qd--; + process_unmap(cache); + } +} + +static void +process_unmap(struct ftl_l2p_cache *cache) +{ + struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx; + + while (ctx->idx < cache->num_pages && ctx->qd < 64) { + struct ftl_l2p_page *page; + + if (!ftl_bitmap_get(cache->dev->unmap_map, ctx->idx)) { + /* Page had not been unmapped, continue */ + ctx->idx++; + continue; + } + + /* All pages were removed in persist phase */ + assert(get_l2p_page_by_df_id(cache, ctx->idx) == NULL); + + /* Allocate page to invalidate it */ + page = ftl_l2p_cache_page_alloc(cache, ctx->idx); + if (!page) { + /* All pages utilized so far, continue when they will be back available */ + assert(ctx->qd); + break; + } + + page->state = L2P_CACHE_PAGE_CLEARING; + page->ctx.cache = cache; + + ftl_l2p_cache_page_insert(cache, page); + process_page_in(page, process_unmap_page_in_cb); + + ctx->qd++; + ctx->idx++; + } + + if (0 == ctx->qd) { + process_finish(cache); + } +} + +void +ftl_l2p_cache_unmap(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx) +{ + struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p; + + process_init_ctx(dev, cache, cb, cb_ctx); + process_unmap(cache); +} + static void clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) { diff --git a/lib/ftl/mngt/ftl_mngt_l2p.c b/lib/ftl/mngt/ftl_mngt_l2p.c index 1e5bf765f..800bb7a00 100644 --- a/lib/ftl/mngt/ftl_mngt_l2p.c +++ b/lib/ftl/mngt/ftl_mngt_l2p.c @@ -52,6 +52,12 @@ ftl_mngt_persist_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) ftl_l2p_persist(dev, l2p_cb, mngt); } +void +ftl_mngt_unmap_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + ftl_l2p_unmap(dev, l2p_cb, mngt); +} + void ftl_mngt_restore_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { diff --git a/lib/ftl/mngt/ftl_mngt_md.c b/lib/ftl/mngt/ftl_mngt_md.c index 086632208..8a7ef805b 100644 --- a/lib/ftl/mngt/ftl_mngt_md.c +++ b/lib/ftl/mngt/ftl_mngt_md.c @@ -236,6 +236,12 @@ ftl_mngt_persist_band_info_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_pr persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_BAND_MD); } +static void +ftl_mngt_persist_trim_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_TRIM_MD); +} + static uint32_t get_sb_crc(struct ftl_superblock *sb) { @@ -294,6 +300,10 @@ static const struct ftl_mngt_process_desc desc_persist = { .name = "persist band info metadata", .action = ftl_mngt_persist_band_info_metadata, }, + { + .name = "persist trim metadata", + .action = ftl_mngt_persist_trim_metadata, + }, { .name = "Persist superblock", .action = ftl_mngt_persist_super_block, @@ -674,6 +684,21 @@ ftl_mngt_restore_band_info_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_pr restore(dev, mngt, FTL_LAYOUT_REGION_TYPE_BAND_MD); } +static void +ftl_mngt_restore_trim_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + if (ftl_fast_startup(dev)) { + FTL_DEBUGLOG(dev, "SHM: found trim md\n"); + if (ftl_md_restore_region(dev, FTL_LAYOUT_REGION_TYPE_TRIM_MD)) { + ftl_mngt_fail_step(mngt); + return; + } + ftl_mngt_next_step(mngt); + return; + } + restore(dev, mngt, FTL_LAYOUT_REGION_TYPE_TRIM_MD); +} + #ifdef SPDK_FTL_VSS_EMU @@ -708,6 +733,10 @@ static const struct ftl_mngt_process_desc desc_restore = { .name = "Restore band info metadata", .action = ftl_mngt_restore_band_info_metadata, }, + { + .name = "Restore trim metadata", + .action = ftl_mngt_restore_trim_metadata, + }, {} } }; diff --git a/lib/ftl/mngt/ftl_mngt_recovery.c b/lib/ftl/mngt/ftl_mngt_recovery.c index 8a293520f..c07944525 100644 --- a/lib/ftl/mngt/ftl_mngt_recovery.c +++ b/lib/ftl/mngt/ftl_mngt_recovery.c @@ -321,9 +321,31 @@ static void ftl_mngt_recovery_iteration_init_seq_ids(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { struct ftl_mngt_recovery_ctx *ctx = ftl_mngt_get_caller_ctx(mngt); - size_t size = sizeof(ctx->l2p_snippet.seq_id[0]) * ctx->l2p_snippet.count; + struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; + uint64_t *trim_map = ftl_md_get_buffer(md); + uint64_t page_id, trim_seq_id; + uint32_t lbas_in_page = FTL_BLOCK_SIZE / dev->layout.l2p.addr_size; + uint64_t lba, lba_off; - memset(ctx->l2p_snippet.seq_id, 0, size); + if (dev->sb->ckpt_seq_id) { + FTL_ERRLOG(dev, "Checkpoint recovery not supported!\n"); + ftl_mngt_fail_step(mngt); + return; + } + + for (lba = ctx->iter.lba_first; lba < ctx->iter.lba_last; lba++) { + lba_off = lba - ctx->iter.lba_first; + page_id = lba / lbas_in_page; + + assert(page_id < ftl_md_get_buffer_size(md) / sizeof(*trim_map)); + assert(page_id < dev->layout.region[FTL_LAYOUT_REGION_TYPE_L2P].current.blocks); + assert(lba_off < ctx->l2p_snippet.count); + + trim_seq_id = trim_map[page_id]; + + ctx->l2p_snippet.seq_id[lba_off] = trim_seq_id; + ftl_addr_store(dev, ctx->l2p_snippet.l2p, lba_off, FTL_ADDR_INVALID); + } ftl_mngt_next_step(mngt); } @@ -691,6 +713,100 @@ ftl_mngt_restore_valid_counters(struct spdk_ftl_dev *dev, struct ftl_mngt_proces ftl_mngt_next_step(mngt); } +static void +ftl_mngt_complete_unmap_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) +{ + struct ftl_mngt_process *mngt = md->owner.cb_ctx; + + dev->sb_shm->trim.in_progress = false; + + if (!status) { + ftl_mngt_next_step(mngt); + } else { + ftl_mngt_fail_step(mngt); + } +} + +static void +ftl_mngt_complete_unmap(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + uint64_t start_lba, num_blocks, seq_id; + struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; + + if (dev->sb_shm->trim.in_progress) { + start_lba = dev->sb_shm->trim.start_lba; + num_blocks = dev->sb_shm->trim.num_blocks; + seq_id = dev->sb_shm->trim.seq_id; + + assert(seq_id <= dev->sb->seq_id); + + FTL_NOTICELOG(dev, "Uncomplete unmap detected lba: %"PRIu64" num_blocks: %"PRIu64"\n", + start_lba, num_blocks); + + ftl_set_unmap_map(dev, start_lba, num_blocks, seq_id); + } + + md->owner.cb_ctx = mngt; + md->cb = ftl_mngt_complete_unmap_cb; + + ftl_md_persist(md); +} + +static void +ftl_mngt_recover_unmap_map_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) +{ + struct ftl_mngt_process *mngt = md->owner.cb_ctx; + uint64_t num_md_blocks, first_page, num_pages; + uint32_t lbas_in_page = FTL_BLOCK_SIZE / dev->layout.l2p.addr_size; + uint64_t *page = ftl_md_get_buffer(md); + union ftl_md_vss *page_vss = ftl_md_get_vss_buffer(md); + uint64_t lba, num_blocks, vss_seq_id; + size_t i, j; + + if (status) { + ftl_mngt_fail_step(mngt); + return; + } + + num_md_blocks = ftl_md_get_buffer_size(md) / lbas_in_page; + + for (i = 0; i < num_md_blocks; ++i, page_vss++) { + lba = page_vss->unmap.start_lba; + num_blocks = page_vss->unmap.num_blocks; + vss_seq_id = page_vss->unmap.seq_id; + + first_page = lba / lbas_in_page; + num_pages = num_blocks / lbas_in_page; + + if (lba % lbas_in_page || num_blocks % lbas_in_page) { + ftl_mngt_fail_step(mngt); + return; + } + + for (j = first_page; j < first_page + num_pages; ++j) { + page[j] = spdk_max(vss_seq_id, page[j]); + } + } + + ftl_mngt_next_step(mngt); +} + +static void +ftl_mngt_recover_unmap_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; + + if (ftl_fast_recovery(dev)) { + FTL_DEBUGLOG(dev, "SHM: skipping unmap map recovery\n"); + ftl_mngt_next_step(mngt); + return; + } + + md->owner.cb_ctx = mngt; + md->cb = ftl_mngt_recover_unmap_map_cb; + ftl_md_restore(md); +} + static void ftl_mngt_recovery_shm_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { @@ -780,6 +896,10 @@ static const struct ftl_mngt_process_desc g_desc_recovery = { .name = "Recover max seq ID", .action = ftl_mngt_recover_seq_id }, + { + .name = "Recover unmap map", + .action = ftl_mngt_recover_unmap_map + }, { .name = "Recover open chunks P2L", .action = ftl_mngt_nv_cache_recover_open_chunk @@ -842,6 +962,10 @@ static const struct ftl_mngt_process_desc g_desc_recovery_shm = { .name = "Restore valid maps counters", .action = ftl_mngt_restore_valid_counters, }, + { + .name = "Complete unmap transaction", + .action = ftl_mngt_complete_unmap, + }, {} } }; diff --git a/lib/ftl/mngt/ftl_mngt_shutdown.c b/lib/ftl/mngt/ftl_mngt_shutdown.c index b875af421..01b63d930 100644 --- a/lib/ftl/mngt/ftl_mngt_shutdown.c +++ b/lib/ftl/mngt/ftl_mngt_shutdown.c @@ -31,6 +31,10 @@ static const struct ftl_mngt_process_desc desc_shutdown = { .name = "Persist L2P", .action = ftl_mngt_persist_l2p }, + { + .name = "Finish L2P unmaps", + .action = ftl_mngt_unmap_l2p, + }, { .name = "Persist metadata", .action = ftl_mngt_persist_md