FTL: Persist metadata on clean shutdown

Add an extra step during FTL shutdown to save all metadata.

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: Idc2f77e15bbd02028548cc88355cd450175830e8
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13359
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/ftl/ftl_l2p.c

@@ -81,6 +81,12 @@ ftl_l2p_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 	FTL_L2P_OP(clear)(dev, cb, cb_ctx);
 }
 
+void
+ftl_l2p_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
+{
+	FTL_L2P_OP(persist)(dev, cb, cb_ctx);
+}
+
 void
 ftl_l2p_process(struct spdk_ftl_dev *dev)
 {

lib/ftl/ftl_l2p.h

@@ -42,6 +42,7 @@ ftl_addr ftl_l2p_get(struct spdk_ftl_dev *dev, uint64_t lba);
 
 void ftl_l2p_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_restore(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
+void ftl_l2p_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_process(struct spdk_ftl_dev *dev);
 bool ftl_l2p_is_halted(struct spdk_ftl_dev *dev);
 void ftl_l2p_halt(struct spdk_ftl_dev *dev);

lib/ftl/ftl_l2p_cache.c

@@ -24,6 +24,7 @@
 struct ftl_l2p_cache_page_io_ctx {
 	struct ftl_l2p_cache *cache;
 	uint64_t updates;
+	spdk_bdev_io_completion_cb cb;
 	struct spdk_bdev_io_wait_entry bdev_io_wait;
 };
 
@@ -509,6 +510,99 @@ ftl_l2p_cache_deinit(struct spdk_ftl_dev *dev)
 	dev->l2p = 0;
 }
 
+static void
+process_init_ctx(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
+		 ftl_l2p_cb cb, void *cb_ctx)
+{
+	struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
+
+	assert(NULL == ctx->cb_ctx);
+	assert(0 == cache->l2_pgs_evicting);
+
+	memset(ctx, 0, sizeof(*ctx));
+
+	ctx->cb = cb;
+	ctx->cb_ctx = cb_ctx;
+}
+
+static void
+process_finish(struct ftl_l2p_cache *cache)
+{
+	struct ftl_l2p_cache_process_ctx ctx = cache->mctx;
+
+	assert(cache->l2_pgs_avail == cache->l2_pgs_resident_max);
+	assert(0 == ctx.qd);
+
+	memset(&cache->mctx, 0, sizeof(cache->mctx));
+	ctx.cb(cache->dev, ctx.status, ctx.cb_ctx);
+}
+
+static void process_page_out_retry(void *_page);
+static void process_persist(struct ftl_l2p_cache *cache);
+
+static void
+process_persist_page_out_cb(struct spdk_bdev_io *bdev_io, bool success, void *arg)
+{
+	struct ftl_l2p_page *page = arg;
+	struct ftl_l2p_cache *cache = page->ctx.cache;
+	struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
+
+	assert(bdev_io);
+	spdk_bdev_free_io(bdev_io);
+
+	if (!success) {
+		ctx->status = -EIO;
+	}
+
+	ftl_l2p_cache_page_remove(cache, page);
+
+	ctx->qd--;
+	process_persist(cache);
+}
+
+static void
+process_page_out(struct ftl_l2p_page *page, spdk_bdev_io_completion_cb cb)
+{
+	struct spdk_bdev *bdev;
+	struct spdk_bdev_io_wait_entry *bdev_io_wait;
+	struct ftl_l2p_cache *cache = page->ctx.cache;
+	struct spdk_ftl_dev *dev = cache->dev;
+	int rc;
+
+	assert(page->page_buffer);
+
+	rc = ftl_nv_cache_bdev_write_blocks_with_md(dev, ftl_l2p_cache_get_bdev_desc(cache),
+			ftl_l2p_cache_get_bdev_iochannel(cache),
+			page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
+			1, cb, page);
+
+	if (spdk_likely(0 == rc)) {
+		return;
+	}
+
+	if (rc == -ENOMEM) {
+		bdev = spdk_bdev_desc_get_bdev(ftl_l2p_cache_get_bdev_desc(cache));
+		bdev_io_wait = &page->ctx.bdev_io_wait;
+		bdev_io_wait->bdev = bdev;
+		bdev_io_wait->cb_fn = process_page_out_retry;
+		bdev_io_wait->cb_arg = page;
+		page->ctx.cb = cb;
+
+		rc = spdk_bdev_queue_io_wait(bdev, ftl_l2p_cache_get_bdev_iochannel(cache), bdev_io_wait);
+		ftl_bug(rc);
+	} else {
+		ftl_abort();
+	}
+}
+
+static void
+process_page_out_retry(void *_page)
+{
+	struct ftl_l2p_page *page = _page;
+
+	process_page_out(page, page->ctx.cb);
+}
+
 static void
 clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status)
 {
@@ -531,6 +625,48 @@ ftl_l2p_cache_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 	ftl_md_clear(md, invalid_addr, NULL);
 }
 
+static void
+process_persist(struct ftl_l2p_cache *cache)
+{
+	struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
+
+	while (ctx->idx < cache->num_pages && ctx->qd < 64) {
+		struct ftl_l2p_page *page = get_l2p_page_by_df_id(cache, ctx->idx);
+		ctx->idx++;
+
+		if (!page) {
+			continue;
+		}
+
+		if (page->on_lru_list) {
+			ftl_l2p_cache_lru_remove_page(cache, page);
+		}
+
+		if (page->updates) {
+			/* Need to persist the page */
+			page->state = L2P_CACHE_PAGE_PERSISTING;
+			page->ctx.cache = cache;
+			ctx->qd++;
+			process_page_out(page, process_persist_page_out_cb);
+		} else {
+			ftl_l2p_cache_page_remove(cache, page);
+		}
+	}
+
+	if (0 == ctx->qd) {
+		process_finish(cache);
+	}
+}
+
+void
+ftl_l2p_cache_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
+{
+	struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
+
+	process_init_ctx(dev, cache, cb, cb_ctx);
+	process_persist(cache);
+}
+
 bool
 ftl_l2p_cache_is_halted(struct spdk_ftl_dev *dev)
 {

lib/ftl/ftl_l2p_cache.h

@@ -17,6 +17,7 @@ void ftl_l2p_cache_unpin(struct spdk_ftl_dev *dev, uint64_t lba, uint64_t count)
 ftl_addr ftl_l2p_cache_get(struct spdk_ftl_dev *dev, uint64_t lba);
 void ftl_l2p_cache_set(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr);
 void ftl_l2p_cache_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
+void ftl_l2p_cache_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_cache_process(struct spdk_ftl_dev *dev);
 bool ftl_l2p_cache_is_halted(struct spdk_ftl_dev *dev);
 void ftl_l2p_cache_halt(struct spdk_ftl_dev *dev);

lib/ftl/ftl_l2p_flat.c

@@ -80,6 +80,18 @@ ftl_l2p_flat_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 	ftl_md_persist(md);
 }
 
+void
+ftl_l2p_flat_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
+{
+	struct ftl_md *md;
+
+	md = get_l2p_md(dev);
+	md->cb = md_cb;
+	md->owner.cb_ctx = cb_ctx;
+	md->owner.private = cb;
+	ftl_md_persist(md);
+}
+
 static int
 ftl_l2p_flat_init_dram(struct spdk_ftl_dev *dev, struct ftl_l2p_flat *l2p_flat,
 		       size_t l2p_size)

lib/ftl/ftl_l2p_flat.h

@@ -14,6 +14,7 @@ ftl_addr ftl_l2p_flat_get(struct spdk_ftl_dev *dev, uint64_t lba);
 void ftl_l2p_flat_set(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr);
 void ftl_l2p_flat_unmap(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_flat_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
+void ftl_l2p_flat_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_flat_process(struct spdk_ftl_dev *dev);
 bool ftl_l2p_flat_is_halted(struct spdk_ftl_dev *dev);
 void ftl_l2p_flat_halt(struct spdk_ftl_dev *dev);

lib/ftl/mngt/ftl_mngt_l2p.c

@@ -45,3 +45,9 @@ ftl_mngt_clear_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {
 	ftl_l2p_clear(dev, l2p_cb, mngt);
 }
+
+void
+ftl_mngt_persist_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	ftl_l2p_persist(dev, l2p_cb, mngt);
+}

lib/ftl/mngt/ftl_mngt_md.c

@@ -138,9 +138,14 @@ ftl_mngt_persist_nv_cache_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_pro
 	persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_NVC_MD);
 }
 
+static void
+ftl_mngt_persist_vld_map_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_VALID_MAP);
+}
+
 void
-ftl_mngt_persist_band_info_metadata(
-	struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+ftl_mngt_persist_band_info_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {
 	persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_BAND_MD);
 }
@@ -163,6 +168,61 @@ get_sb_crc(struct ftl_superblock *sb)
 	return crc;
 }
 
+static void
+ftl_mngt_persist_super_block(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	dev->sb->overprovisioning = dev->conf.overprovisioning;
+	dev->sb->gc_info = dev->sb_shm->gc_info;
+	dev->sb->header.crc = get_sb_crc(dev->sb);
+	persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_SB);
+}
+
+#ifdef SPDK_FTL_VSS_EMU
+static void
+ftl_mngt_persist_vss(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_VSS);
+}
+#endif
+
+/*
+ * Persists all necessary metadata (band state, P2L, etc) during FTL's clean shutdown.
+ */
+static const struct ftl_mngt_process_desc desc_persist = {
+	.name = "Persist metadata",
+	.steps = {
+		{
+			.name = "Persist NV cache metadata",
+			.action = ftl_mngt_persist_nv_cache_metadata,
+		},
+		{
+			.name = "Persist valid map metadata",
+			.action = ftl_mngt_persist_vld_map_metadata,
+		},
+		{
+			.name = "persist band info metadata",
+			.action = ftl_mngt_persist_band_info_metadata,
+		},
+		{
+			.name = "Persist superblock",
+			.action = ftl_mngt_persist_super_block,
+		},
+#ifdef SPDK_FTL_VSS_EMU
+		{
+			.name = "Persist VSS metadata",
+			.action = ftl_mngt_persist_vss,
+		},
+#endif
+		{}
+	}
+};
+
+void
+ftl_mngt_persist_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	ftl_mngt_call_process(mngt, &desc_persist);
+}
+
 void
 ftl_mngt_init_default_sb(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {
@@ -201,6 +261,19 @@ ftl_mngt_set_dirty(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 	persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_SB);
 }
 
+void
+ftl_mngt_set_clean(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	struct ftl_superblock *sb = dev->sb;
+
+	sb->clean = 1;
+	dev->sb_shm->shm_clean = false;
+	sb->header.crc = get_sb_crc(sb);
+	persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_SB);
+
+	dev->sb_shm->shm_ready = false;
+}
+
 /*
  * Initializes the superblock fields during first startup of FTL
  */

lib/ftl/mngt/ftl_mngt_shutdown.c

@@ -27,6 +27,18 @@ static const struct ftl_mngt_process_desc desc_shutdown = {
 			.name = "Stop core poller",
 			.action = ftl_mngt_stop_core_poller
 		},
+		{
+			.name = "Persist L2P",
+			.action = ftl_mngt_persist_l2p
+		},
+		{
+			.name = "Persist metadata",
+			.action = ftl_mngt_persist_md
+		},
+		{
+			.name = "Set FTL clean state",
+			.action = ftl_mngt_set_clean
+		},
 		{
 			.name = "Dump statistics",
 			.action = ftl_mngt_dump_stats

lib/ftl/mngt/ftl_mngt_steps.h

@@ -74,12 +74,16 @@ void ftl_mngt_start_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_proces
 
 void ftl_mngt_stop_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_persist_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_init_layout(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_init_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_deinit_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_persist_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_rollback_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_dump_stats(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
@@ -88,6 +92,8 @@ void ftl_mngt_init_default_sb(struct spdk_ftl_dev *dev, struct ftl_mngt_process
 
 void ftl_mngt_set_dirty(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_set_clean(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);