FTL: Add L2P restore path

Adds initialization code for L2P done after shutdown (both clean and
dirty).

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I7a938b298467c96d68f40cb14c3171d1533e1a08
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13361
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

lib/ftl/ftl_l2p.c

@@ -81,6 +81,12 @@ ftl_l2p_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 	FTL_L2P_OP(clear)(dev, cb, cb_ctx);
 }
 
+void
+ftl_l2p_restore(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
+{
+	FTL_L2P_OP(restore)(dev, cb, cb_ctx);
+}
+
 void
 ftl_l2p_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 {

lib/ftl/ftl_l2p_cache.c

@@ -625,6 +625,107 @@ ftl_l2p_cache_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 	ftl_md_clear(md, invalid_addr, NULL);
 }
 
+static void
+l2p_shm_restore_clean(struct spdk_ftl_dev *dev)
+{
+	struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
+	struct ftl_l2p_l1_map_entry *me = cache->l2_mapping;
+	struct ftl_l2p_page *page;
+	ftl_df_obj_id obj_id;
+	uint64_t page_no;
+
+	for (page_no = 0; page_no < cache->num_pages; ++page_no) {
+		obj_id = me[page_no].page_obj_id;
+		if (obj_id == FTL_DF_OBJ_ID_INVALID) {
+			continue;
+		}
+
+		page = ftl_mempool_claim_df(cache->l2_ctx_pool, obj_id);
+		assert(page);
+		assert(page->obj_id == ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page));
+		assert(page->page_no == page_no);
+		assert(page->state != L2P_CACHE_PAGE_INIT);
+		assert(page->state != L2P_CACHE_PAGE_CLEARING);
+		assert(cache->l2_pgs_avail > 0);
+		cache->l2_pgs_avail--;
+
+		page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
+					    cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
+
+		TAILQ_INIT(&page->ppe_list);
+
+		page->pin_ref_cnt = 0;
+		page->on_lru_list = 0;
+		memset(&page->ctx, 0, sizeof(page->ctx));
+
+		ftl_l2p_cache_lru_add_page(cache, page);
+	}
+
+	ftl_mempool_initialize_ext(cache->l2_ctx_pool);
+}
+
+static void
+l2p_shm_restore_dirty(struct spdk_ftl_dev *dev)
+{
+	struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
+	struct ftl_l2p_l1_map_entry *me = cache->l2_mapping;
+	struct ftl_l2p_page *page;
+	ftl_df_obj_id obj_id;
+	uint64_t page_no;
+
+	for (page_no = 0; page_no < cache->num_pages; ++page_no) {
+		obj_id = me[page_no].page_obj_id;
+		if (obj_id == FTL_DF_OBJ_ID_INVALID) {
+			continue;
+		}
+
+		page = ftl_mempool_claim_df(cache->l2_ctx_pool, obj_id);
+		assert(page);
+		assert(page->obj_id == ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page));
+		assert(page->page_no == page_no);
+		assert(page->state != L2P_CACHE_PAGE_CLEARING);
+		assert(cache->l2_pgs_avail > 0);
+		cache->l2_pgs_avail--;
+
+		if (page->state == L2P_CACHE_PAGE_INIT) {
+			me[page_no].page_obj_id = FTL_DF_OBJ_ID_INVALID;
+			cache->l2_pgs_avail++;
+			ftl_mempool_release_df(cache->l2_ctx_pool, obj_id);
+			continue;
+		}
+
+		page->state = L2P_CACHE_PAGE_READY;
+		/* Assume page is dirty after crash */
+		page->updates = 1;
+		page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
+					    cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
+
+		TAILQ_INIT(&page->ppe_list);
+
+		page->pin_ref_cnt = 0;
+		page->on_lru_list = 0;
+		memset(&page->ctx, 0, sizeof(page->ctx));
+
+		ftl_l2p_cache_lru_add_page(cache, page);
+	}
+
+	ftl_mempool_initialize_ext(cache->l2_ctx_pool);
+}
+
+void
+ftl_l2p_cache_restore(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
+{
+	if (ftl_fast_startup(dev)) {
+		l2p_shm_restore_clean(dev);
+	}
+
+	if (ftl_fast_recovery(dev)) {
+		l2p_shm_restore_dirty(dev);
+	}
+
+	cb(dev, 0, cb_ctx);
+}
+
 static void
 process_persist(struct ftl_l2p_cache *cache)
 {

lib/ftl/ftl_l2p_cache.h

@@ -17,6 +17,7 @@ void ftl_l2p_cache_unpin(struct spdk_ftl_dev *dev, uint64_t lba, uint64_t count)
 ftl_addr ftl_l2p_cache_get(struct spdk_ftl_dev *dev, uint64_t lba);
 void ftl_l2p_cache_set(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr);
 void ftl_l2p_cache_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
+void ftl_l2p_cache_restore(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_cache_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_cache_process(struct spdk_ftl_dev *dev);
 bool ftl_l2p_cache_is_halted(struct spdk_ftl_dev *dev);

lib/ftl/ftl_l2p_flat.c

@@ -80,6 +80,18 @@ ftl_l2p_flat_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 	ftl_md_persist(md);
 }
 
+void
+ftl_l2p_flat_restore(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
+{
+	struct ftl_md *md;
+
+	md = get_l2p_md(dev);
+	md->cb = md_cb;
+	md->owner.cb_ctx = cb_ctx;
+	md->owner.private = cb;
+	ftl_md_restore(md);
+}
+
 void
 ftl_l2p_flat_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx)
 {

lib/ftl/ftl_l2p_flat.h

@@ -14,6 +14,7 @@ ftl_addr ftl_l2p_flat_get(struct spdk_ftl_dev *dev, uint64_t lba);
 void ftl_l2p_flat_set(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr);
 void ftl_l2p_flat_unmap(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_flat_clear(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
+void ftl_l2p_flat_restore(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_flat_persist(struct spdk_ftl_dev *dev, ftl_l2p_cb cb, void *cb_ctx);
 void ftl_l2p_flat_process(struct spdk_ftl_dev *dev);
 bool ftl_l2p_flat_is_halted(struct spdk_ftl_dev *dev);

lib/ftl/mngt/ftl_mngt_l2p.c

@@ -51,3 +51,9 @@ ftl_mngt_persist_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {
 	ftl_l2p_persist(dev, l2p_cb, mngt);
 }
+
+void
+ftl_mngt_restore_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	ftl_l2p_restore(dev, l2p_cb, mngt);
+}

lib/ftl/mngt/ftl_mngt_steps.h

@@ -64,6 +64,8 @@ void ftl_mngt_deinit_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt
 
 void ftl_mngt_clear_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_restore_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);