lib/ftl: track non-volatile cache's write sequence

This patch adds tracking of the phase of the writes to the non-volatile
cache. The phase is changed each time the whole buffer is filled. Along
with every block's LBA, current phase is stored in its metadata. This
allows for replaying the sequence of writes when recovering the data
from the cache after (unclean) shutdown.

Since there are only three possible phases to be stored on the device at
a time, phase is defined as a 2-bit counter cycling through 1 -> 2 -> 3
-> 1, with 0 marking blocks that were never written.

Change-Id: Id47880367934027fd102c32f183110acc9d4c62a
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/458098
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Mateusz Kozlowski <mateusz.kozlowski@intel.com>

lib/ftl/ftl_core.c

@@ -979,6 +979,7 @@ ftl_nv_cache_wrap(void *ctx)
 	hdr->uuid = dev->uuid;
 	hdr->size = spdk_bdev_get_num_blocks(bdev);
 	hdr->version = FTL_NV_CACHE_HEADER_VERSION;
+	hdr->phase = (uint8_t)nv_cache->phase;
 	hdr->checksum = spdk_crc32c_update(hdr, offsetof(struct ftl_nv_cache_header, checksum), 0);
 
 	rc = spdk_bdev_write_blocks(nv_cache->bdev_desc, ioch->cache_ioch, hdr, 0, 1,
@@ -992,7 +993,7 @@ ftl_nv_cache_wrap(void *ctx)
 }
 
 static uint64_t
-ftl_reserve_nv_cache(struct ftl_nv_cache *nv_cache, size_t *num_lbks)
+ftl_reserve_nv_cache(struct ftl_nv_cache *nv_cache, size_t *num_lbks, unsigned int *phase)
 {
 	struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc);
 	struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache);
@@ -1017,9 +1018,11 @@ ftl_reserve_nv_cache(struct ftl_nv_cache *nv_cache, size_t *num_lbks)
 	cache_addr = nv_cache->current_addr;
 	nv_cache->current_addr += *num_lbks;
 	nv_cache->num_available -= *num_lbks;
+	*phase = nv_cache->phase;
 
 	if (nv_cache->current_addr == spdk_bdev_get_num_blocks(bdev)) {
 		nv_cache->current_addr = FTL_NV_CACHE_DATA_OFFSET;
+		nv_cache->phase = ftl_nv_cache_next_phase(nv_cache->phase);
 		nv_cache->ready = false;
 		spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_nv_cache_wrap, dev);
 	}
@@ -1095,15 +1098,19 @@ ftl_submit_nv_cache(void *ctx)
 }
 
 static void
-ftl_nv_cache_fill_md(struct ftl_nv_cache *nv_cache, struct ftl_io *io)
+ftl_nv_cache_fill_md(struct ftl_io *io, unsigned int phase)
 {
-	struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc);
+	struct spdk_bdev *bdev;
+	struct ftl_nv_cache *nv_cache = &io->dev->nv_cache;
+	uint64_t lbk_off, lba;
 	void *md_buf = io->md;
-	size_t lbk_off;
+
+	bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc);
 
 	for (lbk_off = 0; lbk_off < io->lbk_cnt; ++lbk_off) {
-		*(uint64_t *)md_buf = ftl_io_get_lba(io, lbk_off);
-		md_buf = (char *)md_buf + spdk_bdev_get_md_size(bdev);
+		lba = ftl_nv_cache_pack_lba(ftl_io_get_lba(io, lbk_off), phase);
+		memcpy(md_buf, &lba, sizeof(lba));
+		md_buf += spdk_bdev_get_md_size(bdev);
 	}
 }
 
@@ -1113,6 +1120,7 @@ _ftl_write_nv_cache(void *ctx)
 	struct ftl_io *child, *io = ctx;
 	struct spdk_ftl_dev *dev = io->dev;
 	struct spdk_thread *thread;
+	unsigned int phase;
 	uint64_t num_lbks;
 
 	thread = spdk_io_channel_get_thread(io->ioch);
@@ -1134,7 +1142,7 @@ _ftl_write_nv_cache(void *ctx)
 		}
 
 		/* Reserve area on the write buffer cache */
-		child->ppa.ppa = ftl_reserve_nv_cache(&dev->nv_cache, &num_lbks);
+		child->ppa.ppa = ftl_reserve_nv_cache(&dev->nv_cache, &num_lbks, &phase);
 		if (child->ppa.ppa == FTL_LBA_INVALID) {
 			spdk_mempool_put(dev->nv_cache.md_pool, child->md);
 			ftl_io_free(child);
@@ -1147,7 +1155,7 @@ _ftl_write_nv_cache(void *ctx)
 			ftl_io_shrink_iovec(child, num_lbks);
 		}
 
-		ftl_nv_cache_fill_md(&dev->nv_cache, child);
+		ftl_nv_cache_fill_md(child, phase);
 		ftl_submit_nv_cache(child);
 	}
 

lib/ftl/ftl_core.h

@@ -112,6 +112,13 @@ struct ftl_nv_cache {
 	uint64_t				num_available;
 	/* Maximum number of blocks */
 	uint64_t				num_data_blocks;
+	/*
+	 * Phase of the current cycle of writes. Each time whole cache area is filled, the phase is
+	 * advanced. Current phase is saved in every IO's metadata, as well as in the header saved
+	 * in the first sector. By looking at the phase of each block, it's possible to find the
+	 * oldest block and replay the order of the writes when recovering the data from the cache.
+	 */
+	unsigned int				phase;
 	/* Indicates that the data can be written to the cache */
 	bool					ready;
 	/* Metadata pool */
@@ -247,6 +254,8 @@ struct ftl_nv_cache_header {
 	struct spdk_uuid			uuid;
 	/* Size of the non-volatile cache (in blocks) */
 	uint64_t				size;
+	/* Current phase */
+	uint8_t					phase;
 	/* Checksum of the header, needs to be last element */
 	uint32_t				checksum;
 } __attribute__((packed));
@@ -480,7 +489,31 @@ ftl_vld_map_size(const struct spdk_ftl_dev *dev)
 	return (size_t)spdk_divide_round_up(ftl_num_band_lbks(dev), CHAR_BIT);
 }
 
-#define FTL_NV_CACHE_HEADER_VERSION	1
-#define FTL_NV_CACHE_DATA_OFFSET	1
+#define FTL_NV_CACHE_HEADER_VERSION	(1)
+#define FTL_NV_CACHE_DATA_OFFSET	(1)
+#define FTL_NV_CACHE_PHASE_OFFSET	(62)
+#define FTL_NV_CACHE_PHASE_MASK		(3ULL << FTL_NV_CACHE_PHASE_OFFSET)
+
+static inline bool
+ftl_nv_cache_phase_is_valid(unsigned int phase)
+{
+	return phase > 0 && phase <= 3;
+}
+
+static inline unsigned int
+ftl_nv_cache_next_phase(unsigned int current)
+{
+	static const unsigned int phases[] = { 0, 2, 3, 1 };
+	assert(ftl_nv_cache_phase_is_valid(current));
+	return phases[current];
+}
+
+static inline uint64_t
+ftl_nv_cache_pack_lba(uint64_t lba, unsigned int phase)
+{
+	assert(ftl_nv_cache_phase_is_valid(phase));
+	return (lba & ~FTL_NV_CACHE_PHASE_MASK) | ((uint64_t)phase << FTL_NV_CACHE_PHASE_OFFSET);
+}
+
 
 #endif /* FTL_CORE_H */

lib/ftl/ftl_init.c

@@ -909,6 +909,7 @@ ftl_clear_nv_cache_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 	hdr->uuid = dev->uuid;
 	hdr->size = spdk_bdev_get_num_blocks(bdev);
 	hdr->version = FTL_NV_CACHE_HEADER_VERSION;
+	hdr->phase = dev->nv_cache.phase = 1;
 	hdr->checksum = spdk_crc32c_update(hdr, offsetof(struct ftl_nv_cache_header, checksum), 0);
 
 	if (spdk_bdev_write_blocks(nv_cache->bdev_desc, ioch->cache_ioch, hdr, 0, 1,

lib/ftl/ftl_restore.c

@@ -422,6 +422,12 @@ ftl_nv_cache_header_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
 		goto out;
 	}
 
+	if (!ftl_nv_cache_phase_is_valid(hdr->phase)) {
+		SPDK_ERRLOG("Invalid phase of the non-volatile cache (%u)\n", hdr->phase);
+		rc = -ENOTRECOVERABLE;
+		goto out;
+	}
+
 	pthread_spin_lock(&nv_cache->lock);
 	nv_cache->ready = true;
 	pthread_spin_unlock(&nv_cache->lock);