FTL: Add band P2L map usage

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I7f526c80667ab548a2903689066ac76a8d8d3c53
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13325
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

lib/ftl/ftl_band.c

@@ -29,6 +29,36 @@ ftl_band_filled(struct ftl_band *band, size_t offset)
 	return offset == ftl_band_tail_md_offset(band);
 }
 
+static void
+ftl_band_free_p2l_map(struct ftl_band *band)
+{
+	struct spdk_ftl_dev *dev = band->dev;
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
+
+	assert(band->md->state == FTL_BAND_STATE_CLOSED ||
+	       band->md->state == FTL_BAND_STATE_FREE);
+	assert(p2l_map->ref_cnt == 0);
+	assert(p2l_map->band_map != NULL);
+
+	ftl_mempool_put(dev->p2l_pool, p2l_map->band_map);
+	p2l_map->band_map = NULL;
+}
+
+
+static void
+ftl_band_free_md_entry(struct ftl_band *band)
+{
+	struct spdk_ftl_dev *dev = band->dev;
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
+
+	assert(band->md->state == FTL_BAND_STATE_CLOSED ||
+	       band->md->state == FTL_BAND_STATE_FREE);
+	assert(p2l_map->band_dma_md != NULL);
+
+	ftl_mempool_put(dev->band_md_pool, p2l_map->band_dma_md);
+	p2l_map->band_dma_md = NULL;
+}
+
 ftl_addr
 ftl_band_tail_md_addr(struct ftl_band *band)
 {
@@ -56,6 +86,18 @@ ftl_band_set_type(struct ftl_band *band, enum ftl_band_type type)
 	}
 }
 
+void
+ftl_band_set_addr(struct ftl_band *band, uint64_t lba, ftl_addr addr)
+{
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
+	uint64_t offset;
+
+	offset = ftl_band_block_offset_from_addr(band, addr);
+
+	p2l_map->band_map[offset] = lba;
+	p2l_map->num_valid++;
+}
+
 size_t
 ftl_band_user_blocks_left(const struct ftl_band *band, size_t offset)
 {
@@ -140,8 +182,92 @@ ftl_band_next_addr(struct ftl_band *band, ftl_addr addr, size_t offset)
 	return ftl_band_addr_from_block_offset(band, block_off + offset);
 }
 
+void
+ftl_band_acquire_p2l_map(struct ftl_band *band)
+{
+	assert(band->p2l_map.band_map != NULL);
+	band->p2l_map.ref_cnt++;
+}
+
+static int
+ftl_band_alloc_md_entry(struct ftl_band *band)
+{
+	struct spdk_ftl_dev *dev = band->dev;
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
+	struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_BAND_MD];
+
+	p2l_map->band_dma_md = ftl_mempool_get(dev->band_md_pool);
+
+	if (!p2l_map->band_dma_md) {
+		return -1;
+	}
+
+	memset(p2l_map->band_dma_md, 0, region->entry_size * FTL_BLOCK_SIZE);
+	return 0;
+}
+
+int
+ftl_band_alloc_p2l_map(struct ftl_band *band)
+{
+	struct spdk_ftl_dev *dev = band->dev;
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
+
+	assert(p2l_map->ref_cnt == 0);
+	assert(p2l_map->band_map == NULL);
+
+	p2l_map->band_map = ftl_mempool_get(dev->p2l_pool);
+	if (!p2l_map->band_map) {
+		return -1;
+	}
+
+	if (ftl_band_alloc_md_entry(band)) {
+		ftl_band_free_p2l_map(band);
+		return -1;
+	}
+
+	/* Set the P2L to FTL_LBA_INVALID */
+	memset(p2l_map->band_map, -1, FTL_BLOCK_SIZE * ftl_p2l_map_num_blocks(band->dev));
+
+	ftl_band_acquire_p2l_map(band);
+	return 0;
+}
+
+void
+ftl_band_release_p2l_map(struct ftl_band *band)
+{
+	struct ftl_p2l_map *p2l_map = &band->p2l_map;
+
+	assert(p2l_map->band_map != NULL);
+	assert(p2l_map->ref_cnt > 0);
+	p2l_map->ref_cnt--;
+
+	if (p2l_map->ref_cnt == 0) {
+		ftl_band_free_p2l_map(band);
+		ftl_band_free_md_entry(band);
+	}
+}
+
 ftl_addr
 ftl_band_p2l_map_addr(struct ftl_band *band)
 {
 	return band->tail_md_addr;
 }
+
+int
+ftl_band_write_prep(struct ftl_band *band)
+{
+	if (ftl_band_alloc_p2l_map(band)) {
+		return -1;
+	}
+
+	ftl_band_iter_init(band);
+
+	return 0;
+}
+
+size_t
+ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev)
+{
+	/* Map pool element holds the whole tail md */
+	return ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE;
+}

lib/ftl/ftl_band.h

@@ -131,12 +131,18 @@ uint64_t ftl_band_block_offset_from_addr(struct ftl_band *band, ftl_addr addr);
 ftl_addr ftl_band_addr_from_block_offset(struct ftl_band *band, uint64_t block_off);
 void ftl_band_set_type(struct ftl_band *band, enum ftl_band_type type);
 void ftl_band_set_state(struct ftl_band *band, enum ftl_band_state state);
+void ftl_band_acquire_p2l_map(struct ftl_band *band);
+int ftl_band_alloc_p2l_map(struct ftl_band *band);
+void ftl_band_release_p2l_map(struct ftl_band *band);
 ftl_addr ftl_band_next_xfer_addr(struct ftl_band *band, ftl_addr addr, size_t num_blocks);
 ftl_addr ftl_band_next_addr(struct ftl_band *band, ftl_addr addr, size_t offset);
 size_t ftl_band_user_blocks_left(const struct ftl_band *band, size_t offset);
+void ftl_band_set_addr(struct ftl_band *band, uint64_t lba, ftl_addr addr);
 struct ftl_band *ftl_band_from_addr(struct spdk_ftl_dev *dev, ftl_addr addr);
 ftl_addr ftl_band_tail_md_addr(struct ftl_band *band);
 int ftl_band_filled(struct ftl_band *band, size_t offset);
+int ftl_band_write_prep(struct ftl_band *band);
+size_t ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev);
 ftl_addr ftl_band_p2l_map_addr(struct ftl_band *band);
 
 static inline void

lib/ftl/ftl_core.c

@@ -53,9 +53,25 @@ ftl_shutdown_complete(struct spdk_ftl_dev *dev)
 void
 ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr)
 {
+	struct ftl_band *band;
+	struct ftl_p2l_map *p2l_map;
+
 	if (ftl_addr_in_nvc(dev, addr)) {
 		return;
 	}
+
+	band = ftl_band_from_addr(dev, addr);
+	p2l_map = &band->p2l_map;
+
+	/* TODO: fix case when the same address is invalidated from multiple sources */
+	assert(p2l_map->num_valid > 0);
+	p2l_map->num_valid--;
+
+	/* Invalidate open/full band p2l_map entry to keep p2l and l2p
+	 * consistency when band is going to close state */
+	if (FTL_BAND_STATE_OPEN == band->md->state || FTL_BAND_STATE_FULL == band->md->state) {
+		p2l_map->band_map[ftl_band_block_offset_from_addr(band, addr)] = FTL_LBA_INVALID;
+	}
 }
 
 void

lib/ftl/ftl_core.h

@@ -26,7 +26,6 @@
 /*
  * We need to reserve at least 2 buffers for band close / open sequence
  * alone, plus additional (8) buffers for handling relocations.
- *
  */
 #define P2L_MEMPOOL_SIZE (2 + 8)
 
@@ -72,6 +71,12 @@ struct spdk_ftl_dev {
 	/* Non-volatile write buffer cache */
 	struct ftl_nv_cache		nv_cache;
 
+	/* P2L map memory pool */
+	struct ftl_mempool		*p2l_pool;
+
+	/* Band md memory pool */
+	struct ftl_mempool		*band_md_pool;
+
 	/* counters for poller busy, include
 	   1. nv cache read/write
 	   2. metadata read/write

lib/ftl/ftl_l2p.c

@@ -166,6 +166,7 @@ ftl_l2p_update_base(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr new_addr, f
 		/* DO NOT CHANGE ORDER - START (need to set L2P (and valid bits), before invalidating old ones,
 		 * due to dirty shutdown from shm recovery - it's ok to have too many bits set, but not ok to
 		 * have too many cleared) */
+		ftl_band_set_addr(ftl_band_from_addr(dev, new_addr), lba, new_addr);
 		ftl_l2p_set(dev, lba, new_addr);
 		/* DO NOT CHANGE ORDER - END */
 	} else {

lib/ftl/mngt/ftl_mngt_misc.c

@@ -7,6 +7,7 @@
 #include "ftl_utils.h"
 #include "ftl_mngt.h"
 #include "ftl_mngt_steps.h"
+#include "ftl_band.h"
 #include "ftl_internal.h"
 #include "ftl_nv_cache.h"
 #include "ftl_debug.h"
@@ -21,6 +22,66 @@ ftl_mngt_check_conf(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 	}
 }
 
+static int
+init_p2l_map_pool(struct spdk_ftl_dev *dev)
+{
+	size_t p2l_pool_el_blks = spdk_divide_round_up(ftl_p2l_map_pool_elem_size(dev), FTL_BLOCK_SIZE);
+
+	dev->p2l_pool = ftl_mempool_create(P2L_MEMPOOL_SIZE,
+					   p2l_pool_el_blks * FTL_BLOCK_SIZE,
+					   FTL_BLOCK_SIZE,
+					   SPDK_ENV_SOCKET_ID_ANY);
+	if (!dev->p2l_pool) {
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int
+init_band_md_pool(struct spdk_ftl_dev *dev)
+{
+	dev->band_md_pool = ftl_mempool_create(P2L_MEMPOOL_SIZE,
+					       sizeof(struct ftl_band_md),
+					       FTL_BLOCK_SIZE,
+					       SPDK_ENV_SOCKET_ID_ANY);
+	if (!dev->band_md_pool) {
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void
+ftl_mngt_init_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	if (init_p2l_map_pool(dev)) {
+		ftl_mngt_fail_step(mngt);
+	}
+
+	if (init_band_md_pool(dev)) {
+		ftl_mngt_fail_step(mngt);
+	}
+
+	ftl_mngt_next_step(mngt);
+}
+
+void
+ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
+{
+	if (dev->p2l_pool) {
+		ftl_mempool_destroy(dev->p2l_pool);
+		dev->p2l_pool = NULL;
+	}
+
+	if (dev->band_md_pool) {
+		ftl_mempool_destroy(dev->band_md_pool);
+		dev->band_md_pool = NULL;
+	}
+
+	ftl_mngt_next_step(mngt);
+}
+
 void
 ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
 {

lib/ftl/mngt/ftl_mngt_startup.c

@@ -54,6 +54,11 @@ static const struct ftl_mngt_process_desc desc_startup = {
 			.action = ftl_mngt_superblock_init,
 			.cleanup = ftl_mngt_superblock_deinit
 		},
+		{
+			.name = "Initialize memory pools",
+			.action = ftl_mngt_init_mem_pools,
+			.cleanup = ftl_mngt_deinit_mem_pools
+		},
 		{
 			.name = "Register IO device",
 			.action = ftl_mngt_register_io_device,

lib/ftl/mngt/ftl_mngt_steps.h

@@ -32,6 +32,10 @@ void ftl_mngt_register_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_proce
 
 void ftl_mngt_unregister_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
+void ftl_mngt_init_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
+void ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
+
 void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
 
 void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);