From a68a12a478e2db9157d4c5396cb8243e67d4eb73 Mon Sep 17 00:00:00 2001 From: Kozlowski Mateusz Date: Mon, 13 Jun 2022 11:54:12 +0200 Subject: [PATCH] FTL: Initial nv cache structure Signed-off-by: Kozlowski Mateusz Signed-off-by: Artur Paszkiewicz Change-Id: Ie40cc25ed9bf28976a5ae6d6a67491f438152fca Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13317 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Community-CI: Broadcom CI Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- lib/ftl/Makefile | 1 + lib/ftl/ftl_core.c | 11 +- lib/ftl/ftl_core.h | 13 +- lib/ftl/ftl_init.c | 1 + lib/ftl/ftl_internal.h | 29 +++ lib/ftl/ftl_io.c | 1 - lib/ftl/ftl_l2p.c | 1 + lib/ftl/ftl_layout.c | 87 +++++++-- lib/ftl/ftl_layout.h | 9 + lib/ftl/ftl_nv_cache.c | 249 +++++++++++++++++++++++++ lib/ftl/ftl_nv_cache.h | 155 +++++++++++++++ lib/ftl/ftl_nv_cache_io.h | 8 +- lib/ftl/mngt/ftl_mngt_bdev.c | 38 ++-- lib/ftl/mngt/ftl_mngt_ioch.c | 1 - lib/ftl/mngt/ftl_mngt_md.c | 2 + lib/ftl/mngt/ftl_mngt_misc.c | 22 +++ lib/ftl/mngt/ftl_mngt_startup.c | 5 + lib/ftl/mngt/ftl_mngt_steps.h | 4 + lib/ftl/utils/ftl_md.c | 4 +- test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c | 6 +- 20 files changed, 594 insertions(+), 53 deletions(-) create mode 100644 lib/ftl/ftl_nv_cache.c create mode 100644 lib/ftl/ftl_nv_cache.h diff --git a/lib/ftl/Makefile b/lib/ftl/Makefile index 8a1fe66f2..43b42b7cf 100644 --- a/lib/ftl/Makefile +++ b/lib/ftl/Makefile @@ -22,6 +22,7 @@ CFLAGS += -I. FTL_SUBDIRS := mngt utils C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c ftl_sb.c ftl_l2p.c ftl_l2p_flat.c +C_SRCS += ftl_nv_cache.c C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c mngt/ftl_mngt_l2p.c C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 004f5560b..7d3083de8 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -17,7 +17,6 @@ #include "ftl_debug.h" #include "ftl_internal.h" #include "mngt/ftl_mngt.h" -#include "utils/ftl_mempool.h" size_t @@ -33,6 +32,15 @@ ftl_shutdown_complete(struct spdk_ftl_dev *dev) return false; } + if (!ftl_nv_cache_is_halted(&dev->nv_cache)) { + ftl_nv_cache_halt(&dev->nv_cache); + return false; + } + + if (!ftl_nv_cache_chunks_busy(&dev->nv_cache)) { + return false; + } + if (!ftl_l2p_is_halted(dev)) { ftl_l2p_halt(dev); return false; @@ -141,6 +149,7 @@ ftl_core_poller(void *ctx) } ftl_process_io_queue(dev); + ftl_nv_cache_process(dev); ftl_l2p_process(dev); if (io_activity_total_old != dev->io_activity_total) { diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 1cc92cae4..5b5526d46 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -18,6 +18,7 @@ #include "ftl_internal.h" #include "ftl_io.h" +#include "ftl_nv_cache.h" #include "ftl_layout.h" #include "ftl_sb.h" #include "ftl_l2p.h" @@ -46,12 +47,6 @@ struct spdk_ftl_dev { /* Underlying device */ struct spdk_bdev_desc *base_bdev_desc; - /* Cache device */ - struct spdk_bdev_desc *cache_bdev_desc; - - /* Cache VSS metadata size */ - uint64_t cache_md_size; - /* Cached properties of the underlying device */ uint64_t num_blocks_in_band; uint64_t num_zones_in_band; @@ -70,6 +65,9 @@ struct spdk_ftl_dev { /* Management process to be continued after IO device unregistration completes */ struct ftl_mngt_process *unregister_process; + /* Non-volatile write buffer cache */ + struct ftl_nv_cache nv_cache; + /* counters for poller busy, include 1. nv cache read/write 2. metadata read/write @@ -111,9 +109,6 @@ struct spdk_ftl_dev { /* Underlying device IO channel */ struct spdk_io_channel *base_ioch; - /* Cache IO channel */ - struct spdk_io_channel *cache_ioch; - /* Poller */ struct spdk_poller *core_poller; diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index f87612080..d6a0aa408 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -18,6 +18,7 @@ #include "ftl_core.h" #include "ftl_io.h" #include "ftl_debug.h" +#include "ftl_nv_cache.h" #include "ftl_utils.h" #include "mngt/ftl_mngt.h" diff --git a/lib/ftl/ftl_internal.h b/lib/ftl/ftl_internal.h index aa8dc612c..d61696d06 100644 --- a/lib/ftl/ftl_internal.h +++ b/lib/ftl/ftl_internal.h @@ -27,4 +27,33 @@ typedef uint64_t ftl_addr; /* Number of LBAs that could be stored in a single block */ #define FTL_NUM_LBA_IN_BLOCK (FTL_BLOCK_SIZE / sizeof(uint64_t)) +/* + * Mapping of physical (actual location on disk) to logical (user's POV) addresses. Used in two main scenarios: + * - during relocation FTL needs to pin L2P pages (this allows to check which pages to pin) and move still valid blocks + * (valid map allows for preliminary elimination of invalid physical blocks, but user data could invalidate a location + * during read/write operation, so actual comparision against L2P needs to be done) + * - After dirty shutdown the state of the L2P is unknown and needs to be rebuilt - it is done by applying all P2L, taking + * into account ordering of user writes + */ +struct ftl_p2l_map { + /* Number of valid LBAs */ + size_t num_valid; + + /* P2L map's reference count, prevents premature release of resources during dirty shutdown recovery for open bands */ + size_t ref_cnt; + + /* P2L map (only valid for open/relocating bands) */ + union { + uint64_t *band_map; + void *chunk_map; + }; + + /* DMA buffer for region's metadata entry */ + union { + struct ftl_band_md *band_dma_md; + + struct ftl_nv_cache_chunk_md *chunk_dma_md; + }; +}; + #endif /* FTL_INTERNAL_H */ diff --git a/lib/ftl/ftl_io.c b/lib/ftl/ftl_io.c index 24fd09614..778367045 100644 --- a/lib/ftl/ftl_io.c +++ b/lib/ftl/ftl_io.c @@ -11,7 +11,6 @@ #include "ftl_io.h" #include "ftl_core.h" #include "ftl_debug.h" -#include "utils/ftl_mempool.h" void ftl_io_inc_req(struct ftl_io *io) diff --git a/lib/ftl/ftl_l2p.c b/lib/ftl/ftl_l2p.c index 4c3811178..315c473f2 100644 --- a/lib/ftl/ftl_l2p.c +++ b/lib/ftl/ftl_l2p.c @@ -4,6 +4,7 @@ */ #include "ftl_l2p.h" +#include "ftl_nv_cache.h" #include "ftl_l2p_flat.h" #include "ftl_core.h" diff --git a/lib/ftl/ftl_layout.c b/lib/ftl/ftl_layout.c index 27d60950e..0faea992e 100644 --- a/lib/ftl/ftl_layout.c +++ b/lib/ftl/ftl_layout.c @@ -8,8 +8,13 @@ #include "ftl_core.h" #include "ftl_utils.h" #include "ftl_layout.h" +#include "ftl_nv_cache.h" #include "ftl_sb.h" +#define FTL_NV_CACHE_CHUNK_DATA_SIZE(blocks) ((uint64_t)blocks * FTL_BLOCK_SIZE) +#define FTL_NV_CACHE_CHUNK_SIZE(blocks) \ + (FTL_NV_CACHE_CHUNK_DATA_SIZE(blocks) + (2 * FTL_NV_CACHE_CHUNK_MD_SIZE)) + static inline float blocks2mib(uint64_t blocks) { @@ -105,9 +110,9 @@ get_num_user_lbas(struct spdk_ftl_dev *dev) static void set_region_bdev_nvc(struct ftl_layout_region *reg, struct spdk_ftl_dev *dev) { - reg->bdev_desc = dev->cache_bdev_desc; - reg->ioch = dev->cache_ioch; - reg->vss_blksz = dev->cache_md_size; + reg->bdev_desc = dev->nv_cache.bdev_desc; + reg->ioch = dev->nv_cache.cache_ioch; + reg->vss_blksz = dev->nv_cache.md_size; } static void @@ -121,9 +126,9 @@ set_region_bdev_btm(struct ftl_layout_region *reg, struct spdk_ftl_dev *dev) static int setup_layout_nvc(struct spdk_ftl_dev *dev) { - uint64_t offset = 0; + uint64_t left, offset = 0; struct ftl_layout *layout = &dev->layout; - struct ftl_layout_region *region; + struct ftl_layout_region *region, *mirror; #ifdef SPDK_FTL_VSS_EMU /* Skip the already init`d VSS region */ @@ -157,15 +162,69 @@ setup_layout_nvc(struct spdk_ftl_dev *dev) goto error; } + /* + * Initialize NV Cache metadata + */ + if (offset >= layout->nvc.total_blocks) { + goto error; + } + + left = layout->nvc.total_blocks - offset; + layout->nvc.chunk_data_blocks = + FTL_NV_CACHE_CHUNK_DATA_SIZE(ftl_get_num_blocks_in_zone(dev)) / FTL_BLOCK_SIZE; + layout->nvc.chunk_meta_size = FTL_NV_CACHE_CHUNK_MD_SIZE; + layout->nvc.chunk_count = (left * FTL_BLOCK_SIZE) / + FTL_NV_CACHE_CHUNK_SIZE(ftl_get_num_blocks_in_zone(dev)); + layout->nvc.chunk_tail_md_num_blocks = ftl_nv_cache_chunk_tail_md_num_blocks(&dev->nv_cache); + + if (0 == layout->nvc.chunk_count) { + goto error; + } + region = &layout->region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; + region->type = FTL_LAYOUT_REGION_TYPE_NVC_MD; + region->mirror_type = FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR; + region->name = "nvc_md"; + region->current.version = region->prev.version = FTL_NVC_VERSION_CURRENT; + region->current.offset = offset; + region->current.blocks = blocks_region(layout->nvc.chunk_count * + sizeof(struct ftl_nv_cache_chunk_md)); + region->entry_size = sizeof(struct ftl_nv_cache_chunk_md) / FTL_BLOCK_SIZE; + region->num_entries = layout->nvc.chunk_count; + set_region_bdev_nvc(region, dev); + offset += region->current.blocks; + + /* + * Initialize NV Cache metadata mirror + */ + mirror = &layout->region[FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR]; + *mirror = *region; + mirror->type = FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR; + mirror->mirror_type = FTL_LAYOUT_REGION_TYPE_INVALID; + mirror->name = "nvc_md_mirror"; + mirror->current.offset += region->current.blocks; + offset += mirror->current.blocks; + + /* + * Initialize data region on NV cache + */ + if (offset >= layout->nvc.total_blocks) { + goto error; + } region = &layout->region[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; region->type = FTL_LAYOUT_REGION_TYPE_DATA_NVC; region->name = "data_nvc"; region->current.version = region->prev.version = 0; region->current.offset = offset; - region->current.blocks = layout->nvc.total_blocks - offset; + region->current.blocks = layout->nvc.chunk_count * layout->nvc.chunk_data_blocks; set_region_bdev_nvc(region, dev); offset += region->current.blocks; + left = layout->nvc.total_blocks - offset; + if (left > layout->nvc.chunk_data_blocks) { + FTL_ERRLOG(dev, "Error when setup NV cache layout\n"); + return -1; + } + if (offset > layout->nvc.total_blocks) { FTL_ERRLOG(dev, "Error when setup NV cache layout\n"); goto error; @@ -241,7 +300,7 @@ ftl_layout_setup(struct spdk_ftl_dev *dev) bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); layout->base.total_blocks = spdk_bdev_get_num_blocks(bdev); - bdev = spdk_bdev_desc_get_bdev(dev->cache_bdev_desc); + bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); layout->nvc.total_blocks = spdk_bdev_get_num_blocks(bdev); /* Initialize mirrors types */ @@ -310,13 +369,13 @@ ftl_layout_setup_vss_emu(struct spdk_ftl_dev *dev) region->current.version = region->prev.version = 0; region->current.offset = 0; - bdev = spdk_bdev_desc_get_bdev(dev->cache_bdev_desc); + bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); layout->nvc.total_blocks = spdk_bdev_get_num_blocks(bdev); - region->current.blocks = blocks_region(dev->cache_md_size * layout->nvc.total_blocks); + region->current.blocks = blocks_region(dev->nv_cache.md_size * layout->nvc.total_blocks); region->vss_blksz = 0; - region->bdev_desc = dev->cache_bdev_desc; - region->ioch = dev->cache_ioch; + region->bdev_desc = dev->nv_cache.bdev_desc; + region->ioch = dev->nv_cache.cache_ioch; assert(region->bdev_desc != NULL); assert(region->ioch != NULL); @@ -350,8 +409,8 @@ ftl_layout_setup_superblock(struct spdk_ftl_dev *dev) region->current.blocks = blocks_region(FTL_SUPERBLOCK_SIZE); region->vss_blksz = 0; - region->bdev_desc = dev->cache_bdev_desc; - region->ioch = dev->cache_ioch; + region->bdev_desc = dev->nv_cache.bdev_desc; + region->ioch = dev->nv_cache.cache_ioch; assert(region->bdev_desc != NULL); assert(region->ioch != NULL); @@ -389,7 +448,7 @@ ftl_layout_dump(struct spdk_ftl_dev *dev) int i; FTL_NOTICELOG(dev, "NV cache layout:\n"); for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; ++i) { - if (layout->region[i].bdev_desc == dev->cache_bdev_desc) { + if (layout->region[i].bdev_desc == dev->nv_cache.bdev_desc) { dump_region(dev, &layout->region[i]); } } diff --git a/lib/ftl/ftl_layout.h b/lib/ftl/ftl_layout.h index 0425db64a..846efc3da 100644 --- a/lib/ftl/ftl_layout.h +++ b/lib/ftl/ftl_layout.h @@ -23,6 +23,11 @@ enum ftl_layout_region_type { /* If using cached L2P, this region stores the serialized instance of it */ FTL_LAYOUT_REGION_TYPE_L2P, + /* State of chunks */ + FTL_LAYOUT_REGION_TYPE_NVC_MD, + /* Mirrored instance of the state of chunks */ + FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR, + /* User data region on the nv cache device */ FTL_LAYOUT_REGION_TYPE_DATA_NVC, @@ -99,6 +104,10 @@ struct ftl_layout { /* Organization for NV cache */ struct { uint64_t total_blocks; + uint64_t chunk_data_blocks; + uint64_t chunk_meta_size; + uint64_t chunk_count; + uint64_t chunk_tail_md_num_blocks; } nvc; /* Information corresponding to L2P */ diff --git a/lib/ftl/ftl_nv_cache.c b/lib/ftl/ftl_nv_cache.c new file mode 100644 index 000000000..c2113435b --- /dev/null +++ b/lib/ftl/ftl_nv_cache.c @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + + +#include "spdk/bdev.h" +#include "spdk/bdev_module.h" +#include "spdk/ftl.h" +#include "spdk/string.h" + +#include "ftl_nv_cache.h" +#include "ftl_nv_cache_io.h" +#include "ftl_core.h" +#include "utils/ftl_addr_utils.h" +#include "mngt/ftl_mngt.h" + +static inline const struct ftl_layout_region * +nvc_data_region( + struct ftl_nv_cache *nv_cache) +{ + struct spdk_ftl_dev *dev; + + dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); + return &dev->layout.region[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; +} + +static inline void +nvc_validate_md(struct ftl_nv_cache *nv_cache, + struct ftl_nv_cache_chunk_md *chunk_md) +{ + struct ftl_md *md = nv_cache->md; + void *buffer = ftl_md_get_buffer(md); + uint64_t size = ftl_md_get_buffer_size(md); + void *ptr = chunk_md; + + if (ptr < buffer) { + ftl_abort(); + } + + ptr += sizeof(*chunk_md); + if (ptr > buffer + size) { + ftl_abort(); + } +} + +static inline uint64_t +nvc_data_offset(struct ftl_nv_cache *nv_cache) +{ + return nvc_data_region(nv_cache)->current.offset; +} + +static inline uint64_t +nvc_data_blocks(struct ftl_nv_cache *nv_cache) +{ + return nvc_data_region(nv_cache)->current.blocks; +} + +size_t +ftl_nv_cache_chunk_tail_md_num_blocks(const struct ftl_nv_cache *nv_cache) +{ + struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, + struct spdk_ftl_dev, nv_cache); + return spdk_divide_round_up(dev->layout.nvc.chunk_data_blocks * dev->layout.l2p.addr_size, + FTL_BLOCK_SIZE); +} + +static size_t +nv_cache_p2l_map_pool_elem_size(const struct ftl_nv_cache *nv_cache) +{ + /* Map pool element holds the whole tail md */ + return ftl_nv_cache_chunk_tail_md_num_blocks(nv_cache) * FTL_BLOCK_SIZE; +} + +int +ftl_nv_cache_init(struct spdk_ftl_dev *dev) +{ + struct ftl_nv_cache *nv_cache = &dev->nv_cache; + struct ftl_nv_cache_chunk *chunk; + struct ftl_nv_cache_chunk_md *md; + uint64_t i, offset; + + nv_cache->halt = true; + + nv_cache->md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_NVC_MD]; + if (!nv_cache->md) { + FTL_ERRLOG(dev, "No NV cache metadata object\n"); + return -1; + } + + nv_cache->md_pool = ftl_mempool_create(dev->conf.user_io_pool_size, + nv_cache->md_size * dev->xfer_size, + FTL_BLOCK_SIZE, SPDK_ENV_SOCKET_ID_ANY); + if (!nv_cache->md_pool) { + FTL_ERRLOG(dev, "Failed to initialize NV cache metadata pool\n"); + return -1; + } + + /* + * Initialize chunk info + */ + nv_cache->chunk_blocks = dev->layout.nvc.chunk_data_blocks; + nv_cache->chunk_count = dev->layout.nvc.chunk_count; + + /* Allocate chunks */ + nv_cache->chunks = calloc(nv_cache->chunk_count, + sizeof(nv_cache->chunks[0])); + if (!nv_cache->chunks) { + FTL_ERRLOG(dev, "Failed to initialize NV cache chunks\n"); + return -1; + } + + TAILQ_INIT(&nv_cache->chunk_free_list); + TAILQ_INIT(&nv_cache->chunk_open_list); + TAILQ_INIT(&nv_cache->chunk_full_list); + + /* First chunk metadata */ + md = ftl_md_get_buffer(nv_cache->md); + if (!md) { + FTL_ERRLOG(dev, "No NV cache metadata\n"); + return -1; + } + + nv_cache->chunk_free_count = nv_cache->chunk_count; + + chunk = nv_cache->chunks; + offset = nvc_data_offset(nv_cache); + for (i = 0; i < nv_cache->chunk_count; i++, chunk++, md++) { + chunk->nv_cache = nv_cache; + chunk->md = md; + nvc_validate_md(nv_cache, md); + chunk->offset = offset; + offset += nv_cache->chunk_blocks; + TAILQ_INSERT_TAIL(&nv_cache->chunk_free_list, chunk, entry); + } + assert(offset <= nvc_data_offset(nv_cache) + nvc_data_blocks(nv_cache)); + +#define FTL_MAX_OPEN_CHUNKS 2 + nv_cache->p2l_pool = ftl_mempool_create(FTL_MAX_OPEN_CHUNKS, + nv_cache_p2l_map_pool_elem_size(nv_cache), + FTL_BLOCK_SIZE, + SPDK_ENV_SOCKET_ID_ANY); + if (!nv_cache->p2l_pool) { + return -ENOMEM; + } + + /* One entry per open chunk */ + nv_cache->chunk_md_pool = ftl_mempool_create(FTL_MAX_OPEN_CHUNKS, + sizeof(struct ftl_nv_cache_chunk_md), + FTL_BLOCK_SIZE, + SPDK_ENV_SOCKET_ID_ANY); + if (!nv_cache->chunk_md_pool) { + return -ENOMEM; + } + + return 0; +} + +void +ftl_nv_cache_deinit(struct spdk_ftl_dev *dev) +{ + struct ftl_nv_cache *nv_cache = &dev->nv_cache; + + ftl_mempool_destroy(nv_cache->md_pool); + ftl_mempool_destroy(nv_cache->p2l_pool); + ftl_mempool_destroy(nv_cache->chunk_md_pool); + nv_cache->md_pool = NULL; + nv_cache->p2l_pool = NULL; + nv_cache->chunk_md_pool = NULL; + + free(nv_cache->chunks); + nv_cache->chunks = NULL; +} + +void +ftl_nv_cache_fill_md(struct ftl_io *io) +{ + uint64_t i; + union ftl_md_vss *metadata = io->md; + uint64_t lba = ftl_io_get_lba(io, 0); + + for (i = 0; i < io->num_blocks; ++i, lba++, metadata++) { + metadata->nv_cache.lba = lba; + } +} + +uint64_t +chunk_tail_md_offset(struct ftl_nv_cache *nv_cache) +{ + return nv_cache->chunk_blocks - ftl_nv_cache_chunk_tail_md_num_blocks(nv_cache); +} + +int +ftl_nv_cache_read(struct ftl_io *io, ftl_addr addr, uint32_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + int rc; + struct ftl_nv_cache *nv_cache = &io->dev->nv_cache; + + assert(ftl_addr_in_nvc(io->dev, addr)); + + rc = ftl_nv_cache_bdev_read_blocks_with_md(io->dev, nv_cache->bdev_desc, nv_cache->cache_ioch, + ftl_io_iovec_addr(io), NULL, ftl_addr_to_nvc_offset(io->dev, addr), + num_blocks, cb, cb_arg); + + return rc; +} + +bool +ftl_nv_cache_is_halted(struct ftl_nv_cache *nv_cache) +{ + if (nv_cache->chunk_open_count > 0) { + return false; + } + + return true; +} + +void +ftl_nv_cache_process(struct spdk_ftl_dev *dev) +{ + if (!dev->nv_cache.bdev_desc) { + return; + } +} + +bool +ftl_nv_cache_full(struct ftl_nv_cache *nv_cache) +{ + if (0 == nv_cache->chunk_open_count && NULL == nv_cache->chunk_current) { + return true; + } else { + return false; + } +} + +int +ftl_nv_cache_chunks_busy(struct ftl_nv_cache *nv_cache) +{ + /* chunk_current is migrating to closed status when closing, any others should already be + * moved to free chunk list. */ + return nv_cache->chunk_open_count == 0; +} + +void +ftl_nv_cache_halt(struct ftl_nv_cache *nv_cache) +{ + nv_cache->halt = true; +} diff --git a/lib/ftl/ftl_nv_cache.h b/lib/ftl/ftl_nv_cache.h new file mode 100644 index 000000000..ecb518921 --- /dev/null +++ b/lib/ftl/ftl_nv_cache.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#ifndef FTL_NV_CACHE_H +#define FTL_NV_CACHE_H + +#include "spdk/stdinc.h" +#include "spdk/crc32.h" + +#include "ftl_io.h" +#include "ftl_utils.h" + +#define FTL_NVC_VERSION_0 0 +#define FTL_NVC_VERSION_1 1 + +#define FTL_NVC_VERSION_CURRENT FTL_NVC_VERSION_1 + +struct ftl_nvcache_restore; +typedef void (*ftl_nv_cache_restore_fn)(struct ftl_nvcache_restore *, int, void *cb_arg); + +enum ftl_chunk_state { + FTL_CHUNK_STATE_FREE, + FTL_CHUNK_STATE_OPEN, + FTL_CHUNK_STATE_CLOSED, + FTL_CHUNK_STATE_MAX +}; + +struct ftl_nv_cache_chunk_md { + /* Current lba to write */ + uint32_t write_pointer; + + /* Number of blocks written */ + uint32_t blocks_written; + + /* Number of skipped block (case when IO size is greater than blocks left in chunk) */ + uint32_t blocks_skipped; + + /* Next block to be compacted */ + uint32_t read_pointer; + + /* Number of compacted (both valid and invalid) blocks */ + uint32_t blocks_compacted; + + /* Chunk state */ + enum ftl_chunk_state state; + + /* CRC32 checksum of the associated P2L map when chunk is in closed state */ + uint32_t p2l_map_checksum; +} __attribute__((aligned(FTL_BLOCK_SIZE))); + +#define FTL_NV_CACHE_CHUNK_MD_SIZE sizeof(struct ftl_nv_cache_chunk_md) +SPDK_STATIC_ASSERT(FTL_NV_CACHE_CHUNK_MD_SIZE == FTL_BLOCK_SIZE, + "FTL NV Chunk metadata size is invalid"); + +struct ftl_nv_cache_chunk { + struct ftl_nv_cache *nv_cache; + + struct ftl_nv_cache_chunk_md *md; + + /* Offset from start lba of the cache */ + uint64_t offset; + + /* P2L map */ + struct ftl_p2l_map p2l_map; + + /* Metadata request */ + struct ftl_basic_rq metadata_rq; + + TAILQ_ENTRY(ftl_nv_cache_chunk) entry; + + /* This flag is used to indicate chunk is used in recovery */ + bool recovery; + + /* For writing metadata */ + struct ftl_md_io_entry_ctx md_persist_entry_ctx; +}; + +struct ftl_nv_cache { + /* Flag indicating halt request */ + bool halt; + + /* Write buffer cache bdev */ + struct spdk_bdev_desc *bdev_desc; + + /* Persistent cache IO channel */ + struct spdk_io_channel *cache_ioch; + + /* Metadata pool */ + struct ftl_mempool *md_pool; + + /* P2L map memory pool */ + struct ftl_mempool *p2l_pool; + + /* Chunk md memory pool */ + struct ftl_mempool *chunk_md_pool; + + /* Block Metadata size */ + uint64_t md_size; + + /* NV cache metadata object handle */ + struct ftl_md *md; + + /* Number of blocks in chunk */ + uint64_t chunk_blocks; + + /* Number of chunks */ + uint64_t chunk_count; + + /* Current processed chunk */ + struct ftl_nv_cache_chunk *chunk_current; + + /* Free chunks list */ + TAILQ_HEAD(, ftl_nv_cache_chunk) chunk_free_list; + uint64_t chunk_free_count; + + /* Open chunks list */ + TAILQ_HEAD(, ftl_nv_cache_chunk) chunk_open_list; + uint64_t chunk_open_count; + + /* Full chunks list */ + TAILQ_HEAD(, ftl_nv_cache_chunk) chunk_full_list; + uint64_t chunk_full_count; + + struct ftl_nv_cache_chunk *chunks; +}; + +int ftl_nv_cache_init(struct spdk_ftl_dev *dev); +void ftl_nv_cache_deinit(struct spdk_ftl_dev *dev); +void ftl_nv_cache_fill_md(struct ftl_io *io); +int ftl_nv_cache_read(struct ftl_io *io, ftl_addr addr, uint32_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg); +bool ftl_nv_cache_full(struct ftl_nv_cache *nv_cache); +void ftl_nv_cache_process(struct spdk_ftl_dev *dev); + +void ftl_nv_cache_halt(struct ftl_nv_cache *nv_cache); + +int ftl_nv_cache_chunks_busy(struct ftl_nv_cache *nv_cache); + +static inline void +ftl_nv_cache_resume(struct ftl_nv_cache *nv_cache) +{ + nv_cache->halt = false; +} + +bool ftl_nv_cache_is_halted(struct ftl_nv_cache *nv_cache); + +size_t ftl_nv_cache_chunk_tail_md_num_blocks(const struct ftl_nv_cache *nv_cache); + +uint64_t chunk_tail_md_offset(struct ftl_nv_cache *nv_cache); + +typedef int (*ftl_chunk_md_cb)(struct ftl_nv_cache_chunk *chunk, void *cntx); + +#endif /* FTL_NV_CACHE_H */ diff --git a/lib/ftl/ftl_nv_cache_io.h b/lib/ftl/ftl_nv_cache_io.h index f551f0671..ce2f6f4b6 100644 --- a/lib/ftl/ftl_nv_cache_io.h +++ b/lib/ftl/ftl_nv_cache_io.h @@ -99,7 +99,7 @@ ftl_nv_cache_bdev_readv_blocks_with_md(struct spdk_ftl_dev *dev, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - assert(desc == dev->cache_bdev_desc); + assert(desc == dev->nv_cache.bdev_desc); ftl_nv_cache_bdev_get_md(dev, offset_blocks, num_blocks, md); return spdk_bdev_readv_blocks(desc, ch, iov, iovcnt, offset_blocks, num_blocks, cb, cb_arg); @@ -138,7 +138,7 @@ ftl_nv_cache_bdev_writev_blocks_with_md(struct spdk_ftl_dev *dev, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - assert(desc == dev->cache_bdev_desc); + assert(desc == dev->nv_cache.bdev_desc); ftl_nv_cache_bdev_set_md(dev, offset_blocks, num_blocks, md_buf); return spdk_bdev_writev_blocks(desc, ch, iov, iovcnt, offset_blocks, num_blocks, @@ -153,7 +153,7 @@ ftl_nv_cache_bdev_read_blocks_with_md(struct spdk_ftl_dev *dev, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - assert(desc == dev->cache_bdev_desc); + assert(desc == dev->nv_cache.bdev_desc); ftl_nv_cache_bdev_get_md(dev, offset_blocks, num_blocks, md); return spdk_bdev_read_blocks(desc, ch, buf, offset_blocks, num_blocks, cb, cb_arg); @@ -167,7 +167,7 @@ ftl_nv_cache_bdev_write_blocks_with_md(struct spdk_ftl_dev *dev, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - assert(desc == dev->cache_bdev_desc); + assert(desc == dev->nv_cache.bdev_desc); ftl_nv_cache_bdev_set_md(dev, offset_blocks, num_blocks, md); return spdk_bdev_write_blocks(desc, ch, buf, offset_blocks, num_blocks, diff --git a/lib/ftl/mngt/ftl_mngt_bdev.c b/lib/ftl/mngt/ftl_mngt_bdev.c index 8666a5754..82da9184d 100644 --- a/lib/ftl/mngt/ftl_mngt_bdev.c +++ b/lib/ftl/mngt/ftl_mngt_bdev.c @@ -6,6 +6,7 @@ #include "spdk/bdev_module.h" #include "spdk/ftl.h" +#include "ftl_nv_cache.h" #include "ftl_internal.h" #include "ftl_mngt_steps.h" #include "ftl_internal.h" @@ -171,20 +172,21 @@ void ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { struct spdk_bdev *bdev; + struct ftl_nv_cache *nv_cache = &dev->nv_cache; const char *bdev_name = dev->conf.cache_bdev; if (spdk_bdev_open_ext(bdev_name, true, nv_cache_bdev_event_cb, dev, - &dev->cache_bdev_desc)) { + &nv_cache->bdev_desc)) { FTL_ERRLOG(dev, "Unable to open bdev: %s\n", bdev_name); goto error; } - bdev = spdk_bdev_desc_get_bdev(dev->cache_bdev_desc); + bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - if (spdk_bdev_module_claim_bdev(bdev, dev->cache_bdev_desc, &g_ftl_bdev_module)) { + if (spdk_bdev_module_claim_bdev(bdev, nv_cache->bdev_desc, &g_ftl_bdev_module)) { /* clear the desc so that we don't try to release the claim on cleanup */ - spdk_bdev_close(dev->cache_bdev_desc); - dev->cache_bdev_desc = NULL; + spdk_bdev_close(nv_cache->bdev_desc); + nv_cache->bdev_desc = NULL; FTL_ERRLOG(dev, "Unable to claim bdev %s\n", bdev_name); goto error; } @@ -197,8 +199,8 @@ ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt goto error; } - dev->cache_ioch = spdk_bdev_get_io_channel(dev->cache_bdev_desc); - if (!dev->cache_ioch) { + nv_cache->cache_ioch = spdk_bdev_get_io_channel(nv_cache->bdev_desc); + if (!nv_cache->cache_ioch) { FTL_ERRLOG(dev, "Failed to create cache IO channel for NV Cache\n"); goto error; } @@ -210,8 +212,8 @@ ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt goto error; } - dev->cache_md_size = spdk_bdev_get_md_size(bdev); - if (dev->cache_md_size != sizeof(union ftl_md_vss)) { + nv_cache->md_size = spdk_bdev_get_md_size(bdev); + if (nv_cache->md_size != sizeof(union ftl_md_vss)) { FTL_ERRLOG(dev, "Bdev's %s metadata is invalid size (%"PRIu32")\n", spdk_bdev_get_name(bdev), spdk_bdev_get_md_size(bdev)); goto error; @@ -229,7 +231,7 @@ ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt goto error; } - if (ftl_md_xfer_blocks(dev) * dev->cache_md_size > FTL_ZERO_BUFFER_SIZE) { + if (ftl_md_xfer_blocks(dev) * nv_cache->md_size > FTL_ZERO_BUFFER_SIZE) { FTL_ERRLOG(dev, "Zero buffer too small for bdev %s metadata transfer\n", spdk_bdev_get_name(bdev)); goto error; @@ -240,7 +242,7 @@ ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt goto error; } - dev->cache_md_size = 64; + nv_cache->md_size = 64; FTL_NOTICELOG(dev, "FTL uses VSS emulation\n"); #endif @@ -253,18 +255,18 @@ error: void ftl_mngt_close_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { - if (dev->cache_ioch) { - spdk_put_io_channel(dev->cache_ioch); - dev->cache_ioch = NULL; + if (dev->nv_cache.cache_ioch) { + spdk_put_io_channel(dev->nv_cache.cache_ioch); + dev->nv_cache.cache_ioch = NULL; } - if (dev->cache_bdev_desc) { - struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->cache_bdev_desc); + if (dev->nv_cache.bdev_desc) { + struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); spdk_bdev_module_release_bdev(bdev); - spdk_bdev_close(dev->cache_bdev_desc); + spdk_bdev_close(dev->nv_cache.bdev_desc); - dev->cache_bdev_desc = NULL; + dev->nv_cache.bdev_desc = NULL; } ftl_mngt_next_step(mngt); diff --git a/lib/ftl/mngt/ftl_mngt_ioch.c b/lib/ftl/mngt/ftl_mngt_ioch.c index 013c4312e..aa58c9386 100644 --- a/lib/ftl/mngt/ftl_mngt_ioch.c +++ b/lib/ftl/mngt/ftl_mngt_ioch.c @@ -8,7 +8,6 @@ #include "ftl_core.h" #include "ftl_mngt.h" #include "ftl_mngt_steps.h" -#include "utils/ftl_mempool.h" struct ftl_io_channel_ctx { struct ftl_io_channel *ioch; diff --git a/lib/ftl/mngt/ftl_mngt_md.c b/lib/ftl/mngt/ftl_mngt_md.c index dfcf82531..7fbacb133 100644 --- a/lib/ftl/mngt/ftl_mngt_md.c +++ b/lib/ftl/mngt/ftl_mngt_md.c @@ -30,10 +30,12 @@ is_buffer_needed(enum ftl_layout_region_type type) #ifdef SPDK_FTL_VSS_EMU case FTL_LAYOUT_REGION_TYPE_VSS: #endif + case FTL_LAYOUT_REGION_TYPE_SB: case FTL_LAYOUT_REGION_TYPE_SB_BASE: case FTL_LAYOUT_REGION_TYPE_DATA_NVC: case FTL_LAYOUT_REGION_TYPE_DATA_BASE: + case FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR: return false; default: diff --git a/lib/ftl/mngt/ftl_mngt_misc.c b/lib/ftl/mngt/ftl_mngt_misc.c index 19df9d302..3e1abd039 100644 --- a/lib/ftl/mngt/ftl_mngt_misc.c +++ b/lib/ftl/mngt/ftl_mngt_misc.c @@ -8,6 +8,7 @@ #include "ftl_mngt.h" #include "ftl_mngt_steps.h" #include "ftl_internal.h" +#include "ftl_nv_cache.h" #include "ftl_debug.h" void @@ -20,6 +21,25 @@ ftl_mngt_check_conf(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) } } +void +ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + if (ftl_nv_cache_init(dev)) { + FTL_ERRLOG(dev, "Unable to initialize persistent cache\n"); + ftl_mngt_fail_step(mngt); + return; + } + + ftl_mngt_next_step(mngt); +} + +void +ftl_mngt_deinit_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + ftl_nv_cache_deinit(dev); + ftl_mngt_next_step(mngt); +} + static void user_clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) { @@ -59,6 +79,8 @@ ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mng { dev->initialized = 1; + ftl_nv_cache_resume(&dev->nv_cache); + ftl_mngt_next_step(mngt); } diff --git a/lib/ftl/mngt/ftl_mngt_startup.c b/lib/ftl/mngt/ftl_mngt_startup.c index caef1b4db..def2952ac 100644 --- a/lib/ftl/mngt/ftl_mngt_startup.c +++ b/lib/ftl/mngt/ftl_mngt_startup.c @@ -73,6 +73,11 @@ static const struct ftl_mngt_process_desc desc_startup = { .action = ftl_mngt_init_md, .cleanup = ftl_mngt_deinit_md }, + { + .name = "Initialize NV cache", + .action = ftl_mngt_init_nv_cache, + .cleanup = ftl_mngt_deinit_nv_cache + }, { .name = "Select startup mode", .action = ftl_mngt_select_startup_mode diff --git a/lib/ftl/mngt/ftl_mngt_steps.h b/lib/ftl/mngt/ftl_mngt_steps.h index 371da138f..1c98c97c7 100644 --- a/lib/ftl/mngt/ftl_mngt_steps.h +++ b/lib/ftl/mngt/ftl_mngt_steps.h @@ -36,6 +36,10 @@ void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + +void ftl_mngt_deinit_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_init_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); void ftl_mngt_deinit_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); diff --git a/lib/ftl/utils/ftl_md.c b/lib/ftl/utils/ftl_md.c index 66df7a51e..335c3f78c 100644 --- a/lib/ftl/utils/ftl_md.c +++ b/lib/ftl/utils/ftl_md.c @@ -260,7 +260,7 @@ read_blocks(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *desc, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - if (desc == dev->cache_bdev_desc) { + if (desc == dev->nv_cache.bdev_desc) { return ftl_nv_cache_bdev_read_blocks_with_md(dev, desc, ch, buf, md_buf, offset_blocks, num_blocks, cb, cb_arg); @@ -282,7 +282,7 @@ write_blocks(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *desc, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg) { - if (desc == dev->cache_bdev_desc) { + if (desc == dev->nv_cache.bdev_desc) { return ftl_nv_cache_bdev_write_blocks_with_md(dev, desc, ch, buf, md_buf, offset_blocks, num_blocks, cb, cb_arg); diff --git a/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c b/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c index f035581a2..e46b2d881 100644 --- a/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c +++ b/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c @@ -131,10 +131,10 @@ setup_device(uint32_t num_threads, uint32_t xfer_size) dev->conf = g_default_conf; dev->xfer_size = xfer_size; dev->base_bdev_desc = (struct spdk_bdev_desc *)0xdeadbeef; - dev->cache_bdev_desc = (struct spdk_bdev_desc *)0xdead1234; + dev->nv_cache.bdev_desc = (struct spdk_bdev_desc *)0xdead1234; spdk_io_device_register(dev, channel_create_cb, channel_destroy_cb, 0, NULL); spdk_io_device_register(dev->base_bdev_desc, channel_create_cb, channel_destroy_cb, 0, NULL); - spdk_io_device_register(dev->cache_bdev_desc, channel_create_cb, channel_destroy_cb, 0, NULL); + spdk_io_device_register(dev->nv_cache.bdev_desc, channel_create_cb, channel_destroy_cb, 0, NULL); TAILQ_INIT(&dev->ioch_queue); @@ -152,7 +152,7 @@ free_device(struct spdk_ftl_dev *dev) spdk_io_device_unregister(dev, NULL); spdk_io_device_unregister(dev->base_bdev_desc, NULL); - spdk_io_device_unregister(dev->cache_bdev_desc, NULL); + spdk_io_device_unregister(dev->nv_cache.bdev_desc, NULL); while (!TAILQ_EMPTY(&dev->ioch_queue)) { TAILQ_REMOVE(&dev->ioch_queue, TAILQ_FIRST(&dev->ioch_queue), entry);