diff --git a/lib/ftl/Makefile b/lib/ftl/Makefile index 2ade690dd..e0972204e 100644 --- a/lib/ftl/Makefile +++ b/lib/ftl/Makefile @@ -25,6 +25,7 @@ C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c ftl_sb.c ftl_l2 C_SRCS += ftl_nv_cache.c ftl_band.c C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c mngt/ftl_mngt_l2p.c +C_SRCS += mngt/ftl_mngt_band.c C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ftl.map) diff --git a/lib/ftl/ftl_band.c b/lib/ftl/ftl_band.c index e6b9e25c9..c7b265485 100644 --- a/lib/ftl/ftl_band.c +++ b/lib/ftl/ftl_band.c @@ -206,7 +206,7 @@ ftl_band_user_blocks(const struct ftl_band *band) struct ftl_band * ftl_band_from_addr(struct spdk_ftl_dev *dev, ftl_addr addr) { - size_t band_id = ftl_addr_get_band(dev, addr); + uint64_t band_id = ftl_addr_get_band(dev, addr); assert(band_id < ftl_get_num_bands(dev)); return &dev->bands[band_id]; diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index c5b4db087..dee909ede 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -45,6 +45,8 @@ ftl_get_limit(const struct spdk_ftl_dev *dev, int type) static bool ftl_shutdown_complete(struct spdk_ftl_dev *dev) { + uint64_t i; + if (dev->num_inflight) { return false; } @@ -58,6 +60,13 @@ ftl_shutdown_complete(struct spdk_ftl_dev *dev) return false; } + for (i = 0; i < ftl_get_num_bands(dev); ++i) { + if (dev->bands[i].queue_depth || + dev->bands[i].md->state == FTL_BAND_STATE_CLOSING) { + return false; + } + } + if (!ftl_l2p_is_halted(dev)) { ftl_l2p_halt(dev); return false; diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index a3b0a39c5..4b9a71800 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -181,7 +181,7 @@ ftl_get_core_thread(const struct spdk_ftl_dev *dev) return dev->core_thread; } -static inline size_t +static inline uint64_t ftl_get_num_bands(const struct spdk_ftl_dev *dev) { return dev->num_bands; diff --git a/lib/ftl/mngt/ftl_mngt_band.c b/lib/ftl/mngt/ftl_mngt_band.c new file mode 100644 index 000000000..6eaae625e --- /dev/null +++ b/lib/ftl/mngt/ftl_mngt_band.c @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "ftl_core.h" +#include "ftl_mngt_steps.h" +#include "ftl_band.h" +#include "ftl_internal.h" + +static int +ftl_band_init_md(struct ftl_band *band) +{ + struct spdk_ftl_dev *dev = band->dev; + struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; + struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md); + + band->md = &band_md[band->id]; + + return 0; +} + +static int +ftl_dev_init_bands(struct spdk_ftl_dev *dev) +{ + struct ftl_band *band; + uint64_t i; + + TAILQ_INIT(&dev->free_bands); + TAILQ_INIT(&dev->shut_bands); + + dev->num_free = 0; + dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); + if (!dev->bands) { + return -ENOMEM; + } + + for (i = 0; i < ftl_get_num_bands(dev); ++i) { + band = &dev->bands[i]; + band->id = i; + band->dev = dev; + + /* Adding to shut_bands is necessary - see ftl_restore_band_close_cb() */ + TAILQ_INSERT_TAIL(&dev->shut_bands, band, queue_entry); + } + + return 0; +} + +static int +ftl_dev_init_bands_md(struct spdk_ftl_dev *dev) +{ + uint64_t i; + int rc = 0; + + for (i = 0; i < ftl_get_num_bands(dev); ++i) { + rc = ftl_band_init_md(&dev->bands[i]); + if (rc) { + FTL_ERRLOG(dev, "Failed to initialize metadata structures for band [%lu]\n", i); + break; + } + } + + return rc; +} + +static void +ftl_dev_deinit_bands(struct spdk_ftl_dev *dev) +{ + free(dev->bands); +} + +void +ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + if (ftl_dev_init_bands(dev)) { + ftl_mngt_fail_step(mngt); + } else { + ftl_mngt_next_step(mngt); + } +} + +void +ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + if (ftl_dev_init_bands_md(dev)) { + ftl_mngt_fail_step(mngt); + } else { + ftl_mngt_next_step(mngt); + } +} + +void +ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + ftl_dev_deinit_bands(dev); + ftl_mngt_next_step(mngt); +} + +/* + * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's + * perspective. Improves WAF. + */ +#define BASE_BDEV_RECLAIM_UNIT_SIZE (72 * GiB) + +static void +decorate_bands(struct spdk_ftl_dev *dev) +{ + struct ftl_band *band; + uint64_t i, num_to_drop, phys_id = 0; + uint64_t num_blocks, num_bands; + uint64_t num_blocks_in_band = ftl_get_num_blocks_in_band(dev); + uint64_t reclaim_unit_num_blocks = BASE_BDEV_RECLAIM_UNIT_SIZE / FTL_BLOCK_SIZE; + uint32_t num_logical_in_phys = 2; + + assert(reclaim_unit_num_blocks % num_blocks_in_band == 0); + + num_blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); + + /* For base bdev bigger than 1TB take reclaim uint size for grouping GC bands */ + if (num_blocks > (TiB / FTL_BLOCK_SIZE)) { + assert(reclaim_unit_num_blocks < num_blocks); + num_logical_in_phys = reclaim_unit_num_blocks / num_blocks_in_band; + } + + num_to_drop = ftl_get_num_bands(dev) % num_logical_in_phys; + + i = 0; + while (i < ftl_get_num_bands(dev) - num_to_drop) { + band = &dev->bands[i]; + band->start_addr = i * dev->num_blocks_in_band; + band->tail_md_addr = ftl_band_tail_md_addr(band); + + band->phys_id = phys_id; + i++; + if (i % num_logical_in_phys == 0) { + phys_id++; + } + } + + /* Mark not aligned logical bands as broken */ + num_bands = ftl_get_num_bands(dev); + while (i < num_bands) { + band = &dev->bands[i]; + dev->num_bands--; + TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); + i++; + } +} + +void +ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + decorate_bands(dev); + ftl_mngt_next_step(mngt); +} + +void +ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + struct ftl_band *band, *temp_band; + uint64_t num_open = 0, num_shut = 0; + + TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) { + if (band->md->state == FTL_BAND_STATE_OPEN || + band->md->state == FTL_BAND_STATE_FULL) { + TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); + num_open++; + assert(num_open <= FTL_MAX_OPEN_BANDS); + continue; + } + + if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { + TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); + assert(band->md->state == FTL_BAND_STATE_FREE); + band->md->state = FTL_BAND_STATE_CLOSED; + ftl_band_set_state(band, FTL_BAND_STATE_FREE); + } else { + num_shut++; + } + } + + /* Recalculate number of free bands */ + dev->num_free = 0; + TAILQ_FOREACH(band, &dev->free_bands, queue_entry) { + assert(band->md->state == FTL_BAND_STATE_FREE); + dev->num_free++; + } + ftl_apply_limits(dev); + + if ((num_shut + num_open + dev->num_free) != ftl_get_num_bands(dev)) { + FTL_ERRLOG(dev, "ERROR, band list inconsistent state\n"); + ftl_mngt_fail_step(mngt); + return; + } + + ftl_mngt_next_step(mngt); +} diff --git a/lib/ftl/mngt/ftl_mngt_md.c b/lib/ftl/mngt/ftl_mngt_md.c index 2cb5c70b9..ffbd836b0 100644 --- a/lib/ftl/mngt/ftl_mngt_md.c +++ b/lib/ftl/mngt/ftl_mngt_md.c @@ -132,6 +132,13 @@ ftl_mngt_persist_nv_cache_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_pro persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_NVC_MD); } +void +ftl_mngt_persist_band_info_metadata( + struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + persist(dev, mngt, FTL_LAYOUT_REGION_TYPE_BAND_MD); +} + static uint32_t get_sb_crc(struct ftl_superblock *sb) { diff --git a/lib/ftl/mngt/ftl_mngt_startup.c b/lib/ftl/mngt/ftl_mngt_startup.c index 602e35e5b..0a604bc02 100644 --- a/lib/ftl/mngt/ftl_mngt_startup.c +++ b/lib/ftl/mngt/ftl_mngt_startup.c @@ -59,6 +59,11 @@ static const struct ftl_mngt_process_desc desc_startup = { .action = ftl_mngt_init_mem_pools, .cleanup = ftl_mngt_deinit_mem_pools }, + { + .name = "Initialize bands", + .action = ftl_mngt_init_bands, + .cleanup = ftl_mngt_deinit_bands + }, { .name = "Register IO device", .action = ftl_mngt_register_io_device, @@ -69,6 +74,10 @@ static const struct ftl_mngt_process_desc desc_startup = { .action = ftl_mngt_init_io_channel, .cleanup = ftl_mngt_deinit_io_channel }, + { + .name = "Decorate bands", + .action = ftl_mngt_decorate_bands + }, { .name = "Initialize layout", .action = ftl_mngt_init_layout @@ -83,6 +92,10 @@ static const struct ftl_mngt_process_desc desc_startup = { .action = ftl_mngt_init_nv_cache, .cleanup = ftl_mngt_deinit_nv_cache }, + { + .name = "Initialize bands metadata", + .action = ftl_mngt_init_bands_md + }, { .name = "Select startup mode", .action = ftl_mngt_select_startup_mode @@ -113,6 +126,14 @@ static const struct ftl_mngt_process_desc desc_first_start = { .name = "Scrub NV cache", .action = ftl_mngt_scrub_nv_cache, }, + { + .name = "Finalize band initialization", + .action = ftl_mngt_finalize_init_bands, + }, + { + .name = "Save initial band info metadata", + .action = ftl_mngt_persist_band_info_metadata, + }, { .name = "Save initial chunk info metadata", .action = ftl_mngt_persist_nv_cache_metadata, diff --git a/lib/ftl/mngt/ftl_mngt_steps.h b/lib/ftl/mngt/ftl_mngt_steps.h index c89d544c2..d8549c1e1 100644 --- a/lib/ftl/mngt/ftl_mngt_steps.h +++ b/lib/ftl/mngt/ftl_mngt_steps.h @@ -36,10 +36,18 @@ void ftl_mngt_init_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process * void ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + +void ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + +void ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); void ftl_mngt_deinit_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); @@ -52,6 +60,8 @@ void ftl_mngt_clear_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) void ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); void ftl_mngt_start_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); @@ -72,6 +82,8 @@ void ftl_mngt_init_default_sb(struct spdk_ftl_dev *dev, struct ftl_mngt_process void ftl_mngt_set_dirty(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_persist_band_info_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_persist_nv_cache_metadata(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); #endif /* FTL_MNGT_STEPS_H */