diff --git a/lib/ftl/ftl_band.c b/lib/ftl/ftl_band.c index c7b265485..29ba61be1 100644 --- a/lib/ftl/ftl_band.c +++ b/lib/ftl/ftl_band.c @@ -66,6 +66,7 @@ _ftl_band_set_free(struct ftl_band *band) /* Add the band to the free band list */ TAILQ_INSERT_TAIL(&dev->free_bands, band, queue_entry); + band->reloc = false; dev->num_free++; ftl_apply_limits(dev); @@ -364,3 +365,223 @@ ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev) /* Map pool element holds the whole tail md */ return ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE; } + +static double +_band_invalidity(struct ftl_band *band) +{ + double valid = band->p2l_map.num_valid; + double count = ftl_band_user_blocks(band); + + return 1.0 - (valid / count); +} + +static void +dump_bands_under_relocation(struct spdk_ftl_dev *dev) +{ + uint64_t i = dev->sb_shm->gc_info.current_band_id; + uint64_t end = dev->sb_shm->gc_info.current_band_id + dev->num_logical_bands_in_physical; + + for (; i < end; i++) { + struct ftl_band *band = &dev->bands[i]; + + FTL_DEBUGLOG(dev, "Band, id %u, phys_is %u, wr cnt = %u, invalidity = %u%%\n", + band->id, band->phys_id, (uint32_t)band->md->wr_cnt, + (uint32_t)(_band_invalidity(band) * 100)); + } +} + +static bool +is_band_relocateable(struct ftl_band *band) +{ + /* Can only move data from closed bands */ + if (FTL_BAND_STATE_CLOSED != band->md->state) { + return false; + } + + /* Band is already under relocation, skip it */ + if (band->reloc) { + return false; + } + + return true; +} + +static void +get_band_phys_info(struct spdk_ftl_dev *dev, uint64_t phys_id, + double *invalidity, double *wr_cnt) +{ + struct ftl_band *band; + uint64_t band_id = phys_id * dev->num_logical_bands_in_physical; + + *wr_cnt = *invalidity = 0.0L; + for (; band_id < ftl_get_num_bands(dev); band_id++) { + band = &dev->bands[band_id]; + + if (phys_id != band->phys_id) { + break; + } + + *wr_cnt += band->md->wr_cnt; + + if (!is_band_relocateable(band)) { + continue; + } + + *invalidity += _band_invalidity(band); + } + + *invalidity /= dev->num_logical_bands_in_physical; + *wr_cnt /= dev->num_logical_bands_in_physical; +} + +static bool +band_cmp(double a_invalidity, double a_wr_cnt, + double b_invalidity, double b_wr_cnt, + uint64_t a_id, uint64_t b_id) +{ + assert(a_id != FTL_BAND_PHYS_ID_INVALID); + assert(b_id != FTL_BAND_PHYS_ID_INVALID); + double diff = a_invalidity - b_invalidity; + if (diff < 0.0L) { + diff *= -1.0L; + } + + /* Use the following metrics for picking bands for GC (in order): + * - relative invalidity + * - if invalidity is similar (within 10% points), then their write counts (how many times band was written to) + * - if write count is equal, then pick based on their placement on base device (lower LBAs win) + */ + if (diff > 0.1L) { + return a_invalidity > b_invalidity; + } + + if (a_wr_cnt != b_wr_cnt) { + return a_wr_cnt < b_wr_cnt; + } + + return a_id < b_id; +} + +static void +band_start_gc(struct spdk_ftl_dev *dev, struct ftl_band *band) +{ + ftl_bug(false == is_band_relocateable(band)); + + TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); + band->reloc = true; + + FTL_DEBUGLOG(dev, "Band to GC, id %u\n", band->id); +} + +static struct ftl_band * +gc_high_priority_band(struct spdk_ftl_dev *dev) +{ + struct ftl_band *band; + uint64_t high_prio_id = dev->sb_shm->gc_info.band_id_high_prio; + + if (FTL_BAND_ID_INVALID != high_prio_id) { + ftl_bug(high_prio_id >= dev->num_bands); + + band = &dev->bands[high_prio_id]; + dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; + + band_start_gc(dev, band); + FTL_NOTICELOG(dev, "GC takes high priority band, id %u\n", band->id); + return band; + } + + return 0; +} + +static void +ftl_band_reset_gc_iter(struct spdk_ftl_dev *dev) +{ + dev->sb->gc_info.is_valid = 0; + dev->sb->gc_info.current_band_id = FTL_BAND_ID_INVALID; + dev->sb->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; + dev->sb->gc_info.band_phys_id = FTL_BAND_PHYS_ID_INVALID; + + dev->sb_shm->gc_info = dev->sb->gc_info; +} + +struct ftl_band * +ftl_band_search_next_to_reloc(struct spdk_ftl_dev *dev) +{ + double invalidity, max_invalidity = 0.0L; + double wr_cnt, max_wr_cnt = 0.0L; + uint64_t phys_id = FTL_BAND_PHYS_ID_INVALID; + struct ftl_band *band; + uint64_t i, band_count; + uint64_t phys_count; + + band = gc_high_priority_band(dev); + if (spdk_unlikely(NULL != band)) { + return band; + } + + phys_count = dev->num_logical_bands_in_physical; + band_count = ftl_get_num_bands(dev); + + for (; dev->sb_shm->gc_info.current_band_id < band_count;) { + band = &dev->bands[dev->sb_shm->gc_info.current_band_id]; + if (band->phys_id != dev->sb_shm->gc_info.band_phys_id) { + break; + } + + if (false == is_band_relocateable(band)) { + dev->sb_shm->gc_info.current_band_id++; + continue; + } + + band_start_gc(dev, band); + return band; + } + + for (i = 0; i < band_count; i += phys_count) { + band = &dev->bands[i]; + + /* Calculate entire band physical group invalidity */ + get_band_phys_info(dev, band->phys_id, &invalidity, &wr_cnt); + + if (invalidity != 0.0L) { + if (phys_id == FTL_BAND_PHYS_ID_INVALID || + band_cmp(invalidity, wr_cnt, max_invalidity, max_wr_cnt, + band->phys_id, phys_id)) { + max_invalidity = invalidity; + max_wr_cnt = wr_cnt; + phys_id = band->phys_id; + } + } + } + + if (FTL_BAND_PHYS_ID_INVALID != phys_id) { + FTL_DEBUGLOG(dev, "Band physical id %"PRIu64" to GC\n", phys_id); + dev->sb_shm->gc_info.is_valid = 0; + dev->sb_shm->gc_info.current_band_id = phys_id * phys_count; + dev->sb_shm->gc_info.band_phys_id = phys_id; + dev->sb_shm->gc_info.is_valid = 1; + dump_bands_under_relocation(dev); + return ftl_band_search_next_to_reloc(dev); + } else { + ftl_band_reset_gc_iter(dev); + } + + return NULL; +} + +void +ftl_band_init_gc_iter(struct spdk_ftl_dev *dev) +{ + if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { + ftl_band_reset_gc_iter(dev); + return; + } + + if (dev->sb->clean) { + dev->sb_shm->gc_info = dev->sb->gc_info; + return; + } + + /* We lost GC state due to dirty shutdown, reset GC state to start over */ + ftl_band_reset_gc_iter(dev); +} diff --git a/lib/ftl/ftl_band.h b/lib/ftl/ftl_band.h index 7d8f0c43f..4bbfcc416 100644 --- a/lib/ftl/ftl_band.h +++ b/lib/ftl/ftl_band.h @@ -97,6 +97,9 @@ struct ftl_band { /* P2L map */ struct ftl_p2l_map p2l_map; + /* Band relocation is in progress */ + bool reloc; + /* Band's index */ uint32_t id; @@ -144,6 +147,8 @@ ftl_addr ftl_band_tail_md_addr(struct ftl_band *band); int ftl_band_filled(struct ftl_band *band, size_t offset); int ftl_band_write_prep(struct ftl_band *band); size_t ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev); +struct ftl_band *ftl_band_search_next_to_reloc(struct spdk_ftl_dev *dev); +void ftl_band_init_gc_iter(struct spdk_ftl_dev *dev); ftl_addr ftl_band_p2l_map_addr(struct ftl_band *band); void ftl_band_open(struct ftl_band *band, enum ftl_band_type type); void ftl_band_close(struct ftl_band *band); @@ -152,6 +157,7 @@ void ftl_band_rq_write(struct ftl_band *band, struct ftl_rq *rq); void ftl_band_rq_read(struct ftl_band *band, struct ftl_rq *rq); void ftl_band_basic_rq_write(struct ftl_band *band, struct ftl_basic_rq *brq); void ftl_band_basic_rq_read(struct ftl_band *band, struct ftl_basic_rq *brq); +void ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx); void ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx); static inline void diff --git a/lib/ftl/ftl_band_ops.c b/lib/ftl/ftl_band_ops.c index 0d594cf6a..fbcac8c84 100644 --- a/lib/ftl/ftl_band_ops.c +++ b/lib/ftl/ftl_band_ops.c @@ -340,7 +340,7 @@ ftl_band_close(struct ftl_band *band) void *metadata = band->p2l_map.band_map; uint64_t num_blocks = ftl_tail_md_num_blocks(dev); - /* Write LBA map first, after completion, set the state to close on nvcache, then internally */ + /* Write P2L map first, after completion, set the state to close on nvcache, then internally */ ftl_band_set_state(band, FTL_BAND_STATE_CLOSING); ftl_basic_rq_init(dev, &band->metadata_rq, metadata, num_blocks); ftl_basic_rq_set_owner(&band->metadata_rq, band_map_write_cb, band); @@ -382,6 +382,68 @@ ftl_band_free(struct ftl_band *band) /* TODO: The whole band erase code should probably be done here instead */ } +static void +read_md_cb(struct ftl_basic_rq *brq) +{ + struct ftl_band *band = brq->owner.priv; + struct spdk_ftl_dev *dev = band->dev; + ftl_band_ops_cb cb; + uint32_t band_map_crc; + bool success = true; + void *priv; + + cb = band->owner.ops_fn; + priv = band->owner.priv; + + if (!brq->success) { + ftl_band_basic_rq_read(band, &band->metadata_rq); + return; + } + + band_map_crc = spdk_crc32c_update(band->p2l_map.band_map, + ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE, 0); + if (band->md->p2l_map_checksum && band->md->p2l_map_checksum != band_map_crc) { + FTL_ERRLOG(dev, "GC error, inconsistent P2L map CRC\n"); + success = false; + } + band->owner.ops_fn = NULL; + band->owner.priv = NULL; + cb(band, priv, success); +} + +static int +_read_md(struct ftl_band *band) +{ + struct spdk_ftl_dev *dev = band->dev; + struct ftl_basic_rq *rq = &band->metadata_rq; + + if (ftl_band_alloc_p2l_map(band)) { + return -ENOMEM; + } + + /* Read P2L map */ + ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_p2l_map_num_blocks(dev)); + ftl_basic_rq_set_owner(rq, read_md_cb, band); + + rq->io.band = band; + rq->io.addr = ftl_band_p2l_map_addr(band); + + ftl_band_basic_rq_read(band, &band->metadata_rq); + + return 0; +} + +static void +read_md(void *band) +{ + int rc; + + rc = _read_md(band); + if (spdk_unlikely(rc)) { + spdk_thread_send_msg(spdk_get_thread(), read_md, band); + } +} + static void read_tail_md_cb(struct ftl_basic_rq *brq) { @@ -426,3 +488,24 @@ ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx) ftl_band_basic_rq_read(band, &band->metadata_rq); } + +void +ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx) +{ + struct ftl_band *band = ftl_band_search_next_to_reloc(dev); + + /* if disk is very small, GC start very early that no band is ready for it */ + if (spdk_unlikely(!band)) { + cb(NULL, cntx, false); + return; + } + + /* Only one owner is allowed */ + assert(!band->queue_depth); + assert(!band->owner.ops_fn); + assert(!band->owner.priv); + band->owner.ops_fn = cb; + band->owner.priv = cntx; + + read_md(band); +} diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 56d85c6c5..b9eaf1455 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -150,6 +150,8 @@ struct spdk_ftl_dev { /* Writer for GC IOs */ struct ftl_writer writer_gc; + uint32_t num_logical_bands_in_physical; + /* Retry init sequence */ bool init_retry; }; diff --git a/lib/ftl/mngt/ftl_mngt_band.c b/lib/ftl/mngt/ftl_mngt_band.c index 4af9ee913..4757032ce 100644 --- a/lib/ftl/mngt/ftl_mngt_band.c +++ b/lib/ftl/mngt/ftl_mngt_band.c @@ -146,6 +146,8 @@ decorate_bands(struct spdk_ftl_dev *dev) TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); i++; } + + dev->num_logical_bands_in_physical = num_logical_in_phys; } void @@ -155,6 +157,65 @@ ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) ftl_mngt_next_step(mngt); } +static struct ftl_band * +next_high_prio_band(struct spdk_ftl_dev *dev) +{ + struct ftl_band *result = NULL, *band; + uint64_t validity = UINT64_MAX; + + TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { + if (band->p2l_map.num_valid < validity) { + result = band; + validity = result->p2l_map.num_valid; + } + } + + return result; +} + +static int +finalize_init_gc(struct spdk_ftl_dev *dev) +{ + struct ftl_band *band; + uint64_t free_blocks, blocks_to_move; + + ftl_band_init_gc_iter(dev); + dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; + + if (0 == dev->num_free) { + /* Get number of available blocks in writer */ + free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc); + + /* + * First, check a band candidate to GC + */ + band = ftl_band_search_next_to_reloc(dev); + ftl_bug(NULL == band); + blocks_to_move = band->p2l_map.num_valid; + if (blocks_to_move <= free_blocks) { + /* This GC band can be moved */ + return 0; + } + + /* + * The GC candidate cannot be moved because no enough space. We need to find + * another band. + */ + band = next_high_prio_band(dev); + ftl_bug(NULL == band); + + if (band->p2l_map.num_valid > free_blocks) { + FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n"); + return -1; + } else { + /* GC needs to start using this band */ + dev->sb_shm->gc_info.band_id_high_prio = band->id; + } + } + + return 0; +} + void ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) { @@ -233,5 +294,9 @@ ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process * return; } - ftl_mngt_next_step(mngt); + if (finalize_init_gc(dev)) { + ftl_mngt_fail_step(mngt); + } else { + ftl_mngt_next_step(mngt); + } } diff --git a/lib/ftl/mngt/ftl_mngt_md.c b/lib/ftl/mngt/ftl_mngt_md.c index 5b50ef8d5..7c50fe8e0 100644 --- a/lib/ftl/mngt/ftl_mngt_md.c +++ b/lib/ftl/mngt/ftl_mngt_md.c @@ -177,6 +177,8 @@ ftl_mngt_init_default_sb(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt sb->overprovisioning = dev->conf.overprovisioning; + ftl_band_init_gc_iter(dev); + /* md layout isn't initialized yet. * empty region list => all regions in the default location */ sb->md_layout_head.type = FTL_LAYOUT_REGION_TYPE_INVALID;