diff --git a/autotest.sh b/autotest.sh index 6a29a155d..0b97175dd 100755 --- a/autotest.sh +++ b/autotest.sh @@ -305,10 +305,6 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then run_test "ocf" ./test/ocf/ocf.sh fi - if [ $SPDK_TEST_FTL -eq 1 ]; then - run_test "ftl" ./test/ftl/ftl.sh - fi - if [ $SPDK_TEST_VMD -eq 1 ]; then run_test "vmd" ./test/vmd/vmd.sh fi diff --git a/include/spdk/ftl.h b/include/spdk/ftl.h deleted file mode 100644 index 3fe87d5e3..000000000 --- a/include/spdk/ftl.h +++ /dev/null @@ -1,223 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef SPDK_FTL_H -#define SPDK_FTL_H - -#include "spdk/stdinc.h" -#include "spdk/uuid.h" -#include "spdk/thread.h" -#include "spdk/bdev.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct spdk_ftl_dev; - -/* Limit thresholds */ -enum { - SPDK_FTL_LIMIT_CRIT, - SPDK_FTL_LIMIT_HIGH, - SPDK_FTL_LIMIT_LOW, - SPDK_FTL_LIMIT_START, - SPDK_FTL_LIMIT_MAX -}; - -struct spdk_ftl_limit { - /* Threshold from which the limiting starts */ - size_t thld; - - /* Limit percentage */ - size_t limit; -}; - -struct spdk_ftl_conf { - /* Number of reserved addresses not exposed to the user */ - size_t lba_rsvd; - - /* Size of the per-io_channel write buffer */ - size_t write_buffer_size; - - /* Threshold for opening new band */ - size_t band_thld; - - /* Maximum IO depth per band relocate */ - size_t max_reloc_qdepth; - - /* Maximum active band relocates */ - size_t max_active_relocs; - - /* IO pool size per user thread */ - size_t user_io_pool_size; - - /* Lowest percentage of invalid blocks for a band to be defragged */ - size_t invalid_thld; - - /* User writes limits */ - struct spdk_ftl_limit limits[SPDK_FTL_LIMIT_MAX]; - - /* Allow for partial recovery from open bands instead of returning error */ - bool allow_open_bands; - - /* Use append instead of write */ - bool use_append; - - /* Maximum supported number of IO channels */ - uint32_t max_io_channels; - - struct { - /* Maximum number of concurrent requests */ - size_t max_request_cnt; - /* Maximum number of blocks per one request */ - size_t max_request_size; - } nv_cache; - - /* Create l2p table on l2p_path persistent memory file or device instead of in DRAM */ - const char *l2p_path; -}; - -enum spdk_ftl_mode { - /* Create new device */ - SPDK_FTL_MODE_CREATE = (1 << 0), -}; - -struct spdk_ftl_dev_init_opts { - /* Underlying device */ - const char *base_bdev; - /* Write buffer cache */ - const char *cache_bdev; - - /* Thread responsible for core tasks execution */ - struct spdk_thread *core_thread; - - /* Device's config */ - const struct spdk_ftl_conf *conf; - /* Device's name */ - const char *name; - /* Mode flags */ - unsigned int mode; - /* Device UUID (valid when restoring device from disk) */ - struct spdk_uuid uuid; -}; - -struct spdk_ftl_attrs { - /* Device's UUID */ - struct spdk_uuid uuid; - /* Number of logical blocks */ - uint64_t num_blocks; - /* Logical block size */ - size_t block_size; - /* Underlying device */ - const char *base_bdev; - /* Write buffer cache */ - const char *cache_bdev; - /* Number of zones per parallel unit in the underlying device (including any offline ones) */ - size_t num_zones; - /* Number of logical blocks per zone */ - size_t zone_size; - /* Device specific configuration */ - struct spdk_ftl_conf conf; -}; - -typedef void (*spdk_ftl_fn)(void *, int); -typedef void (*spdk_ftl_init_fn)(struct spdk_ftl_dev *, void *, int); - -/** - * Initialize the FTL on given NVMe device and parallel unit range. - * - * Covers the following: - * - retrieve zone device information, - * - allocate buffers and resources, - * - initialize internal structures, - * - initialize internal thread(s), - * - restore or create L2P table. - * - * \param opts configuration for new device - * \param cb callback function to call when the device is created - * \param cb_arg callback's argument - * - * \return 0 if initialization was started successfully, negative errno otherwise. - */ -int spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *opts, spdk_ftl_init_fn cb, void *cb_arg); - -/** - * Deinitialize and free given device. - * - * \param dev device - * \param cb callback function to call when the device is freed - * \param cb_arg callback's argument - * - * \return 0 if successfully scheduled free, negative errno otherwise. - */ -int spdk_ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_init_fn cb, void *cb_arg); - -/** - * Initialize FTL configuration structure with default values. - * - * \param conf FTL configuration to initialize - */ -void spdk_ftl_conf_init_defaults(struct spdk_ftl_conf *conf); - -/** - * Retrieve device’s attributes. - * - * \param dev device - * \param attr Attribute structure to fill - */ -void spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attr); - -/** - * Submits a read to the specified device. - * - * \param dev Device - * \param ch I/O channel - * \param lba Starting LBA to read the data - * \param lba_cnt Number of sectors to read - * \param iov Single IO vector or pointer to IO vector table - * \param iov_cnt Number of IO vectors - * \param cb_fn Callback function to invoke when the I/O is completed - * \param cb_arg Argument to pass to the callback function - * - * \return 0 if successfully submitted, negative errno otherwise. - */ -int spdk_ftl_read(struct spdk_ftl_dev *dev, struct spdk_io_channel *ch, uint64_t lba, - size_t lba_cnt, - struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg); - -/** - * Submits a write to the specified device. - * - * \param dev Device - * \param ch I/O channel - * \param lba Starting LBA to write the data - * \param lba_cnt Number of sectors to write - * \param iov Single IO vector or pointer to IO vector table - * \param iov_cnt Number of IO vectors - * \param cb_fn Callback function to invoke when the I/O is completed - * \param cb_arg Argument to pass to the callback function - * - * \return 0 if successfully submitted, negative errno otherwise. - */ -int spdk_ftl_write(struct spdk_ftl_dev *dev, struct spdk_io_channel *ch, uint64_t lba, - size_t lba_cnt, - struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg); - -/** - * Submits a flush request to the specified device. - * - * \param dev device - * \param cb_fn Callback function to invoke when all prior IOs have been completed - * \param cb_arg Argument to pass to the callback function - * - * \return 0 if successfully submitted, negative errno otherwise. - */ -int spdk_ftl_flush(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); - -#ifdef __cplusplus -} -#endif - -#endif /* SPDK_FTL_H */ diff --git a/lib/Makefile b/lib/Makefile index 3d0054cc4..4e59921d5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -11,7 +11,7 @@ DIRS-y += bdev blob blobfs conf dma accel event json jsonrpc \ log lvol rpc sock thread trace util nvme vmd nvmf scsi \ ioat ut_mock iscsi notify init trace_parser ifeq ($(OS),Linux) -DIRS-y += nbd ftl +DIRS-y += nbd endif DIRS-$(CONFIG_OCF) += env_ocf diff --git a/lib/ftl/ftl_addr.h b/lib/ftl/ftl_addr.h deleted file mode 100644 index d7fe91828..000000000 --- a/lib/ftl/ftl_addr.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef FTL_ADDR_H -#define FTL_ADDR_H - -#include "spdk/stdinc.h" - -/* Marks address as invalid */ -#define FTL_ADDR_INVALID (-1) -/* Marks LBA as invalid */ -#define FTL_LBA_INVALID ((uint64_t)-1) -/* Smallest data unit size */ -#define FTL_BLOCK_SIZE 4096 - -/* This structure represents on-disk address. It can have one of the following */ -/* formats: */ -/* - offset inside the disk */ -/* - cache_offset inside the cache (indicated by the cached flag) */ -/* - packed version of the two formats above (can be only used when the */ -/* offset can be represented in less than 32 bits) */ -/* Packed format is used, when possible, to avoid wasting RAM on the L2P table. */ -struct ftl_addr { - union { - struct { - uint64_t cache_offset : 63; - uint64_t cached : 1; - }; - - struct { - union { - struct { - uint32_t cache_offset : 31; - uint32_t cached : 1; - }; - - uint32_t offset; - }; - uint32_t rsvd; - } pack; - - uint64_t offset; - }; -}; - -#endif /* FTL_ADDR_H */ diff --git a/lib/ftl/ftl_band.c b/lib/ftl/ftl_band.c deleted file mode 100644 index f9a0c7c7d..000000000 --- a/lib/ftl/ftl_band.c +++ /dev/null @@ -1,1069 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/crc32.h" -#include "spdk/likely.h" -#include "spdk/util.h" -#include "spdk/ftl.h" - -#include "ftl_band.h" -#include "ftl_io.h" -#include "ftl_core.h" -#include "ftl_reloc.h" -#include "ftl_debug.h" - -/* TODO: define some signature for meta version */ -#define FTL_MD_VER 1 - -struct __attribute__((packed)) ftl_md_hdr { - /* Device instance */ - struct spdk_uuid uuid; - - /* Meta version */ - uint8_t ver; - - /* Sequence number */ - uint64_t seq; - - /* CRC32 checksum */ - uint32_t checksum; -}; - -/* End metadata layout stored on media (with all three being aligned to block size): */ -/* - header */ -/* - valid bitmap */ -/* - LBA map */ -struct __attribute__((packed)) ftl_tail_md { - struct ftl_md_hdr hdr; - - /* Max number of blocks */ - uint64_t num_blocks; - - uint8_t reserved[4059]; -}; -SPDK_STATIC_ASSERT(sizeof(struct ftl_tail_md) == FTL_BLOCK_SIZE, "Incorrect metadata size"); - -struct __attribute__((packed)) ftl_head_md { - struct ftl_md_hdr hdr; - - /* Number of defrag cycles */ - uint64_t wr_cnt; - - /* Number of surfaced LBAs */ - uint64_t lba_cnt; - - /* Transfer size */ - uint32_t xfer_size; -}; - -size_t -ftl_tail_md_hdr_num_blocks(void) -{ - return spdk_divide_round_up(sizeof(struct ftl_tail_md), FTL_BLOCK_SIZE); -} - -size_t -ftl_vld_map_num_blocks(const struct spdk_ftl_dev *dev) -{ - return spdk_divide_round_up(ftl_vld_map_size(dev), FTL_BLOCK_SIZE); -} - -size_t -ftl_lba_map_num_blocks(const struct spdk_ftl_dev *dev) -{ - return spdk_divide_round_up(ftl_get_num_blocks_in_band(dev) * sizeof(uint64_t), FTL_BLOCK_SIZE); -} - -size_t -ftl_head_md_num_blocks(const struct spdk_ftl_dev *dev) -{ - return dev->xfer_size; -} - -size_t -ftl_tail_md_num_blocks(const struct spdk_ftl_dev *dev) -{ - return spdk_divide_round_up(ftl_tail_md_hdr_num_blocks() + - ftl_vld_map_num_blocks(dev) + - ftl_lba_map_num_blocks(dev), - dev->xfer_size) * dev->xfer_size; -} - -static uint64_t -ftl_band_tail_md_offset(const struct ftl_band *band) -{ - return ftl_band_num_usable_blocks(band) - - ftl_tail_md_num_blocks(band->dev); -} - -int -ftl_band_full(struct ftl_band *band, size_t offset) -{ - return offset == ftl_band_tail_md_offset(band); -} - -void -ftl_band_write_failed(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - band->high_prio = 1; - - ftl_reloc_add(dev->reloc, band, 0, ftl_get_num_blocks_in_band(dev), 1, true); - ftl_band_set_state(band, FTL_BAND_STATE_CLOSED); -} - -static void -ftl_band_free_lba_map(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_lba_map *lba_map = &band->lba_map; - - assert(band->state == FTL_BAND_STATE_CLOSED || - band->state == FTL_BAND_STATE_FREE); - assert(lba_map->ref_cnt == 0); - assert(lba_map->map != NULL); - assert(!band->high_prio); - - /* Verify that band's metadata is consistent with l2p */ - if (band->num_zones) { - assert(ftl_band_validate_md(band) == true); - } - - spdk_mempool_put(dev->lba_pool, lba_map->dma_buf); - lba_map->map = NULL; - lba_map->dma_buf = NULL; -} - -static void -_ftl_band_set_free(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_band *lband, *prev; - - /* Remove the band from the closed band list */ - LIST_REMOVE(band, list_entry); - - /* Keep the list sorted by band's write count */ - LIST_FOREACH(lband, &dev->free_bands, list_entry) { - if (lband->wr_cnt > band->wr_cnt) { - LIST_INSERT_BEFORE(lband, band, list_entry); - break; - } - prev = lband; - } - - if (!lband) { - if (LIST_EMPTY(&dev->free_bands)) { - LIST_INSERT_HEAD(&dev->free_bands, band, list_entry); - } else { - LIST_INSERT_AFTER(prev, band, list_entry); - } - } - -#if defined(DEBUG) - prev = NULL; - LIST_FOREACH(lband, &dev->free_bands, list_entry) { - if (!prev) { - continue; - } - assert(prev->wr_cnt <= lband->wr_cnt); - } -#endif - dev->num_free++; - ftl_apply_limits(dev); -} - -static void -_ftl_band_set_preparing(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - /* Remove band from free list */ - LIST_REMOVE(band, list_entry); - - band->wr_cnt++; - - assert(dev->num_free > 0); - dev->num_free--; - - ftl_apply_limits(dev); -} - -static void -_ftl_band_set_closed(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - /* Set the state as free_md() checks for that */ - band->state = FTL_BAND_STATE_CLOSED; - - /* Free the lba map if there are no outstanding IOs */ - ftl_band_release_lba_map(band); - - if (spdk_likely(band->num_zones)) { - LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); - } else { - LIST_REMOVE(band, list_entry); - } -} - -static uint32_t -ftl_md_calc_crc(const struct ftl_md_hdr *hdr, size_t size) -{ - size_t checkoff = offsetof(struct ftl_md_hdr, checksum); - size_t mdoff = checkoff + sizeof(hdr->checksum); - uint32_t crc; - - crc = spdk_crc32c_update(hdr, checkoff, 0); - return spdk_crc32c_update((const char *)hdr + mdoff, size - mdoff, crc); -} - -static void -ftl_set_md_hdr(struct ftl_band *band, struct ftl_md_hdr *hdr, size_t size) -{ - hdr->seq = band->seq; - hdr->ver = FTL_MD_VER; - hdr->uuid = band->dev->uuid; - hdr->checksum = ftl_md_calc_crc(hdr, size); -} - -static int -ftl_pack_head_md(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_head_md *head = band->lba_map.dma_buf; - - head->wr_cnt = band->wr_cnt; - head->lba_cnt = dev->num_lbas; - head->xfer_size = dev->xfer_size; - ftl_set_md_hdr(band, &head->hdr, sizeof(struct ftl_head_md)); - - return FTL_MD_SUCCESS; -} - -static int -ftl_pack_tail_md(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_lba_map *lba_map = &band->lba_map; - struct ftl_tail_md *tail = lba_map->dma_buf; - void *vld_offset; - - vld_offset = (char *)tail + ftl_tail_md_hdr_num_blocks() * FTL_BLOCK_SIZE; - - /* Clear out the buffer */ - memset(tail, 0, ftl_tail_md_hdr_num_blocks() * FTL_BLOCK_SIZE); - tail->num_blocks = ftl_get_num_blocks_in_band(dev); - - pthread_spin_lock(&lba_map->lock); - spdk_bit_array_store_mask(lba_map->vld, vld_offset); - pthread_spin_unlock(&lba_map->lock); - - ftl_set_md_hdr(band, &tail->hdr, ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE); - - return FTL_MD_SUCCESS; -} - -static int -ftl_md_hdr_vld(struct spdk_ftl_dev *dev, const struct ftl_md_hdr *hdr, size_t size) -{ - if (spdk_uuid_compare(&dev->uuid, &hdr->uuid) != 0) { - return FTL_MD_NO_MD; - } - - if (hdr->ver != FTL_MD_VER) { - return FTL_MD_INVALID_VER; - } - - if (ftl_md_calc_crc(hdr, size) != hdr->checksum) { - return FTL_MD_INVALID_CRC; - } - - return FTL_MD_SUCCESS; -} - -static int -ftl_unpack_tail_md(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - void *vld_offset; - struct ftl_lba_map *lba_map = &band->lba_map; - struct ftl_tail_md *tail = lba_map->dma_buf; - int rc; - - vld_offset = (char *)tail + ftl_tail_md_hdr_num_blocks() * FTL_BLOCK_SIZE; - - rc = ftl_md_hdr_vld(dev, &tail->hdr, ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE); - if (rc) { - return rc; - } - - /* - * When restoring from a dirty shutdown it's possible old tail meta wasn't yet cleared - - * band had saved head meta, but didn't manage to send erase to all zones. - * The already found tail md header is valid, but inconsistent with the head meta. Treat - * such a band as open/without valid tail md. - */ - if (band->seq != tail->hdr.seq) { - return FTL_MD_NO_MD; - } - - if (tail->num_blocks != ftl_get_num_blocks_in_band(dev)) { - return FTL_MD_INVALID_SIZE; - } - - spdk_bit_array_load_mask(lba_map->vld, vld_offset); - - return FTL_MD_SUCCESS; -} - -static int -ftl_unpack_head_md(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_head_md *head = band->lba_map.dma_buf; - int rc; - - rc = ftl_md_hdr_vld(dev, &head->hdr, sizeof(struct ftl_head_md)); - if (rc) { - return rc; - } - - band->seq = head->hdr.seq; - band->wr_cnt = head->wr_cnt; - - if (dev->global_md.num_lbas == 0) { - dev->global_md.num_lbas = head->lba_cnt; - } - - if (dev->global_md.num_lbas != head->lba_cnt) { - return FTL_MD_INVALID_SIZE; - } - - if (dev->xfer_size != head->xfer_size) { - return FTL_MD_INVALID_SIZE; - } - - return FTL_MD_SUCCESS; -} - -struct ftl_addr -ftl_band_tail_md_addr(struct ftl_band *band) -{ - struct ftl_addr addr = {}; - struct ftl_zone *zone; - struct spdk_ftl_dev *dev = band->dev; - size_t xfer_size = dev->xfer_size; - size_t num_req = ftl_band_tail_md_offset(band) / xfer_size; - size_t i; - - if (spdk_unlikely(!band->num_zones)) { - return ftl_to_addr(FTL_ADDR_INVALID); - } - - /* Metadata should be aligned to xfer size */ - assert(ftl_band_tail_md_offset(band) % xfer_size == 0); - - zone = CIRCLEQ_FIRST(&band->zones); - for (i = 0; i < num_req % band->num_zones; ++i) { - zone = ftl_band_next_zone(band, zone); - } - - addr.offset = (num_req / band->num_zones) * xfer_size; - addr.offset += zone->info.zone_id; - - return addr; -} - -struct ftl_addr -ftl_band_head_md_addr(struct ftl_band *band) -{ - if (spdk_unlikely(!band->num_zones)) { - return ftl_to_addr(FTL_ADDR_INVALID); - } - - return ftl_to_addr(CIRCLEQ_FIRST(&band->zones)->info.zone_id); -} - -void -ftl_band_set_state(struct ftl_band *band, enum ftl_band_state state) -{ - switch (state) { - case FTL_BAND_STATE_FREE: - assert(band->state == FTL_BAND_STATE_CLOSED); - _ftl_band_set_free(band); - break; - - case FTL_BAND_STATE_PREP: - assert(band->state == FTL_BAND_STATE_FREE); - _ftl_band_set_preparing(band); - break; - - case FTL_BAND_STATE_CLOSED: - if (band->state != FTL_BAND_STATE_CLOSED) { - assert(band->state == FTL_BAND_STATE_CLOSING || band->high_prio); - _ftl_band_set_closed(band); - } - break; - - default: - break; - } - - band->state = state; -} - -void -ftl_band_set_addr(struct ftl_band *band, uint64_t lba, struct ftl_addr addr) -{ - struct ftl_lba_map *lba_map = &band->lba_map; - uint64_t offset; - - assert(lba != FTL_LBA_INVALID); - - offset = ftl_band_block_offset_from_addr(band, addr); - pthread_spin_lock(&lba_map->lock); - - lba_map->num_vld++; - lba_map->map[offset] = lba; - spdk_bit_array_set(lba_map->vld, offset); - - pthread_spin_unlock(&lba_map->lock); -} - -size_t -ftl_band_age(const struct ftl_band *band) -{ - return (size_t)(band->dev->seq - band->seq); -} - -size_t -ftl_band_num_usable_blocks(const struct ftl_band *band) -{ - return band->num_zones * ftl_get_num_blocks_in_zone(band->dev); -} - -size_t -ftl_band_user_blocks_left(const struct ftl_band *band, size_t offset) -{ - size_t tail_md_offset = ftl_band_tail_md_offset(band); - - if (spdk_unlikely(offset <= ftl_head_md_num_blocks(band->dev))) { - return ftl_band_user_blocks(band); - } - - if (spdk_unlikely(offset > tail_md_offset)) { - return 0; - } - - return tail_md_offset - offset; -} - -size_t -ftl_band_user_blocks(const struct ftl_band *band) -{ - return ftl_band_num_usable_blocks(band) - - ftl_head_md_num_blocks(band->dev) - - ftl_tail_md_num_blocks(band->dev); -} - -struct ftl_band * -ftl_band_from_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - size_t band_id = ftl_addr_get_band(dev, addr); - - assert(band_id < ftl_get_num_bands(dev)); - return &dev->bands[band_id]; -} - -struct ftl_zone * -ftl_band_zone_from_addr(struct ftl_band *band, struct ftl_addr addr) -{ - size_t pu_id = ftl_addr_get_punit(band->dev, addr); - - assert(pu_id < ftl_get_num_punits(band->dev)); - return &band->zone_buf[pu_id]; -} - -uint64_t -ftl_band_block_offset_from_addr(struct ftl_band *band, struct ftl_addr addr) -{ - assert(ftl_addr_get_band(band->dev, addr) == band->id); - assert(ftl_addr_get_punit(band->dev, addr) < ftl_get_num_punits(band->dev)); - return addr.offset % ftl_get_num_blocks_in_band(band->dev); -} - -struct ftl_addr -ftl_band_next_xfer_addr(struct ftl_band *band, struct ftl_addr addr, size_t num_blocks) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_zone *zone; - size_t num_xfers, num_stripes; - uint64_t offset; - - assert(ftl_addr_get_band(dev, addr) == band->id); - - offset = ftl_addr_get_zone_offset(dev, addr); - zone = ftl_band_zone_from_addr(band, addr); - - num_blocks += (offset % dev->xfer_size); - offset -= (offset % dev->xfer_size); - -#if defined(DEBUG) - /* Check that the number of zones has not been changed */ - struct ftl_zone *_zone; - size_t _num_zones = 0; - CIRCLEQ_FOREACH(_zone, &band->zones, circleq) { - if (spdk_likely(_zone->info.state != SPDK_BDEV_ZONE_STATE_OFFLINE)) { - _num_zones++; - } - } - assert(band->num_zones == _num_zones); -#endif - assert(band->num_zones != 0); - num_stripes = (num_blocks / dev->xfer_size) / band->num_zones; - offset += num_stripes * dev->xfer_size; - num_blocks -= num_stripes * dev->xfer_size * band->num_zones; - - if (offset > ftl_get_num_blocks_in_zone(dev)) { - return ftl_to_addr(FTL_ADDR_INVALID); - } - - num_xfers = num_blocks / dev->xfer_size; - for (size_t i = 0; i < num_xfers; ++i) { - /* When the last zone is reached the block part of the address */ - /* needs to be increased by xfer_size */ - if (ftl_band_zone_is_last(band, zone)) { - offset += dev->xfer_size; - if (offset > ftl_get_num_blocks_in_zone(dev)) { - return ftl_to_addr(FTL_ADDR_INVALID); - } - } - - zone = ftl_band_next_operational_zone(band, zone); - assert(zone); - - num_blocks -= dev->xfer_size; - } - - if (num_blocks) { - offset += num_blocks; - if (offset > ftl_get_num_blocks_in_zone(dev)) { - return ftl_to_addr(FTL_ADDR_INVALID); - } - } - - addr.offset = zone->info.zone_id + offset; - return addr; -} - -static size_t -ftl_xfer_offset_from_addr(struct ftl_band *band, struct ftl_addr addr) -{ - struct ftl_zone *zone, *current_zone; - unsigned int punit_offset = 0; - size_t num_stripes, xfer_size = band->dev->xfer_size; - uint64_t offset; - - assert(ftl_addr_get_band(band->dev, addr) == band->id); - - offset = ftl_addr_get_zone_offset(band->dev, addr); - num_stripes = (offset / xfer_size) * band->num_zones; - - current_zone = ftl_band_zone_from_addr(band, addr); - CIRCLEQ_FOREACH(zone, &band->zones, circleq) { - if (current_zone == zone) { - break; - } - punit_offset++; - } - - return xfer_size * (num_stripes + punit_offset) + offset % xfer_size; -} - -struct ftl_addr -ftl_band_addr_from_block_offset(struct ftl_band *band, uint64_t block_off) -{ - struct ftl_addr addr = { .offset = 0 }; - - addr.offset = block_off + band->id * ftl_get_num_blocks_in_band(band->dev); - return addr; -} - -struct ftl_addr -ftl_band_next_addr(struct ftl_band *band, struct ftl_addr addr, size_t offset) -{ - uint64_t block_off = ftl_band_block_offset_from_addr(band, addr); - return ftl_band_addr_from_block_offset(band, block_off + offset); -} - -void -ftl_band_acquire_lba_map(struct ftl_band *band) -{ - assert(band->lba_map.map != NULL); - band->lba_map.ref_cnt++; -} - -int -ftl_band_alloc_lba_map(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_lba_map *lba_map = &band->lba_map; - - assert(lba_map->ref_cnt == 0); - assert(lba_map->map == NULL); - - lba_map->dma_buf = spdk_mempool_get(dev->lba_pool); - - if (!lba_map->dma_buf) { - return -1; - } - - memset(lba_map->dma_buf, 0, ftl_lba_map_pool_elem_size(band->dev)); - - lba_map->map = (uint64_t *)((char *)lba_map->dma_buf + FTL_BLOCK_SIZE * - (ftl_tail_md_hdr_num_blocks() + ftl_vld_map_num_blocks(dev))); - - lba_map->segments = (char *)lba_map->dma_buf + ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE; - - ftl_band_acquire_lba_map(band); - return 0; -} - -void -ftl_band_release_lba_map(struct ftl_band *band) -{ - struct ftl_lba_map *lba_map = &band->lba_map; - - assert(lba_map->map != NULL); - assert(lba_map->ref_cnt > 0); - lba_map->ref_cnt--; - - if (lba_map->ref_cnt == 0) { - ftl_band_free_lba_map(band); - } -} - -static void -ftl_read_md_cb(struct ftl_io *io, void *arg, int status) -{ - struct ftl_md_io *md_io = (struct ftl_md_io *)io; - - if (!status) { - status = md_io->pack_fn(md_io->io.band); - } else { - status = FTL_MD_IO_FAILURE; - } - - md_io->cb_fn(io, md_io->cb_ctx, status); -} - -static struct ftl_md_io * -ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_addr addr, - struct ftl_band *band, size_t num_blocks, void *buf, - ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) -{ - struct ftl_md_io *io; - struct ftl_io_init_opts opts = { - .dev = dev, - .io = NULL, - .band = band, - .size = sizeof(*io), - .flags = FTL_IO_MD | FTL_IO_PHYSICAL_MODE, - .type = FTL_IO_READ, - .num_blocks = num_blocks, - .cb_fn = fn, - .iovs = { - { - .iov_base = buf, - .iov_len = num_blocks * FTL_BLOCK_SIZE, - } - }, - .iovcnt = 1, - }; - - io = (struct ftl_md_io *)ftl_io_init_internal(&opts); - if (!io) { - return NULL; - } - - io->io.addr = addr; - io->pack_fn = pack_fn; - io->cb_fn = cb_fn; - io->cb_ctx = cb_ctx; - - return io; -} - -static struct ftl_io * -ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, - void *data, size_t num_blocks, ftl_io_fn cb) -{ - struct ftl_io_init_opts opts = { - .dev = dev, - .io = NULL, - .band = band, - .size = sizeof(struct ftl_io), - .flags = FTL_IO_MD | FTL_IO_PHYSICAL_MODE, - .type = FTL_IO_WRITE, - .num_blocks = num_blocks, - .cb_fn = cb, - .iovs = { - { - .iov_base = data, - .iov_len = num_blocks * FTL_BLOCK_SIZE, - } - }, - .iovcnt = 1, - .md = NULL, - }; - - return ftl_io_init_internal(&opts); -} - -static int -ftl_band_write_md(struct ftl_band *band, size_t num_blocks, - ftl_md_pack_fn md_fn, ftl_io_fn cb) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_io *io; - - io = ftl_io_init_md_write(dev, band, band->lba_map.dma_buf, num_blocks, cb); - if (!io) { - return -ENOMEM; - } - - md_fn(band); - - ftl_io_write(io); - return 0; -} - -void -ftl_band_md_clear(struct ftl_band *band) -{ - band->seq = 0; - band->wr_cnt = 0; - band->lba_map.num_vld = 0; - band->lba_map.map = NULL; -} - -int -ftl_band_write_head_md(struct ftl_band *band, ftl_io_fn cb) -{ - return ftl_band_write_md(band, ftl_head_md_num_blocks(band->dev), - ftl_pack_head_md, cb); -} - -int -ftl_band_write_tail_md(struct ftl_band *band, ftl_io_fn cb) -{ - return ftl_band_write_md(band, ftl_tail_md_num_blocks(band->dev), - ftl_pack_tail_md, cb); -} - -static struct ftl_addr -ftl_band_lba_map_addr(struct ftl_band *band, size_t offset) -{ - return ftl_band_next_xfer_addr(band, band->tail_md_addr, - ftl_tail_md_hdr_num_blocks() + - ftl_vld_map_num_blocks(band->dev) + - offset); -} - -static int -ftl_band_read_md(struct ftl_band *band, size_t num_blocks, struct ftl_addr start_addr, - void *buf, ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_md_io *io; - - if (spdk_unlikely(!band->num_zones)) { - return -ENOENT; - } - - io = ftl_io_init_md_read(dev, start_addr, band, num_blocks, buf, fn, pack_fn, cb_fn, cb_ctx); - if (!io) { - return -ENOMEM; - } - - ftl_io_read((struct ftl_io *)io); - return 0; -} - -int -ftl_band_read_tail_md(struct ftl_band *band, struct ftl_addr addr, ftl_io_fn cb_fn, void *cb_ctx) -{ - return ftl_band_read_md(band, ftl_tail_md_num_blocks(band->dev), addr, band->lba_map.dma_buf, - ftl_read_md_cb, ftl_unpack_tail_md, cb_fn, cb_ctx); -} - -static size_t -ftl_lba_map_request_segment_done(struct ftl_lba_map_request *request, size_t offset, - size_t num_segments) -{ - size_t i, num_done = 0; - - for (i = offset; i < offset + num_segments; ++i) { - if (spdk_bit_array_get(request->segments, i)) { - spdk_bit_array_clear(request->segments, offset); - num_done++; - } - } - - assert(request->num_pending >= num_done); - request->num_pending -= num_done; - - return num_done; -} - -static void -ftl_lba_map_set_segment_state(struct ftl_lba_map *lba_map, size_t offset, size_t num_segments, - enum ftl_lba_map_seg_state state) -{ - size_t i; - - for (i = offset; i < offset + num_segments; ++i) { - lba_map->segments[i] = state; - } -} - -static void -ftl_lba_map_request_free(struct spdk_ftl_dev *dev, struct ftl_lba_map_request *request) -{ - spdk_bit_array_clear_mask(request->segments); - spdk_mempool_put(dev->lba_request_pool, request); -} - -static void -ftl_process_lba_map_requests(struct spdk_ftl_dev *dev, struct ftl_lba_map *lba_map, size_t offset, - size_t num_segments, int status) -{ - struct ftl_lba_map_request *request, *trequest; - size_t num_done; - - LIST_FOREACH_SAFE(request, &lba_map->request_list, list_entry, trequest) { - num_done = ftl_lba_map_request_segment_done(request, offset, num_segments); - if (request->num_pending == 0 || (status && num_done)) { - request->cb(NULL, request->cb_ctx, status); - LIST_REMOVE(request, list_entry); - ftl_lba_map_request_free(dev, request); - } - } -} - -static size_t -ftl_lba_map_offset_from_addr(struct ftl_band *band, struct ftl_addr addr) -{ - size_t offset; - struct ftl_addr start_addr = ftl_band_lba_map_addr(band, 0); - - offset = ftl_xfer_offset_from_addr(band, addr) - ftl_xfer_offset_from_addr(band, start_addr); - assert(offset < ftl_lba_map_num_blocks(band->dev)); - - return offset; -} - -static void -ftl_read_lba_map_cb(struct ftl_io *io, void *arg, int status) -{ - struct ftl_lba_map *lba_map = &io->band->lba_map; - uint64_t block_off; - - block_off = ftl_lba_map_offset_from_addr(io->band, io->addr); - assert(block_off + io->num_blocks <= ftl_lba_map_num_blocks(io->dev)); - - if (!status) { - ftl_lba_map_set_segment_state(lba_map, block_off, io->num_blocks, - FTL_LBA_MAP_SEG_CACHED); - } - - ftl_process_lba_map_requests(io->dev, lba_map, block_off, io->num_blocks, status); -} - -static struct ftl_lba_map_request * -ftl_lba_map_alloc_request(struct ftl_band *band, size_t offset, size_t num_segments, - ftl_io_fn cb, void *cb_ctx) -{ - struct ftl_lba_map_request *request; - struct spdk_ftl_dev *dev = band->dev; - size_t i; - - request = spdk_mempool_get(dev->lba_request_pool); - if (!request) { - return NULL; - } - - request->cb = cb; - request->cb_ctx = cb_ctx; - request->num_pending = num_segments; - - for (i = offset; i < offset + num_segments; ++i) { - spdk_bit_array_set(request->segments, i); - } - - return request; -} - -static size_t -ftl_lba_map_num_clear_segments(struct ftl_lba_map *lba_map, - size_t offset, size_t num_segments) -{ - size_t i, cnt = 0; - - for (i = offset; i < offset + num_segments; ++i) { - if (lba_map->segments[i] != FTL_LBA_MAP_SEG_CLEAR) { - break; - } - cnt++; - } - - return cnt; -} - -int -ftl_band_read_lba_map(struct ftl_band *band, size_t offset, size_t lba_cnt, - ftl_io_fn cb_fn, void *cb_ctx) -{ - size_t num_blocks, block_off, num_read, num_segments; - struct ftl_lba_map *lba_map = &band->lba_map; - struct ftl_lba_map_request *request; - int rc = 0; - - block_off = offset / FTL_NUM_LBA_IN_BLOCK; - num_segments = spdk_divide_round_up(offset + lba_cnt, FTL_NUM_LBA_IN_BLOCK); - num_blocks = num_segments - block_off; - assert(block_off + num_blocks <= ftl_lba_map_num_blocks(band->dev)); - - request = ftl_lba_map_alloc_request(band, block_off, num_blocks, cb_fn, cb_ctx); - if (!request) { - return -ENOMEM; - } - - while (num_blocks) { - if (lba_map->segments[block_off] != FTL_LBA_MAP_SEG_CLEAR) { - if (lba_map->segments[block_off] == FTL_LBA_MAP_SEG_CACHED) { - ftl_lba_map_request_segment_done(request, block_off, 1); - } - num_blocks--; - block_off++; - continue; - } - - num_read = ftl_lba_map_num_clear_segments(lba_map, block_off, num_blocks); - ftl_lba_map_set_segment_state(lba_map, block_off, num_read, - FTL_LBA_MAP_SEG_PENDING); - - rc = ftl_band_read_md(band, num_read, ftl_band_lba_map_addr(band, block_off), - (char *)band->lba_map.map + block_off * FTL_BLOCK_SIZE, - ftl_read_lba_map_cb, NULL, cb_fn, cb_ctx); - if (rc) { - ftl_lba_map_request_free(band->dev, request); - return rc; - } - - assert(num_blocks >= num_read); - num_blocks -= num_read; - block_off += num_read; - } - - if (request->num_pending) { - LIST_INSERT_HEAD(&lba_map->request_list, request, list_entry); - } else { - cb_fn(NULL, cb_ctx, 0); - ftl_lba_map_request_free(band->dev, request); - } - - return rc; -} - -int -ftl_band_read_head_md(struct ftl_band *band, ftl_io_fn cb_fn, void *cb_ctx) -{ - return ftl_band_read_md(band, - ftl_head_md_num_blocks(band->dev), - ftl_band_head_md_addr(band), - band->lba_map.dma_buf, - ftl_read_md_cb, - ftl_unpack_head_md, - cb_fn, - cb_ctx); -} - -void -ftl_band_remove_zone(struct ftl_band *band, struct ftl_zone *zone) -{ - CIRCLEQ_REMOVE(&band->zones, zone, circleq); - band->num_zones--; -} - -int -ftl_band_write_prep(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - if (ftl_band_alloc_lba_map(band)) { - return -1; - } - - band->seq = ++dev->seq; - return 0; -} - -struct ftl_zone * -ftl_band_next_operational_zone(struct ftl_band *band, struct ftl_zone *zone) -{ - struct ftl_zone *result = NULL; - struct ftl_zone *entry; - - if (spdk_unlikely(!band->num_zones)) { - return NULL; - } - - /* Erasing band may fail after it was assigned to wptr. */ - /* In such a case zone is no longer in band->zones queue. */ - if (spdk_likely(zone->info.state != SPDK_BDEV_ZONE_STATE_OFFLINE)) { - result = ftl_band_next_zone(band, zone); - } else { - CIRCLEQ_FOREACH_REVERSE(entry, &band->zones, circleq) { - if (entry->info.zone_id > zone->info.zone_id) { - result = entry; - } else { - if (!result) { - result = CIRCLEQ_FIRST(&band->zones); - } - break; - } - } - } - - return result; -} - -void -ftl_band_clear_lba_map(struct ftl_band *band) -{ - struct ftl_lba_map *lba_map = &band->lba_map; - size_t num_segments; - - spdk_bit_array_clear_mask(lba_map->vld); - memset(lba_map->map, 0, ftl_lba_map_num_blocks(band->dev) * FTL_BLOCK_SIZE); - - /* For open band all lba map segments are already cached */ - assert(band->state == FTL_BAND_STATE_PREP); - num_segments = spdk_divide_round_up(ftl_get_num_blocks_in_band(band->dev), FTL_NUM_LBA_IN_BLOCK); - ftl_lba_map_set_segment_state(&band->lba_map, 0, num_segments, FTL_LBA_MAP_SEG_CACHED); - - lba_map->num_vld = 0; -} - -size_t -ftl_lba_map_pool_elem_size(struct spdk_ftl_dev *dev) -{ - /* Map pool element holds the whole tail md + segments map */ - return ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE + - spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), FTL_NUM_LBA_IN_BLOCK); -} diff --git a/lib/ftl/ftl_band.h b/lib/ftl/ftl_band.h deleted file mode 100644 index 34bf973a0..000000000 --- a/lib/ftl/ftl_band.h +++ /dev/null @@ -1,259 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef FTL_BAND_H -#define FTL_BAND_H - -#include "spdk/stdinc.h" -#include "spdk/bit_array.h" -#include "spdk/queue.h" -#include "spdk/bdev_zone.h" - -#include "ftl_io.h" -#include "ftl_addr.h" -#include "ftl_core.h" - -/* Number of LBAs that could be stored in a single block */ -#define FTL_NUM_LBA_IN_BLOCK (FTL_BLOCK_SIZE / sizeof(uint64_t)) - -struct spdk_ftl_dev; -struct ftl_lba_map_request; - -struct ftl_zone { - struct spdk_bdev_zone_info info; - - /* Indicates that there is inflight write */ - bool busy; - - CIRCLEQ_ENTRY(ftl_zone) circleq; -}; - -enum ftl_md_status { - FTL_MD_SUCCESS, - /* Metadata read failure */ - FTL_MD_IO_FAILURE, - /* Invalid version */ - FTL_MD_INVALID_VER, - /* UUID doesn't match */ - FTL_MD_NO_MD, - /* UUID and version matches but CRC doesn't */ - FTL_MD_INVALID_CRC, - /* Vld or lba map size doesn't match */ - FTL_MD_INVALID_SIZE -}; - -enum ftl_lba_map_seg_state { - FTL_LBA_MAP_SEG_CLEAR, - FTL_LBA_MAP_SEG_PENDING, - FTL_LBA_MAP_SEG_CACHED -}; - -struct ftl_lba_map { - /* LBA/vld map lock */ - pthread_spinlock_t lock; - - /* Number of valid LBAs */ - size_t num_vld; - - /* LBA map's reference count */ - size_t ref_cnt; - - /* Bitmap of valid LBAs */ - struct spdk_bit_array *vld; - - /* LBA map (only valid for open/relocating bands) */ - uint64_t *map; - - /* LBA map segment state map (clear, pending, cached) */ - uint8_t *segments; - - LIST_HEAD(, ftl_lba_map_request) request_list; - - /* Metadata DMA buffer (only valid for open/relocating bands) */ - void *dma_buf; -}; - -enum ftl_band_state { - FTL_BAND_STATE_FREE, - FTL_BAND_STATE_PREP, - FTL_BAND_STATE_OPENING, - FTL_BAND_STATE_OPEN, - FTL_BAND_STATE_FULL, - FTL_BAND_STATE_CLOSING, - FTL_BAND_STATE_CLOSED, - FTL_BAND_STATE_MAX -}; - -struct ftl_lba_map_request { - /* Completion callback */ - ftl_io_fn cb; - - /* Completion callback context */ - void *cb_ctx; - - /* Bit array of requested segments */ - struct spdk_bit_array *segments; - - /* Number of pending segments to read */ - size_t num_pending; - - LIST_ENTRY(ftl_lba_map_request) list_entry; -}; - -struct ftl_band { - /* Device this band belongs to */ - struct spdk_ftl_dev *dev; - - /* Number of operational zones */ - size_t num_zones; - - /* Array of zones */ - struct ftl_zone *zone_buf; - - /* List of operational zones */ - CIRCLEQ_HEAD(, ftl_zone) zones; - - /* LBA map */ - struct ftl_lba_map lba_map; - - /* Band's state */ - enum ftl_band_state state; - - /* Band's index */ - unsigned int id; - - /* Latest merit calculation */ - double merit; - - /* High defrag priority - means that the metadata should be copied and */ - /* the band should be defragged immediately */ - int high_prio; - - /* Sequence number */ - uint64_t seq; - - /* Number of defrag cycles */ - uint64_t wr_cnt; - - /* End metadata start addr */ - struct ftl_addr tail_md_addr; - - /* Bitmap of all bands that have its data moved onto this band */ - struct spdk_bit_array *reloc_bitmap; - /* Number of open bands containing data moved from this band */ - size_t num_reloc_bands; - /* Number of blocks currently being moved from this band */ - size_t num_reloc_blocks; - - /* Free/shut bands' lists */ - LIST_ENTRY(ftl_band) list_entry; - - /* High priority queue link */ - STAILQ_ENTRY(ftl_band) prio_stailq; -}; - -uint64_t ftl_band_block_offset_from_addr(struct ftl_band *band, struct ftl_addr addr); -struct ftl_addr ftl_band_addr_from_block_offset(struct ftl_band *band, uint64_t block_off); -void ftl_band_set_state(struct ftl_band *band, enum ftl_band_state state); -size_t ftl_band_age(const struct ftl_band *band); -void ftl_band_acquire_lba_map(struct ftl_band *band); -int ftl_band_alloc_lba_map(struct ftl_band *band); -void ftl_band_clear_lba_map(struct ftl_band *band); -void ftl_band_release_lba_map(struct ftl_band *band); -int ftl_band_read_lba_map(struct ftl_band *band, - size_t offset, size_t lba_cnt, - ftl_io_fn cb_fn, void *cb_ctx); -struct ftl_addr ftl_band_next_xfer_addr(struct ftl_band *band, struct ftl_addr addr, - size_t num_blocks); -struct ftl_addr ftl_band_next_addr(struct ftl_band *band, struct ftl_addr addr, - size_t offset); -size_t ftl_band_num_usable_blocks(const struct ftl_band *band); -size_t ftl_band_user_blocks_left(const struct ftl_band *band, size_t offset); -size_t ftl_band_user_blocks(const struct ftl_band *band); -void ftl_band_set_addr(struct ftl_band *band, uint64_t lba, - struct ftl_addr addr); -struct ftl_band *ftl_band_from_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr); -struct ftl_zone *ftl_band_zone_from_addr(struct ftl_band *band, struct ftl_addr); -void ftl_band_md_clear(struct ftl_band *band); -int ftl_band_read_tail_md(struct ftl_band *band, struct ftl_addr, - ftl_io_fn cb_fn, void *cb_ctx); -int ftl_band_read_head_md(struct ftl_band *band, ftl_io_fn cb_fn, void *cb_ctx); -int ftl_band_write_tail_md(struct ftl_band *band, ftl_io_fn cb); -int ftl_band_write_head_md(struct ftl_band *band, ftl_io_fn cb); -struct ftl_addr ftl_band_tail_md_addr(struct ftl_band *band); -struct ftl_addr ftl_band_head_md_addr(struct ftl_band *band); -void ftl_band_write_failed(struct ftl_band *band); -int ftl_band_full(struct ftl_band *band, size_t offset); -int ftl_band_write_prep(struct ftl_band *band); -struct ftl_zone *ftl_band_next_operational_zone(struct ftl_band *band, - struct ftl_zone *zone); -size_t ftl_lba_map_pool_elem_size(struct spdk_ftl_dev *dev); -void ftl_band_remove_zone(struct ftl_band *band, struct ftl_zone *zone); - - -static inline int -ftl_band_empty(const struct ftl_band *band) -{ - return band->lba_map.num_vld == 0; -} - -static inline struct ftl_zone * -ftl_band_next_zone(struct ftl_band *band, struct ftl_zone *zone) -{ - assert(zone->info.state != SPDK_BDEV_ZONE_STATE_OFFLINE); - return CIRCLEQ_LOOP_NEXT(&band->zones, zone, circleq); -} - -static inline void -ftl_band_set_next_state(struct ftl_band *band) -{ - ftl_band_set_state(band, (band->state + 1) % FTL_BAND_STATE_MAX); -} - -static inline int -ftl_band_state_changing(struct ftl_band *band) -{ - return band->state == FTL_BAND_STATE_OPENING || - band->state == FTL_BAND_STATE_CLOSING; -} - -static inline int -ftl_band_block_offset_valid(struct ftl_band *band, size_t block_off) -{ - struct ftl_lba_map *lba_map = &band->lba_map; - - pthread_spin_lock(&lba_map->lock); - if (spdk_bit_array_get(lba_map->vld, block_off)) { - pthread_spin_unlock(&lba_map->lock); - return 1; - } - - pthread_spin_unlock(&lba_map->lock); - return 0; -} - -static inline int -ftl_band_zone_is_last(struct ftl_band *band, struct ftl_zone *zone) -{ - return zone == CIRCLEQ_LAST(&band->zones); -} - -static inline int -ftl_band_zone_is_first(struct ftl_band *band, struct ftl_zone *zone) -{ - return zone == CIRCLEQ_FIRST(&band->zones); -} - -static inline int -ftl_zone_is_writable(const struct spdk_ftl_dev *dev, const struct ftl_zone *zone) -{ - bool busy = ftl_is_append_supported(dev) ? false : zone->busy; - - return (zone->info.state == SPDK_BDEV_ZONE_STATE_OPEN || - zone->info.state == SPDK_BDEV_ZONE_STATE_EMPTY) && - !busy; -} - -#endif /* FTL_BAND_H */ diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c deleted file mode 100644 index 50c6f7562..000000000 --- a/lib/ftl/ftl_core.c +++ /dev/null @@ -1,2442 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/likely.h" -#include "spdk/stdinc.h" -#include "spdk/nvme.h" -#include "spdk/thread.h" -#include "spdk/bdev_module.h" -#include "spdk/string.h" -#include "spdk/log.h" -#include "spdk/ftl.h" -#include "spdk/crc32.h" - -#include "ftl_core.h" -#include "ftl_band.h" -#include "ftl_io.h" -#include "ftl_debug.h" -#include "ftl_reloc.h" - -struct ftl_band_flush { - struct spdk_ftl_dev *dev; - /* Number of bands left to be flushed */ - size_t num_bands; - /* User callback */ - spdk_ftl_fn cb_fn; - /* Callback's argument */ - void *cb_arg; - /* List link */ - LIST_ENTRY(ftl_band_flush) list_entry; -}; - -struct ftl_wptr { - /* Owner device */ - struct spdk_ftl_dev *dev; - - /* Current address */ - struct ftl_addr addr; - - /* Band currently being written to */ - struct ftl_band *band; - - /* Current logical block's offset */ - uint64_t offset; - - /* Current zone */ - struct ftl_zone *zone; - - /* Pending IO queue */ - TAILQ_HEAD(, ftl_io) pending_queue; - - /* List link */ - LIST_ENTRY(ftl_wptr) list_entry; - - /* - * If setup in direct mode, there will be no offset or band state update after IO. - * The zoned bdev address is not assigned by wptr, and is instead taken directly - * from the request. - */ - bool direct_mode; - - /* Number of outstanding write requests */ - uint32_t num_outstanding; - - /* Marks that the band related to this wptr needs to be closed as soon as possible */ - bool flush; -}; - -struct ftl_flush { - /* Owner device */ - struct spdk_ftl_dev *dev; - - /* Number of batches to wait for */ - size_t num_req; - - /* Callback */ - struct { - spdk_ftl_fn fn; - void *ctx; - } cb; - - /* Batch bitmap */ - struct spdk_bit_array *bmap; - - /* List link */ - LIST_ENTRY(ftl_flush) list_entry; -}; - -static void -ftl_wptr_free(struct ftl_wptr *wptr) -{ - if (!wptr) { - return; - } - - free(wptr); -} - -static void -ftl_remove_wptr(struct ftl_wptr *wptr) -{ - struct spdk_ftl_dev *dev = wptr->dev; - struct ftl_band_flush *flush, *tmp; - - if (spdk_unlikely(wptr->flush)) { - LIST_FOREACH_SAFE(flush, &dev->band_flush_list, list_entry, tmp) { - assert(flush->num_bands > 0); - if (--flush->num_bands == 0) { - flush->cb_fn(flush->cb_arg, 0); - LIST_REMOVE(flush, list_entry); - free(flush); - } - } - } - - LIST_REMOVE(wptr, list_entry); - ftl_wptr_free(wptr); -} - -static struct ftl_wbuf_entry * -ftl_acquire_wbuf_entry(struct ftl_io_channel *io_channel, int io_flags) -{ - struct ftl_wbuf_entry *entry = NULL; - uint32_t qdepth; - - if (!(io_flags & FTL_IO_INTERNAL)) { - qdepth = __atomic_fetch_add(&io_channel->qdepth_current, 1, __ATOMIC_SEQ_CST); - if (qdepth >= io_channel->qdepth_limit) { - __atomic_fetch_sub(&io_channel->qdepth_current, 1, __ATOMIC_SEQ_CST); - return NULL; - } - } - - if (spdk_ring_dequeue(io_channel->free_queue, (void **)&entry, 1) != 1) { - if (!(io_flags & FTL_IO_INTERNAL)) { - __atomic_fetch_sub(&io_channel->qdepth_current, 1, __ATOMIC_SEQ_CST); - } - - return NULL; - } - - assert(entry != NULL); - - ftl_evict_cache_entry(io_channel->dev, entry); - - entry->io_flags = io_flags; - entry->addr.offset = FTL_ADDR_INVALID; - entry->lba = FTL_LBA_INVALID; - entry->band = NULL; - entry->valid = false; - - return entry; -} - -static void -ftl_release_wbuf_entry(struct ftl_wbuf_entry *entry) -{ - struct ftl_io_channel *io_channel = entry->ioch; - - if (!(entry->io_flags & FTL_IO_INTERNAL)) { - __atomic_fetch_sub(&io_channel->qdepth_current, 1, __ATOMIC_SEQ_CST); - } - - spdk_ring_enqueue(io_channel->free_queue, (void **)&entry, 1, NULL); -} - -static struct ftl_batch * -ftl_get_next_batch(struct spdk_ftl_dev *dev) -{ - struct ftl_batch *batch = dev->current_batch; - struct ftl_io_channel *ioch; -#define FTL_DEQUEUE_ENTRIES 128 - struct ftl_wbuf_entry *entries[FTL_DEQUEUE_ENTRIES]; - TAILQ_HEAD(, ftl_io_channel) ioch_queue; - size_t i, num_dequeued, num_remaining; - uint64_t *metadata; - - if (batch == NULL) { - batch = TAILQ_FIRST(&dev->pending_batches); - if (batch != NULL) { - TAILQ_REMOVE(&dev->pending_batches, batch, tailq); - return batch; - } - - batch = TAILQ_FIRST(&dev->free_batches); - if (spdk_unlikely(batch == NULL)) { - return NULL; - } - - assert(TAILQ_EMPTY(&batch->entries)); - assert(batch->num_entries == 0); - TAILQ_REMOVE(&dev->free_batches, batch, tailq); - } - - /* - * Keep shifting the queue to ensure fairness in IO channel selection. Each time - * ftl_get_next_batch() is called, we're starting to dequeue write buffer entries from a - * different IO channel. - */ - TAILQ_INIT(&ioch_queue); - while (!TAILQ_EMPTY(&dev->ioch_queue)) { - ioch = TAILQ_FIRST(&dev->ioch_queue); - TAILQ_REMOVE(&dev->ioch_queue, ioch, tailq); - TAILQ_INSERT_TAIL(&ioch_queue, ioch, tailq); - - num_remaining = dev->xfer_size - batch->num_entries; - while (num_remaining > 0) { - num_dequeued = spdk_ring_dequeue(ioch->submit_queue, (void **)entries, - spdk_min(num_remaining, - FTL_DEQUEUE_ENTRIES)); - if (num_dequeued == 0) { - break; - } - - for (i = 0; i < num_dequeued; ++i) { - batch->iov[batch->num_entries + i].iov_base = entries[i]->payload; - batch->iov[batch->num_entries + i].iov_len = FTL_BLOCK_SIZE; - - if (batch->metadata != NULL) { - metadata = (uint64_t *)((char *)batch->metadata + - i * dev->md_size); - *metadata = entries[i]->lba; - } - - TAILQ_INSERT_TAIL(&batch->entries, entries[i], tailq); - } - - batch->num_entries += num_dequeued; - num_remaining -= num_dequeued; - } - - if (num_remaining == 0) { - break; - } - } - - TAILQ_CONCAT(&dev->ioch_queue, &ioch_queue, tailq); - - if (batch->num_entries == dev->xfer_size) { - dev->current_batch = NULL; - } else { - dev->current_batch = batch; - batch = NULL; - } - - return batch; -} - -static void -ftl_release_batch(struct spdk_ftl_dev *dev, struct ftl_batch *batch) -{ - struct ftl_wbuf_entry *entry; - - while (!TAILQ_EMPTY(&batch->entries)) { - entry = TAILQ_FIRST(&batch->entries); - TAILQ_REMOVE(&batch->entries, entry, tailq); - ftl_release_wbuf_entry(entry); - } - - batch->num_entries = 0; - TAILQ_INSERT_TAIL(&dev->free_batches, batch, tailq); -} - -static struct ftl_wbuf_entry * -ftl_get_entry_from_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - struct ftl_io_channel *ioch; - uint64_t ioch_offset, entry_offset; - - ioch_offset = addr.cache_offset & ((1 << dev->ioch_shift) - 1); - entry_offset = addr.cache_offset >> dev->ioch_shift; - ioch = dev->ioch_array[ioch_offset]; - - assert(ioch_offset < dev->conf.max_io_channels); - assert(entry_offset < ioch->num_entries); - assert(addr.cached == 1); - - return &ioch->wbuf_entries[entry_offset]; -} - -static struct ftl_addr -ftl_get_addr_from_entry(struct ftl_wbuf_entry *entry) -{ - struct ftl_io_channel *ioch = entry->ioch; - struct ftl_addr addr = {}; - - addr.cached = 1; - addr.cache_offset = (uint64_t)entry->index << ioch->dev->ioch_shift | ioch->index; - - return addr; -} - -static void -ftl_io_cmpl_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_io *io = cb_arg; - struct spdk_ftl_dev *dev = io->dev; - - if (spdk_unlikely(!success)) { - io->status = -EIO; - } - - ftl_trace_completion(dev, io, FTL_TRACE_COMPLETION_DISK); - - if (io->type == FTL_IO_WRITE && ftl_is_append_supported(dev)) { - assert(io->parent); - io->parent->addr.offset = spdk_bdev_io_get_append_location(bdev_io); - } - - ftl_io_dec_req(io); - if (ftl_io_done(io)) { - ftl_io_complete(io); - } - - spdk_bdev_free_io(bdev_io); -} - -static void -ftl_halt_writes(struct spdk_ftl_dev *dev, struct ftl_band *band) -{ - struct ftl_wptr *wptr = NULL; - - LIST_FOREACH(wptr, &dev->wptr_list, list_entry) { - if (wptr->band == band) { - break; - } - } - - /* If the band already has the high_prio flag set, other writes must */ - /* have failed earlier, so it's already taken care of. */ - if (band->high_prio) { - assert(wptr == NULL); - return; - } - - ftl_band_write_failed(band); - ftl_remove_wptr(wptr); -} - -static struct ftl_wptr * -ftl_wptr_from_band(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_wptr *wptr = NULL; - - LIST_FOREACH(wptr, &dev->wptr_list, list_entry) { - if (wptr->band == band) { - return wptr; - } - } - assert(false); - return NULL; -} - -static void -ftl_md_write_fail(struct ftl_io *io, int status) -{ - struct ftl_band *band = io->band; - struct ftl_wptr *wptr; - char buf[128]; - - wptr = ftl_wptr_from_band(band); - SPDK_ERRLOG("Metadata write failed @addr: %s, status: %d\n", - ftl_addr2str(wptr->addr, buf, sizeof(buf)), status); - - ftl_halt_writes(io->dev, band); -} - -static void -ftl_md_write_cb(struct ftl_io *io, void *arg, int status) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_nv_cache *nv_cache = &dev->nv_cache; - struct ftl_band *band = io->band; - struct ftl_wptr *wptr; - size_t id; - - wptr = ftl_wptr_from_band(band); - - if (status) { - ftl_md_write_fail(io, status); - return; - } - - ftl_band_set_next_state(band); - if (band->state == FTL_BAND_STATE_CLOSED) { - if (ftl_dev_has_nv_cache(dev)) { - pthread_spin_lock(&nv_cache->lock); - nv_cache->num_available += ftl_band_user_blocks(band); - - if (spdk_unlikely(nv_cache->num_available > nv_cache->num_data_blocks)) { - nv_cache->num_available = nv_cache->num_data_blocks; - } - pthread_spin_unlock(&nv_cache->lock); - } - - /* - * Go through the reloc_bitmap, checking for all the bands that had its data moved - * onto current band and update their counters to allow them to be used for writing - * (once they're closed and empty). - */ - for (id = 0; id < ftl_get_num_bands(dev); ++id) { - if (spdk_bit_array_get(band->reloc_bitmap, id)) { - assert(dev->bands[id].num_reloc_bands > 0); - dev->bands[id].num_reloc_bands--; - - spdk_bit_array_clear(band->reloc_bitmap, id); - } - } - - ftl_remove_wptr(wptr); - } -} - -static int -ftl_read_next_physical_addr(struct ftl_io *io, struct ftl_addr *addr) -{ - struct spdk_ftl_dev *dev = io->dev; - size_t num_blocks, max_blocks; - - assert(ftl_io_mode_physical(io)); - assert(io->iov_pos < io->iov_cnt); - - if (io->pos == 0) { - *addr = io->addr; - } else { - *addr = ftl_band_next_xfer_addr(io->band, io->addr, io->pos); - } - - assert(!ftl_addr_invalid(*addr)); - - /* Metadata has to be read in the way it's written (jumping across */ - /* the zones in xfer_size increments) */ - if (io->flags & FTL_IO_MD) { - max_blocks = dev->xfer_size - (addr->offset % dev->xfer_size); - num_blocks = spdk_min(ftl_io_iovec_len_left(io), max_blocks); - assert(addr->offset / dev->xfer_size == - (addr->offset + num_blocks - 1) / dev->xfer_size); - } else { - num_blocks = ftl_io_iovec_len_left(io); - } - - return num_blocks; -} - -static int -ftl_wptr_close_band(struct ftl_wptr *wptr) -{ - struct ftl_band *band = wptr->band; - - ftl_band_set_state(band, FTL_BAND_STATE_CLOSING); - - return ftl_band_write_tail_md(band, ftl_md_write_cb); -} - -static int -ftl_wptr_open_band(struct ftl_wptr *wptr) -{ - struct ftl_band *band = wptr->band; - - assert(ftl_band_zone_is_first(band, wptr->zone)); - assert(band->lba_map.num_vld == 0); - - ftl_band_clear_lba_map(band); - - assert(band->state == FTL_BAND_STATE_PREP); - ftl_band_set_state(band, FTL_BAND_STATE_OPENING); - - return ftl_band_write_head_md(band, ftl_md_write_cb); -} - -static int -ftl_submit_erase(struct ftl_io *io) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_band *band = io->band; - struct ftl_addr addr = io->addr; - struct ftl_io_channel *ioch; - struct ftl_zone *zone; - int rc = 0; - size_t i; - - ioch = ftl_io_channel_get_ctx(ftl_get_io_channel(dev)); - - for (i = 0; i < io->num_blocks; ++i) { - if (i != 0) { - zone = ftl_band_next_zone(band, ftl_band_zone_from_addr(band, addr)); - assert(zone->info.state == SPDK_BDEV_ZONE_STATE_FULL); - addr.offset = zone->info.zone_id; - } - - assert(ftl_addr_get_zone_offset(dev, addr) == 0); - - ftl_trace_submission(dev, io, addr, 1); - rc = spdk_bdev_zone_management(dev->base_bdev_desc, ioch->base_ioch, addr.offset, - SPDK_BDEV_ZONE_RESET, ftl_io_cmpl_cb, io); - if (spdk_unlikely(rc)) { - ftl_io_fail(io, rc); - SPDK_ERRLOG("Vector reset failed with status: %d\n", rc); - break; - } - - ftl_io_inc_req(io); - ftl_io_advance(io, 1); - } - - if (ftl_io_done(io)) { - ftl_io_complete(io); - } - - return rc; -} - -static bool -ftl_check_core_thread(const struct spdk_ftl_dev *dev) -{ - return dev->core_thread == spdk_get_thread(); -} - -struct spdk_io_channel * -ftl_get_io_channel(const struct spdk_ftl_dev *dev) -{ - if (ftl_check_core_thread(dev)) { - return dev->ioch; - } - - return NULL; -} - -static void -ftl_erase_fail(struct ftl_io *io, int status) -{ - struct ftl_zone *zone; - struct ftl_band *band = io->band; - char buf[128]; - - SPDK_ERRLOG("Erase failed at address: %s, status: %d\n", - ftl_addr2str(io->addr, buf, sizeof(buf)), status); - - zone = ftl_band_zone_from_addr(band, io->addr); - zone->info.state = SPDK_BDEV_ZONE_STATE_OFFLINE; - ftl_band_remove_zone(band, zone); - band->tail_md_addr = ftl_band_tail_md_addr(band); -} - -static void -ftl_zone_erase_cb(struct ftl_io *io, void *ctx, int status) -{ - struct ftl_zone *zone; - - zone = ftl_band_zone_from_addr(io->band, io->addr); - zone->busy = false; - - if (spdk_unlikely(status)) { - ftl_erase_fail(io, status); - return; - } - - zone->info.state = SPDK_BDEV_ZONE_STATE_EMPTY; - zone->info.write_pointer = zone->info.zone_id; -} - -static int -ftl_band_erase(struct ftl_band *band) -{ - struct ftl_zone *zone; - struct ftl_io *io; - int rc = 0; - - assert(band->state == FTL_BAND_STATE_CLOSED || - band->state == FTL_BAND_STATE_FREE); - - ftl_band_set_state(band, FTL_BAND_STATE_PREP); - - CIRCLEQ_FOREACH(zone, &band->zones, circleq) { - if (zone->info.state == SPDK_BDEV_ZONE_STATE_EMPTY) { - continue; - } - - io = ftl_io_erase_init(band, 1, ftl_zone_erase_cb); - if (!io) { - rc = -ENOMEM; - break; - } - - zone->busy = true; - io->addr.offset = zone->info.zone_id; - rc = ftl_submit_erase(io); - if (rc) { - zone->busy = false; - assert(0); - /* TODO: change band's state back to close? */ - break; - } - } - - return rc; -} - -static struct ftl_band * -ftl_next_write_band(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band; - - /* Find a free band that has all of its data moved onto other closed bands */ - LIST_FOREACH(band, &dev->free_bands, list_entry) { - assert(band->state == FTL_BAND_STATE_FREE); - if (band->num_reloc_bands == 0 && band->num_reloc_blocks == 0) { - break; - } - } - - if (spdk_unlikely(!band)) { - return NULL; - } - - if (ftl_band_erase(band)) { - /* TODO: handle erase failure */ - return NULL; - } - - return band; -} - -static struct ftl_band * -ftl_next_wptr_band(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band; - - if (!dev->next_band) { - band = ftl_next_write_band(dev); - } else { - assert(dev->next_band->state == FTL_BAND_STATE_PREP); - band = dev->next_band; - dev->next_band = NULL; - } - - return band; -} - -static struct ftl_wptr * -ftl_wptr_init(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_wptr *wptr; - - wptr = calloc(1, sizeof(*wptr)); - if (!wptr) { - return NULL; - } - - wptr->dev = dev; - wptr->band = band; - wptr->zone = CIRCLEQ_FIRST(&band->zones); - wptr->addr.offset = wptr->zone->info.zone_id; - TAILQ_INIT(&wptr->pending_queue); - - return wptr; -} - -static int -ftl_add_direct_wptr(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_wptr *wptr; - - assert(band->state == FTL_BAND_STATE_OPEN); - - wptr = ftl_wptr_init(band); - if (!wptr) { - return -1; - } - - wptr->direct_mode = true; - - if (ftl_band_alloc_lba_map(band)) { - ftl_wptr_free(wptr); - return -1; - } - - LIST_INSERT_HEAD(&dev->wptr_list, wptr, list_entry); - - SPDK_DEBUGLOG(ftl_core, "wptr: direct band %u\n", band->id); - ftl_trace_write_band(dev, band); - return 0; -} - -static void -ftl_close_direct_wptr(struct ftl_band *band) -{ - struct ftl_wptr *wptr = ftl_wptr_from_band(band); - - assert(wptr->direct_mode); - assert(band->state == FTL_BAND_STATE_CLOSED); - - ftl_band_release_lba_map(band); - - ftl_remove_wptr(wptr); -} - -int -ftl_band_set_direct_access(struct ftl_band *band, bool access) -{ - if (access) { - return ftl_add_direct_wptr(band); - } else { - ftl_close_direct_wptr(band); - return 0; - } -} - -static int -ftl_add_wptr(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band; - struct ftl_wptr *wptr; - - band = ftl_next_wptr_band(dev); - if (!band) { - return -1; - } - - wptr = ftl_wptr_init(band); - if (!wptr) { - return -1; - } - - if (ftl_band_write_prep(band)) { - ftl_wptr_free(wptr); - return -1; - } - - LIST_INSERT_HEAD(&dev->wptr_list, wptr, list_entry); - - SPDK_DEBUGLOG(ftl_core, "wptr: band %u\n", band->id); - ftl_trace_write_band(dev, band); - return 0; -} - -static void -ftl_wptr_advance(struct ftl_wptr *wptr, size_t xfer_size) -{ - struct ftl_band *band = wptr->band; - struct spdk_ftl_dev *dev = wptr->dev; - struct spdk_ftl_conf *conf = &dev->conf; - size_t next_thld; - - if (spdk_unlikely(wptr->direct_mode)) { - return; - } - - wptr->offset += xfer_size; - next_thld = (ftl_band_num_usable_blocks(band) * conf->band_thld) / 100; - - if (ftl_band_full(band, wptr->offset)) { - ftl_band_set_state(band, FTL_BAND_STATE_FULL); - } - - wptr->zone->busy = true; - wptr->addr = ftl_band_next_xfer_addr(band, wptr->addr, xfer_size); - wptr->zone = ftl_band_next_operational_zone(band, wptr->zone); - - assert(!ftl_addr_invalid(wptr->addr)); - - SPDK_DEBUGLOG(ftl_core, "wptr: pu:%lu band:%lu, offset:%lu\n", - ftl_addr_get_punit(dev, wptr->addr), - ftl_addr_get_band(dev, wptr->addr), - wptr->addr.offset); - - if (wptr->offset >= next_thld && !dev->next_band) { - dev->next_band = ftl_next_write_band(dev); - } -} - -static size_t -ftl_wptr_user_blocks_left(const struct ftl_wptr *wptr) -{ - return ftl_band_user_blocks_left(wptr->band, wptr->offset); -} - -static bool -ftl_wptr_ready(struct ftl_wptr *wptr) -{ - struct ftl_band *band = wptr->band; - - /* TODO: add handling of empty bands */ - - if (spdk_unlikely(!ftl_zone_is_writable(wptr->dev, wptr->zone))) { - /* Erasing band may fail after it was assigned to wptr. */ - if (spdk_unlikely(wptr->zone->info.state == SPDK_BDEV_ZONE_STATE_OFFLINE)) { - ftl_wptr_advance(wptr, wptr->dev->xfer_size); - } - return false; - } - - /* If we're in the process of writing metadata, wait till it is */ - /* completed. */ - /* TODO: we should probably change bands once we're writing tail md */ - if (ftl_band_state_changing(band)) { - return false; - } - - if (band->state == FTL_BAND_STATE_FULL) { - if (wptr->num_outstanding == 0) { - if (ftl_wptr_close_band(wptr)) { - /* TODO: need recovery here */ - assert(false); - } - } - - return false; - } - - if (band->state != FTL_BAND_STATE_OPEN) { - if (ftl_wptr_open_band(wptr)) { - /* TODO: need recovery here */ - assert(false); - } - - return false; - } - - return true; -} - -int -ftl_flush_active_bands(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg) -{ - struct ftl_wptr *wptr; - struct ftl_band_flush *flush; - - assert(ftl_get_core_thread(dev) == spdk_get_thread()); - - flush = calloc(1, sizeof(*flush)); - if (spdk_unlikely(!flush)) { - return -ENOMEM; - } - - LIST_INSERT_HEAD(&dev->band_flush_list, flush, list_entry); - - flush->cb_fn = cb_fn; - flush->cb_arg = cb_arg; - flush->dev = dev; - - LIST_FOREACH(wptr, &dev->wptr_list, list_entry) { - wptr->flush = true; - flush->num_bands++; - } - - return 0; -} - -static const struct spdk_ftl_limit * -ftl_get_limit(const struct spdk_ftl_dev *dev, int type) -{ - assert(type < SPDK_FTL_LIMIT_MAX); - return &dev->conf.limits[type]; -} - -static bool -ftl_cache_lba_valid(struct spdk_ftl_dev *dev, struct ftl_wbuf_entry *entry) -{ - struct ftl_addr addr; - - /* If the LBA is invalid don't bother checking the md and l2p */ - if (spdk_unlikely(entry->lba == FTL_LBA_INVALID)) { - return false; - } - - addr = ftl_l2p_get(dev, entry->lba); - if (!(ftl_addr_cached(addr) && entry == ftl_get_entry_from_addr(dev, addr))) { - return false; - } - - return true; -} - -void -ftl_evict_cache_entry(struct spdk_ftl_dev *dev, struct ftl_wbuf_entry *entry) -{ - pthread_spin_lock(&entry->lock); - - if (!entry->valid) { - goto unlock; - } - - /* If the l2p wasn't updated and still points at the entry, fill it with the */ - /* on-disk address and clear the cache status bit. Otherwise, skip the l2p update */ - /* and just clear the cache status. */ - if (!ftl_cache_lba_valid(dev, entry)) { - goto clear; - } - - ftl_l2p_set(dev, entry->lba, entry->addr); -clear: - entry->valid = false; -unlock: - pthread_spin_unlock(&entry->lock); -} - -static void -ftl_pad_wbuf(struct spdk_ftl_dev *dev, size_t size) -{ - struct ftl_wbuf_entry *entry; - struct ftl_io_channel *ioch; - int flags = FTL_IO_PAD | FTL_IO_INTERNAL; - - ioch = ftl_io_channel_get_ctx(ftl_get_io_channel(dev)); - - for (size_t i = 0; i < size; ++i) { - entry = ftl_acquire_wbuf_entry(ioch, flags); - if (!entry) { - break; - } - - entry->lba = FTL_LBA_INVALID; - entry->addr = ftl_to_addr(FTL_ADDR_INVALID); - memset(entry->payload, 0, FTL_BLOCK_SIZE); - - spdk_ring_enqueue(ioch->submit_queue, (void **)&entry, 1, NULL); - } -} - -static void -ftl_remove_free_bands(struct spdk_ftl_dev *dev) -{ - while (!LIST_EMPTY(&dev->free_bands)) { - LIST_REMOVE(LIST_FIRST(&dev->free_bands), list_entry); - } - - dev->next_band = NULL; -} - -static void -ftl_wptr_pad_band(struct ftl_wptr *wptr) -{ - struct spdk_ftl_dev *dev = wptr->dev; - struct ftl_batch *batch = dev->current_batch; - struct ftl_io_channel *ioch; - struct ftl_io *io; - size_t size, pad_size, blocks_left; - - size = batch != NULL ? batch->num_entries : 0; - TAILQ_FOREACH(ioch, &dev->ioch_queue, tailq) { - size += spdk_ring_count(ioch->submit_queue); - - TAILQ_FOREACH(io, &ioch->retry_queue, ioch_entry) { - if (io->type == FTL_IO_WRITE) { - size += io->num_blocks - io->pos; - } - } - } - - ioch = ftl_io_channel_get_ctx(ftl_get_io_channel(dev)); - - blocks_left = ftl_wptr_user_blocks_left(wptr); - assert(size <= blocks_left); - assert(blocks_left % dev->xfer_size == 0); - pad_size = spdk_min(blocks_left - size, spdk_ring_count(ioch->free_queue)); - - ftl_pad_wbuf(dev, pad_size); -} - -static void -ftl_wptr_process_shutdown(struct ftl_wptr *wptr) -{ - struct spdk_ftl_dev *dev = wptr->dev; - struct ftl_batch *batch = dev->current_batch; - struct ftl_io_channel *ioch; - struct ftl_io *io; - size_t size; - - size = batch != NULL ? batch->num_entries : 0; - TAILQ_FOREACH(ioch, &dev->ioch_queue, tailq) { - size += spdk_ring_count(ioch->submit_queue); - - TAILQ_FOREACH(io, &ioch->retry_queue, ioch_entry) { - if (io->type == FTL_IO_WRITE) { - size += io->num_blocks - io->pos; - } - } - } - - if (size >= dev->xfer_size) { - return; - } - - /* If we reach this point we need to remove free bands */ - /* and pad current wptr band to the end */ - ftl_remove_free_bands(dev); - ftl_wptr_pad_band(wptr); -} - -static int -ftl_shutdown_complete(struct spdk_ftl_dev *dev) -{ - struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(dev->ioch); - - return !__atomic_load_n(&dev->num_inflight, __ATOMIC_SEQ_CST) && - dev->num_io_channels == 1 && LIST_EMPTY(&dev->wptr_list) && - TAILQ_EMPTY(&ioch->retry_queue); -} - -void -ftl_apply_limits(struct spdk_ftl_dev *dev) -{ - const struct spdk_ftl_limit *limit; - struct ftl_io_channel *ioch; - struct ftl_stats *stats = &dev->stats; - uint32_t qdepth_limit = 100; - int i; - - /* Clear existing limit */ - dev->limit = SPDK_FTL_LIMIT_MAX; - - for (i = SPDK_FTL_LIMIT_CRIT; i < SPDK_FTL_LIMIT_MAX; ++i) { - limit = ftl_get_limit(dev, i); - - if (dev->num_free <= limit->thld) { - qdepth_limit = limit->limit; - stats->limits[i]++; - dev->limit = i; - break; - } - } - - ftl_trace_limits(dev, dev->limit, dev->num_free); - TAILQ_FOREACH(ioch, &dev->ioch_queue, tailq) { - __atomic_store_n(&ioch->qdepth_limit, (qdepth_limit * ioch->num_entries) / 100, - __ATOMIC_SEQ_CST); - } -} - -static int -ftl_invalidate_addr_unlocked(struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - struct ftl_band *band = ftl_band_from_addr(dev, addr); - struct ftl_lba_map *lba_map = &band->lba_map; - uint64_t offset; - - offset = ftl_band_block_offset_from_addr(band, addr); - - /* The bit might be already cleared if two writes are scheduled to the */ - /* same LBA at the same time */ - if (spdk_bit_array_get(lba_map->vld, offset)) { - assert(lba_map->num_vld > 0); - spdk_bit_array_clear(lba_map->vld, offset); - lba_map->num_vld--; - return 1; - } - - return 0; -} - -int -ftl_invalidate_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - struct ftl_band *band; - int rc; - - assert(!ftl_addr_cached(addr)); - band = ftl_band_from_addr(dev, addr); - - pthread_spin_lock(&band->lba_map.lock); - rc = ftl_invalidate_addr_unlocked(dev, addr); - pthread_spin_unlock(&band->lba_map.lock); - - return rc; -} - -static int -ftl_read_retry(int rc) -{ - return rc == -EAGAIN; -} - -static int -ftl_read_canceled(int rc) -{ - return rc == -EFAULT || rc == 0; -} - -static int -ftl_cache_read(struct ftl_io *io, uint64_t lba, - struct ftl_addr addr, void *buf) -{ - struct ftl_wbuf_entry *entry; - struct ftl_addr naddr; - int rc = 0; - - entry = ftl_get_entry_from_addr(io->dev, addr); - pthread_spin_lock(&entry->lock); - - naddr = ftl_l2p_get(io->dev, lba); - if (addr.offset != naddr.offset) { - rc = -1; - goto out; - } - - memcpy(buf, entry->payload, FTL_BLOCK_SIZE); -out: - pthread_spin_unlock(&entry->lock); - return rc; -} - -static int -ftl_read_next_logical_addr(struct ftl_io *io, struct ftl_addr *addr) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_addr next_addr; - size_t i; - - *addr = ftl_l2p_get(dev, ftl_io_current_lba(io)); - - SPDK_DEBUGLOG(ftl_core, "Read addr:%lx, lba:%lu\n", - addr->offset, ftl_io_current_lba(io)); - - /* If the address is invalid, skip it (the buffer should already be zeroed) */ - if (ftl_addr_invalid(*addr)) { - return -EFAULT; - } - - if (ftl_addr_cached(*addr)) { - if (!ftl_cache_read(io, ftl_io_current_lba(io), *addr, ftl_io_iovec_addr(io))) { - return 0; - } - - /* If the state changed, we have to re-read the l2p */ - return -EAGAIN; - } - - for (i = 1; i < ftl_io_iovec_len_left(io); ++i) { - next_addr = ftl_l2p_get(dev, ftl_io_get_lba(io, io->pos + i)); - - if (ftl_addr_invalid(next_addr) || ftl_addr_cached(next_addr)) { - break; - } - - if (addr->offset + i != next_addr.offset) { - break; - } - } - - return i; -} - -static int -ftl_submit_read(struct ftl_io *io) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_io_channel *ioch; - struct ftl_addr addr; - int rc = 0, num_blocks; - - ioch = ftl_io_channel_get_ctx(io->ioch); - - assert(LIST_EMPTY(&io->children)); - - while (io->pos < io->num_blocks) { - if (ftl_io_mode_physical(io)) { - num_blocks = rc = ftl_read_next_physical_addr(io, &addr); - } else { - num_blocks = rc = ftl_read_next_logical_addr(io, &addr); - } - - /* We might need to retry the read from scratch (e.g. */ - /* because write was under way and completed before */ - /* we could read it from the write buffer */ - if (ftl_read_retry(rc)) { - continue; - } - - /* We don't have to schedule the read, as it was read from cache */ - if (ftl_read_canceled(rc)) { - ftl_io_advance(io, 1); - ftl_trace_completion(io->dev, io, rc ? FTL_TRACE_COMPLETION_INVALID : - FTL_TRACE_COMPLETION_CACHE); - rc = 0; - continue; - } - - assert(num_blocks > 0); - - ftl_trace_submission(dev, io, addr, num_blocks); - rc = spdk_bdev_read_blocks(dev->base_bdev_desc, ioch->base_ioch, - ftl_io_iovec_addr(io), - addr.offset, - num_blocks, ftl_io_cmpl_cb, io); - if (spdk_unlikely(rc)) { - if (rc == -ENOMEM) { - TAILQ_INSERT_TAIL(&ioch->retry_queue, io, ioch_entry); - rc = 0; - } else { - ftl_io_fail(io, rc); - } - break; - } - - ftl_io_inc_req(io); - ftl_io_advance(io, num_blocks); - } - - /* If we didn't have to read anything from the device, */ - /* complete the request right away */ - if (ftl_io_done(io)) { - ftl_io_complete(io); - } - - return rc; -} - -static void -ftl_complete_flush(struct ftl_flush *flush) -{ - assert(flush->num_req == 0); - LIST_REMOVE(flush, list_entry); - - flush->cb.fn(flush->cb.ctx, 0); - - spdk_bit_array_free(&flush->bmap); - free(flush); -} - -static void -ftl_process_flush(struct spdk_ftl_dev *dev, struct ftl_batch *batch) -{ - struct ftl_flush *flush, *tflush; - size_t offset; - - LIST_FOREACH_SAFE(flush, &dev->flush_list, list_entry, tflush) { - offset = batch->index; - - if (spdk_bit_array_get(flush->bmap, offset)) { - spdk_bit_array_clear(flush->bmap, offset); - if (!(--flush->num_req)) { - ftl_complete_flush(flush); - } - } - } -} - -static void -ftl_nv_cache_wrap_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_nv_cache *nv_cache = cb_arg; - - if (!success) { - SPDK_ERRLOG("Unable to write non-volatile cache metadata header\n"); - /* TODO: go into read-only mode */ - assert(0); - } - - pthread_spin_lock(&nv_cache->lock); - nv_cache->ready = true; - pthread_spin_unlock(&nv_cache->lock); - - spdk_bdev_free_io(bdev_io); -} - -static void -ftl_nv_cache_wrap(void *ctx) -{ - struct ftl_nv_cache *nv_cache = ctx; - int rc; - - rc = ftl_nv_cache_write_header(nv_cache, false, ftl_nv_cache_wrap_cb, nv_cache); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to write non-volatile cache metadata header: %s\n", - spdk_strerror(-rc)); - /* TODO: go into read-only mode */ - assert(0); - } -} - -static uint64_t -ftl_reserve_nv_cache(struct ftl_nv_cache *nv_cache, size_t *num_blocks, unsigned int *phase) -{ - struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); - uint64_t num_available, cache_size, cache_addr = FTL_LBA_INVALID; - - cache_size = spdk_bdev_get_num_blocks(bdev); - - pthread_spin_lock(&nv_cache->lock); - if (spdk_unlikely(nv_cache->num_available == 0 || !nv_cache->ready)) { - goto out; - } - - num_available = spdk_min(nv_cache->num_available, *num_blocks); - num_available = spdk_min(num_available, dev->conf.nv_cache.max_request_cnt); - - if (spdk_unlikely(nv_cache->current_addr + num_available > cache_size)) { - *num_blocks = cache_size - nv_cache->current_addr; - } else { - *num_blocks = num_available; - } - - cache_addr = nv_cache->current_addr; - nv_cache->current_addr += *num_blocks; - nv_cache->num_available -= *num_blocks; - *phase = nv_cache->phase; - - if (nv_cache->current_addr == spdk_bdev_get_num_blocks(bdev)) { - nv_cache->current_addr = FTL_NV_CACHE_DATA_OFFSET; - nv_cache->phase = ftl_nv_cache_next_phase(nv_cache->phase); - nv_cache->ready = false; - spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_nv_cache_wrap, nv_cache); - } -out: - pthread_spin_unlock(&nv_cache->lock); - return cache_addr; -} - -static struct ftl_io * -ftl_alloc_io_nv_cache(struct ftl_io *parent, size_t num_blocks) -{ - struct ftl_io_init_opts opts = { - .dev = parent->dev, - .parent = parent, - .iovcnt = 0, - .num_blocks = num_blocks, - .flags = parent->flags | FTL_IO_CACHE, - }; - - return ftl_io_init_internal(&opts); -} - -static void -ftl_nv_cache_submit_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_io *io = cb_arg; - struct ftl_nv_cache *nv_cache = &io->dev->nv_cache; - - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Non-volatile cache write failed at %"PRIx64"\n", io->addr.offset); - io->status = -EIO; - } - - ftl_io_dec_req(io); - if (ftl_io_done(io)) { - spdk_mempool_put(nv_cache->md_pool, io->md); - ftl_io_complete(io); - } - - spdk_bdev_free_io(bdev_io); -} - -static void -ftl_submit_nv_cache(void *ctx) -{ - struct ftl_io *io = ctx; - struct spdk_ftl_dev *dev = io->dev; - struct spdk_thread *thread; - struct ftl_nv_cache *nv_cache = &dev->nv_cache; - struct ftl_io_channel *ioch; - int rc; - - ioch = ftl_io_channel_get_ctx(io->ioch); - thread = spdk_io_channel_get_thread(io->ioch); - - rc = spdk_bdev_write_blocks_with_md(nv_cache->bdev_desc, ioch->cache_ioch, - ftl_io_iovec_addr(io), io->md, io->addr.offset, - io->num_blocks, ftl_nv_cache_submit_cb, io); - if (rc == -ENOMEM) { - spdk_thread_send_msg(thread, ftl_submit_nv_cache, io); - return; - } else if (rc) { - SPDK_ERRLOG("Write to persistent cache failed: %s (%"PRIu64", %"PRIu64")\n", - spdk_strerror(-rc), io->addr.offset, io->num_blocks); - spdk_mempool_put(nv_cache->md_pool, io->md); - io->status = -EIO; - ftl_io_complete(io); - return; - } - - ftl_io_advance(io, io->num_blocks); - ftl_io_inc_req(io); -} - -static void -ftl_nv_cache_fill_md(struct ftl_io *io, unsigned int phase) -{ - struct spdk_bdev *bdev; - struct ftl_nv_cache *nv_cache = &io->dev->nv_cache; - uint64_t block_off, lba; - void *md_buf = io->md; - - bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - - for (block_off = 0; block_off < io->num_blocks; ++block_off) { - lba = ftl_nv_cache_pack_lba(ftl_io_get_lba(io, block_off), phase); - memcpy(md_buf, &lba, sizeof(lba)); - md_buf += spdk_bdev_get_md_size(bdev); - } -} - -static void -_ftl_write_nv_cache(void *ctx) -{ - struct ftl_io *child, *io = ctx; - struct spdk_ftl_dev *dev = io->dev; - struct spdk_thread *thread; - unsigned int phase; - uint64_t num_blocks; - - thread = spdk_io_channel_get_thread(io->ioch); - - while (io->pos < io->num_blocks) { - num_blocks = ftl_io_iovec_len_left(io); - - child = ftl_alloc_io_nv_cache(io, num_blocks); - if (spdk_unlikely(!child)) { - spdk_thread_send_msg(thread, _ftl_write_nv_cache, io); - return; - } - - child->md = spdk_mempool_get(dev->nv_cache.md_pool); - if (spdk_unlikely(!child->md)) { - ftl_io_free(child); - spdk_thread_send_msg(thread, _ftl_write_nv_cache, io); - break; - } - - /* Reserve area on the write buffer cache */ - child->addr.offset = ftl_reserve_nv_cache(&dev->nv_cache, &num_blocks, &phase); - if (child->addr.offset == FTL_LBA_INVALID) { - spdk_mempool_put(dev->nv_cache.md_pool, child->md); - ftl_io_free(child); - spdk_thread_send_msg(thread, _ftl_write_nv_cache, io); - break; - } - - /* Shrink the IO if there isn't enough room in the cache to fill the whole iovec */ - if (spdk_unlikely(num_blocks != ftl_io_iovec_len_left(io))) { - ftl_io_shrink_iovec(child, num_blocks); - } - - ftl_nv_cache_fill_md(child, phase); - ftl_submit_nv_cache(child); - } - - if (ftl_io_done(io)) { - ftl_io_complete(io); - } -} - -static void -ftl_write_nv_cache(struct ftl_io *parent) -{ - ftl_io_reset(parent); - parent->flags |= FTL_IO_CACHE; - _ftl_write_nv_cache(parent); -} - -int -ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, - spdk_bdev_io_completion_cb cb_fn, void *cb_arg) -{ - struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); - struct ftl_nv_cache_header *hdr = nv_cache->dma_buf; - struct spdk_bdev *bdev; - struct ftl_io_channel *ioch; - - bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - ioch = ftl_io_channel_get_ctx(ftl_get_io_channel(dev)); - - memset(hdr, 0, spdk_bdev_get_block_size(bdev)); - - hdr->phase = (uint8_t)nv_cache->phase; - hdr->size = spdk_bdev_get_num_blocks(bdev); - hdr->uuid = dev->uuid; - hdr->version = FTL_NV_CACHE_HEADER_VERSION; - hdr->current_addr = shutdown ? nv_cache->current_addr : FTL_LBA_INVALID; - hdr->checksum = spdk_crc32c_update(hdr, offsetof(struct ftl_nv_cache_header, checksum), 0); - - return spdk_bdev_write_blocks(nv_cache->bdev_desc, ioch->cache_ioch, hdr, 0, 1, - cb_fn, cb_arg); -} - -int -ftl_nv_cache_scrub(struct ftl_nv_cache *nv_cache, spdk_bdev_io_completion_cb cb_fn, void *cb_arg) -{ - struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); - struct ftl_io_channel *ioch; - struct spdk_bdev *bdev; - - ioch = ftl_io_channel_get_ctx(ftl_get_io_channel(dev)); - bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - - return spdk_bdev_write_zeroes_blocks(nv_cache->bdev_desc, ioch->cache_ioch, 1, - spdk_bdev_get_num_blocks(bdev) - 1, - cb_fn, cb_arg); -} - -static void -ftl_write_fail(struct ftl_io *io, int status) -{ - struct ftl_batch *batch = io->batch; - struct spdk_ftl_dev *dev = io->dev; - struct ftl_wbuf_entry *entry; - struct ftl_band *band; - char buf[128]; - - entry = TAILQ_FIRST(&batch->entries); - - band = ftl_band_from_addr(io->dev, entry->addr); - SPDK_ERRLOG("Write failed @addr: %s, status: %d\n", - ftl_addr2str(entry->addr, buf, sizeof(buf)), status); - - /* Close the band and, halt wptr and defrag */ - ftl_halt_writes(dev, band); - - TAILQ_FOREACH(entry, &batch->entries, tailq) { - /* Invalidate meta set by process_writes() */ - ftl_invalidate_addr(dev, entry->addr); - } - - /* Reset the batch back to the write buffer to resend it later */ - TAILQ_INSERT_TAIL(&dev->pending_batches, batch, tailq); -} - -static void -ftl_write_cb(struct ftl_io *io, void *arg, int status) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_batch *batch = io->batch; - struct ftl_wbuf_entry *entry; - struct ftl_band *band; - struct ftl_addr prev_addr, addr = io->addr; - - if (status) { - ftl_write_fail(io, status); - return; - } - - assert(io->num_blocks == dev->xfer_size); - assert(!(io->flags & FTL_IO_MD)); - - TAILQ_FOREACH(entry, &batch->entries, tailq) { - band = entry->band; - if (!(entry->io_flags & FTL_IO_PAD)) { - /* Verify that the LBA is set for user blocks */ - assert(entry->lba != FTL_LBA_INVALID); - } - - if (band != NULL) { - assert(band->num_reloc_blocks > 0); - band->num_reloc_blocks--; - } - - entry->addr = addr; - if (entry->lba != FTL_LBA_INVALID) { - pthread_spin_lock(&entry->lock); - prev_addr = ftl_l2p_get(dev, entry->lba); - - /* If the l2p was updated in the meantime, don't update band's metadata */ - if (ftl_addr_cached(prev_addr) && - entry == ftl_get_entry_from_addr(dev, prev_addr)) { - /* Setting entry's cache bit needs to be done after metadata */ - /* within the band is updated to make sure that writes */ - /* invalidating the entry clear the metadata as well */ - ftl_band_set_addr(io->band, entry->lba, entry->addr); - entry->valid = true; - } - pthread_spin_unlock(&entry->lock); - } - - SPDK_DEBUGLOG(ftl_core, "Write addr:%lu, lba:%lu\n", - entry->addr.offset, entry->lba); - - addr = ftl_band_next_addr(io->band, addr, 1); - } - - ftl_process_flush(dev, batch); - ftl_release_batch(dev, batch); -} - -static void -ftl_update_stats(struct spdk_ftl_dev *dev, const struct ftl_wbuf_entry *entry) -{ - if (!(entry->io_flags & FTL_IO_INTERNAL)) { - dev->stats.write_user++; - } - dev->stats.write_total++; -} - -static void -ftl_update_l2p(struct spdk_ftl_dev *dev, const struct ftl_wbuf_entry *entry, - struct ftl_addr addr) -{ - struct ftl_addr prev_addr; - struct ftl_wbuf_entry *prev; - struct ftl_band *band; - int valid; - bool io_weak = entry->io_flags & FTL_IO_WEAK; - - prev_addr = ftl_l2p_get(dev, entry->lba); - if (ftl_addr_invalid(prev_addr)) { - ftl_l2p_set(dev, entry->lba, addr); - return; - } - - if (ftl_addr_cached(prev_addr)) { - prev = ftl_get_entry_from_addr(dev, prev_addr); - pthread_spin_lock(&prev->lock); - - /* Re-read the L2P under the lock to protect against updates */ - /* to this LBA from other threads */ - prev_addr = ftl_l2p_get(dev, entry->lba); - - /* If the entry is no longer in cache, another write has been */ - /* scheduled in the meantime, so we can return to evicted path */ - if (!ftl_addr_cached(prev_addr)) { - pthread_spin_unlock(&prev->lock); - goto evicted; - } - - /* - * Relocating block could still reside in cache due to fact that write - * buffers are independent for each IO channel and enough amount of data - * (write unit size) must be collected before it will be submitted to lower - * layer. - * When previous entry wasn't overwritten invalidate old address and entry. - * Otherwise skip relocating block. - */ - if (io_weak && - /* Check if prev_addr was updated in meantime */ - !(ftl_addr_cmp(prev_addr, ftl_get_addr_from_entry(prev)) && - /* Check if relocating address it the same as in previous entry */ - ftl_addr_cmp(prev->addr, entry->addr))) { - pthread_spin_unlock(&prev->lock); - return; - } - - /* - * If previous entry is part of cache and was written into disk remove - * and invalidate it - */ - if (prev->valid) { - ftl_invalidate_addr(dev, prev->addr); - prev->valid = false; - } - - ftl_l2p_set(dev, entry->lba, addr); - pthread_spin_unlock(&prev->lock); - return; - } - -evicted: - /* - * If the L2P's physical address is different than what we expected we don't need to - * do anything (someone's already overwritten our data). - */ - if (io_weak && !ftl_addr_cmp(prev_addr, entry->addr)) { - return; - } - - /* Lock the band containing previous physical address. This assures atomic changes to */ - /* the L2P as wall as metadata. The valid bits in metadata are used to */ - /* check weak writes validity. */ - band = ftl_band_from_addr(dev, prev_addr); - pthread_spin_lock(&band->lba_map.lock); - - valid = ftl_invalidate_addr_unlocked(dev, prev_addr); - - /* If the address has been invalidated already, we don't want to update */ - /* the L2P for weak writes, as it means the write is no longer valid. */ - if (!io_weak || valid) { - ftl_l2p_set(dev, entry->lba, addr); - } - - pthread_spin_unlock(&band->lba_map.lock); -} - -static struct ftl_io * -ftl_io_init_child_write(struct ftl_io *parent, struct ftl_addr addr, ftl_io_fn cb) -{ - struct ftl_io *io; - struct spdk_ftl_dev *dev = parent->dev; - struct ftl_io_init_opts opts = { - .dev = dev, - .io = NULL, - .parent = parent, - .band = parent->band, - .size = sizeof(struct ftl_io), - .flags = 0, - .type = parent->type, - .num_blocks = dev->xfer_size, - .cb_fn = cb, - .iovcnt = 0, - }; - - io = ftl_io_init_internal(&opts); - if (!io) { - return NULL; - } - - io->addr = addr; - - return io; -} - -static void -ftl_io_child_write_cb(struct ftl_io *io, void *ctx, int status) -{ - struct ftl_zone *zone; - struct ftl_wptr *wptr; - - zone = ftl_band_zone_from_addr(io->band, io->addr); - wptr = ftl_wptr_from_band(io->band); - - zone->busy = false; - zone->info.write_pointer += io->num_blocks; - - if (zone->info.write_pointer == zone->info.zone_id + zone->info.capacity) { - zone->info.state = SPDK_BDEV_ZONE_STATE_FULL; - } - - /* If some other write on the same band failed the write pointer would already be freed */ - if (spdk_likely(wptr)) { - wptr->num_outstanding--; - } -} - -static int -ftl_submit_child_write(struct ftl_wptr *wptr, struct ftl_io *io) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_io_channel *ioch; - struct ftl_io *child; - struct ftl_addr addr; - int rc; - - ioch = ftl_io_channel_get_ctx(io->ioch); - - if (spdk_likely(!wptr->direct_mode)) { - addr = wptr->addr; - } else { - assert(io->flags & FTL_IO_DIRECT_ACCESS); - assert(ftl_addr_get_band(dev, io->addr) == wptr->band->id); - addr = io->addr; - } - - /* Split IO to child requests and release zone immediately after child is completed */ - child = ftl_io_init_child_write(io, addr, ftl_io_child_write_cb); - if (!child) { - return -EAGAIN; - } - - wptr->num_outstanding++; - - if (ftl_is_append_supported(dev)) { - rc = spdk_bdev_zone_appendv(dev->base_bdev_desc, ioch->base_ioch, - child->iov, child->iov_cnt, - ftl_addr_get_zone_slba(dev, addr), - dev->xfer_size, ftl_io_cmpl_cb, child); - } else { - rc = spdk_bdev_writev_blocks(dev->base_bdev_desc, ioch->base_ioch, - child->iov, child->iov_cnt, addr.offset, - dev->xfer_size, ftl_io_cmpl_cb, child); - } - - if (rc) { - wptr->num_outstanding--; - ftl_io_fail(child, rc); - ftl_io_complete(child); - SPDK_ERRLOG("spdk_bdev_write_blocks_with_md failed with status:%d, addr:%lu\n", - rc, addr.offset); - return -EIO; - } - - ftl_io_inc_req(child); - ftl_io_advance(child, dev->xfer_size); - - return 0; -} - -static int -ftl_submit_write(struct ftl_wptr *wptr, struct ftl_io *io) -{ - struct spdk_ftl_dev *dev = io->dev; - int rc = 0; - - assert(io->num_blocks % dev->xfer_size == 0); - - while (io->iov_pos < io->iov_cnt) { - /* There are no guarantees of the order of completion of NVMe IO submission queue */ - /* so wait until zone is not busy before submitting another write */ - if (!ftl_is_append_supported(dev) && wptr->zone->busy) { - TAILQ_INSERT_TAIL(&wptr->pending_queue, io, ioch_entry); - rc = -EAGAIN; - break; - } - - rc = ftl_submit_child_write(wptr, io); - if (spdk_unlikely(rc)) { - if (rc == -EAGAIN) { - TAILQ_INSERT_TAIL(&wptr->pending_queue, io, ioch_entry); - } else { - ftl_io_fail(io, rc); - } - break; - } - - ftl_trace_submission(dev, io, wptr->addr, dev->xfer_size); - ftl_wptr_advance(wptr, dev->xfer_size); - } - - if (ftl_io_done(io)) { - /* Parent IO will complete after all children are completed */ - ftl_io_complete(io); - } - - return rc; -} - -static void -ftl_flush_pad_batch(struct spdk_ftl_dev *dev) -{ - struct ftl_batch *batch = dev->current_batch; - struct ftl_io_channel *ioch; - size_t size = 0, num_entries = 0; - - assert(batch != NULL); - assert(batch->num_entries < dev->xfer_size); - - TAILQ_FOREACH(ioch, &dev->ioch_queue, tailq) { - size += spdk_ring_count(ioch->submit_queue); - } - - num_entries = dev->xfer_size - batch->num_entries; - if (size < num_entries) { - ftl_pad_wbuf(dev, num_entries - size); - } -} - -static bool -ftl_check_io_channel_flush(struct spdk_ftl_dev *dev) -{ - struct ftl_io_channel *ioch; - - TAILQ_FOREACH(ioch, &dev->ioch_queue, tailq) { - if (ioch->flush && spdk_ring_count(ioch->free_queue) != ioch->num_entries) { - return true; - } - } - - return false; -} - -static int -ftl_wptr_process_writes(struct ftl_wptr *wptr) -{ - struct spdk_ftl_dev *dev = wptr->dev; - struct ftl_batch *batch; - struct ftl_wbuf_entry *entry; - struct ftl_io *io; - - if (spdk_unlikely(!TAILQ_EMPTY(&wptr->pending_queue))) { - io = TAILQ_FIRST(&wptr->pending_queue); - TAILQ_REMOVE(&wptr->pending_queue, io, ioch_entry); - - if (ftl_submit_write(wptr, io) == -EAGAIN) { - return 0; - } - } - - /* Make sure the band is prepared for writing */ - if (!ftl_wptr_ready(wptr)) { - return 0; - } - - if (dev->halt) { - ftl_wptr_process_shutdown(wptr); - } - - if (spdk_unlikely(wptr->flush)) { - ftl_wptr_pad_band(wptr); - } - - batch = ftl_get_next_batch(dev); - if (!batch) { - /* If there are queued flush requests we need to pad the write buffer to */ - /* force out remaining entries */ - if (!LIST_EMPTY(&dev->flush_list) || ftl_check_io_channel_flush(dev)) { - ftl_flush_pad_batch(dev); - } - - return 0; - } - - io = ftl_io_wbuf_init(dev, wptr->addr, wptr->band, batch, ftl_write_cb); - if (!io) { - goto error; - } - - TAILQ_FOREACH(entry, &batch->entries, tailq) { - /* Update band's relocation stats if the IO comes from reloc */ - if (entry->io_flags & FTL_IO_WEAK) { - if (!spdk_bit_array_get(wptr->band->reloc_bitmap, entry->band->id)) { - spdk_bit_array_set(wptr->band->reloc_bitmap, entry->band->id); - entry->band->num_reloc_bands++; - } - } - - ftl_trace_wbuf_pop(dev, entry); - ftl_update_stats(dev, entry); - } - - SPDK_DEBUGLOG(ftl_core, "Write addr:%lx\n", wptr->addr.offset); - - if (ftl_submit_write(wptr, io)) { - /* TODO: we need some recovery here */ - assert(0 && "Write submit failed"); - if (ftl_io_done(io)) { - ftl_io_free(io); - } - } - - return dev->xfer_size; -error: - TAILQ_INSERT_TAIL(&dev->pending_batches, batch, tailq); - return 0; -} - -static bool -ftl_process_writes(struct spdk_ftl_dev *dev) -{ - struct ftl_wptr *wptr, *twptr; - size_t num_active = 0, num_writes = 0; - enum ftl_band_state state; - - LIST_FOREACH_SAFE(wptr, &dev->wptr_list, list_entry, twptr) { - num_writes += ftl_wptr_process_writes(wptr); - state = wptr->band->state; - - if (state != FTL_BAND_STATE_FULL && - state != FTL_BAND_STATE_CLOSING && - state != FTL_BAND_STATE_CLOSED) { - num_active++; - } - } - - if (num_active < 1) { - ftl_add_wptr(dev); - } - - return num_writes != 0; -} - -static void -ftl_fill_wbuf_entry(struct ftl_wbuf_entry *entry, struct ftl_io *io) -{ - memcpy(entry->payload, ftl_io_iovec_addr(io), FTL_BLOCK_SIZE); - - if (entry->io_flags & FTL_IO_WEAK) { - entry->band = ftl_band_from_addr(io->dev, io->addr); - entry->addr = ftl_band_next_addr(entry->band, io->addr, io->pos); - entry->band->num_reloc_blocks++; - } - - entry->trace = io->trace; - entry->lba = ftl_io_current_lba(io); -} - -static int -ftl_wbuf_fill(struct ftl_io *io) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_io_channel *ioch; - struct ftl_wbuf_entry *entry; - - ioch = ftl_io_channel_get_ctx(io->ioch); - - while (io->pos < io->num_blocks) { - if (ftl_io_current_lba(io) == FTL_LBA_INVALID) { - ftl_io_advance(io, 1); - continue; - } - - entry = ftl_acquire_wbuf_entry(ioch, io->flags); - if (!entry) { - TAILQ_INSERT_TAIL(&ioch->retry_queue, io, ioch_entry); - return 0; - } - - ftl_fill_wbuf_entry(entry, io); - - ftl_trace_wbuf_fill(dev, io); - ftl_update_l2p(dev, entry, ftl_get_addr_from_entry(entry)); - ftl_io_advance(io, 1); - - /* Needs to be done after L2P is updated to avoid race with */ - /* write completion callback when it's processed faster than */ - /* L2P is set in update_l2p(). */ - spdk_ring_enqueue(ioch->submit_queue, (void **)&entry, 1, NULL); - } - - if (ftl_io_done(io)) { - if (ftl_dev_has_nv_cache(dev) && !(io->flags & FTL_IO_BYPASS_CACHE)) { - ftl_write_nv_cache(io); - } else { - TAILQ_INSERT_TAIL(&ioch->write_cmpl_queue, io, ioch_entry); - } - } - - return 0; -} - -static bool -ftl_dev_needs_defrag(struct spdk_ftl_dev *dev) -{ - const struct spdk_ftl_limit *limit = ftl_get_limit(dev, SPDK_FTL_LIMIT_START); - - if (ftl_reloc_is_halted(dev->reloc)) { - return false; - } - - if (ftl_reloc_is_defrag_active(dev->reloc)) { - return false; - } - - if (dev->num_free <= limit->thld) { - return true; - } - - return false; -} - -static double -ftl_band_calc_merit(struct ftl_band *band, size_t *threshold_valid) -{ - size_t usable, valid, invalid; - double vld_ratio; - - /* If the band doesn't have any usable blocks it's of no use */ - usable = ftl_band_num_usable_blocks(band); - if (usable == 0) { - return 0.0; - } - - valid = threshold_valid ? (usable - *threshold_valid) : band->lba_map.num_vld; - invalid = usable - valid; - - /* Add one to avoid division by 0 */ - vld_ratio = (double)invalid / (double)(valid + 1); - return vld_ratio * ftl_band_age(band); -} - -static bool -ftl_band_needs_defrag(struct ftl_band *band, struct spdk_ftl_dev *dev) -{ - struct spdk_ftl_conf *conf = &dev->conf; - size_t thld_vld; - - /* If we're in dire need of free bands, every band is worth defragging */ - if (ftl_current_limit(dev) == SPDK_FTL_LIMIT_CRIT) { - return true; - } - - thld_vld = (ftl_band_num_usable_blocks(band) * conf->invalid_thld) / 100; - - return band->merit > ftl_band_calc_merit(band, &thld_vld); -} - -static struct ftl_band * -ftl_select_defrag_band(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band, *mband = NULL; - double merit = 0; - - LIST_FOREACH(band, &dev->shut_bands, list_entry) { - assert(band->state == FTL_BAND_STATE_CLOSED); - band->merit = ftl_band_calc_merit(band, NULL); - if (band->merit > merit) { - merit = band->merit; - mband = band; - } - } - - if (mband && !ftl_band_needs_defrag(mband, dev)) { - mband = NULL; - } - - return mband; -} - -static bool -ftl_process_relocs(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band; - - if (ftl_dev_needs_defrag(dev)) { - band = ftl_select_defrag_band(dev); - if (band) { - ftl_reloc_add(dev->reloc, band, 0, ftl_get_num_blocks_in_band(dev), 0, true); - ftl_trace_defrag_band(dev, band); - } - } - - return ftl_reloc(dev->reloc); -} - -int -ftl_current_limit(const struct spdk_ftl_dev *dev) -{ - return dev->limit; -} - -void -spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attrs) -{ - attrs->uuid = dev->uuid; - attrs->num_blocks = dev->num_lbas; - attrs->block_size = FTL_BLOCK_SIZE; - attrs->num_zones = ftl_get_num_zones(dev); - attrs->zone_size = ftl_get_num_blocks_in_zone(dev); - attrs->conf = dev->conf; - attrs->base_bdev = spdk_bdev_get_name(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); - - attrs->cache_bdev = NULL; - if (dev->nv_cache.bdev_desc) { - attrs->cache_bdev = spdk_bdev_get_name( - spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc)); - } -} - -static void -_ftl_io_write(void *ctx) -{ - ftl_io_write((struct ftl_io *)ctx); -} - -static int -ftl_submit_write_leaf(struct ftl_io *io) -{ - int rc; - - rc = ftl_submit_write(ftl_wptr_from_band(io->band), io); - if (rc == -EAGAIN) { - /* EAGAIN means that the request was put on the pending queue */ - return 0; - } - - return rc; -} - -void -ftl_io_write(struct ftl_io *io) -{ - struct spdk_ftl_dev *dev = io->dev; - struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch); - - /* Put the IO on retry queue in case IO channel is not initialized */ - if (spdk_unlikely(ioch->index == FTL_IO_CHANNEL_INDEX_INVALID)) { - TAILQ_INSERT_TAIL(&ioch->retry_queue, io, ioch_entry); - return; - } - - /* For normal IOs we just need to copy the data onto the write buffer */ - if (!(io->flags & FTL_IO_MD)) { - ftl_io_call_foreach_child(io, ftl_wbuf_fill); - } else { - /* Metadata has its own buffer, so it doesn't have to be copied, so just */ - /* send it the the core thread and schedule the write immediately */ - if (ftl_check_core_thread(dev)) { - ftl_io_call_foreach_child(io, ftl_submit_write_leaf); - } else { - spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_io_write, io); - } - } -} - -int -spdk_ftl_write(struct spdk_ftl_dev *dev, struct spdk_io_channel *ch, uint64_t lba, size_t lba_cnt, - struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg) -{ - struct ftl_io *io; - - if (iov_cnt == 0) { - return -EINVAL; - } - - if (lba_cnt == 0) { - return -EINVAL; - } - - if (lba_cnt != ftl_iovec_num_blocks(iov, iov_cnt)) { - return -EINVAL; - } - - if (!dev->initialized) { - return -EBUSY; - } - - io = ftl_io_user_init(ch, lba, lba_cnt, iov, iov_cnt, cb_fn, cb_arg, FTL_IO_WRITE); - if (!io) { - return -ENOMEM; - } - - ftl_io_write(io); - - return 0; -} - -void -ftl_io_read(struct ftl_io *io) -{ - ftl_io_call_foreach_child(io, ftl_submit_read); -} - -int -spdk_ftl_read(struct spdk_ftl_dev *dev, struct spdk_io_channel *ch, uint64_t lba, size_t lba_cnt, - struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg) -{ - struct ftl_io *io; - - if (iov_cnt == 0) { - return -EINVAL; - } - - if (lba_cnt == 0) { - return -EINVAL; - } - - if (lba_cnt != ftl_iovec_num_blocks(iov, iov_cnt)) { - return -EINVAL; - } - - if (!dev->initialized) { - return -EBUSY; - } - - io = ftl_io_user_init(ch, lba, lba_cnt, iov, iov_cnt, cb_fn, cb_arg, FTL_IO_READ); - if (!io) { - return -ENOMEM; - } - - ftl_io_read(io); - return 0; -} - -static struct ftl_flush * -ftl_flush_init(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg) -{ - struct ftl_flush *flush; - - flush = calloc(1, sizeof(*flush)); - if (!flush) { - return NULL; - } - - flush->bmap = spdk_bit_array_create(FTL_BATCH_COUNT); - if (!flush->bmap) { - goto error; - } - - flush->dev = dev; - flush->cb.fn = cb_fn; - flush->cb.ctx = cb_arg; - - return flush; -error: - free(flush); - return NULL; -} - -static void -_ftl_flush(void *ctx) -{ - struct ftl_flush *flush = ctx; - struct spdk_ftl_dev *dev = flush->dev; - uint32_t i; - - /* Attach flush object to all non-empty batches */ - for (i = 0; i < FTL_BATCH_COUNT; ++i) { - if (dev->batch_array[i].num_entries > 0) { - spdk_bit_array_set(flush->bmap, i); - flush->num_req++; - } - } - - LIST_INSERT_HEAD(&dev->flush_list, flush, list_entry); - - /* If the write buffer was already empty, the flush can be completed right away */ - if (!flush->num_req) { - ftl_complete_flush(flush); - } -} - -int -ftl_flush_wbuf(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg) -{ - struct ftl_flush *flush; - - flush = ftl_flush_init(dev, cb_fn, cb_arg); - if (!flush) { - return -ENOMEM; - } - - spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_flush, flush); - return 0; -} - -int -spdk_ftl_flush(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg) -{ - if (!dev->initialized) { - return -EBUSY; - } - - return ftl_flush_wbuf(dev, cb_fn, cb_arg); -} - -bool -ftl_addr_is_written(struct ftl_band *band, struct ftl_addr addr) -{ - struct ftl_zone *zone = ftl_band_zone_from_addr(band, addr); - - return addr.offset < zone->info.write_pointer; -} - -static void ftl_process_media_event(struct spdk_ftl_dev *dev, struct spdk_bdev_media_event event); - -static void -_ftl_process_media_event(void *ctx) -{ - struct ftl_media_event *event = ctx; - struct spdk_ftl_dev *dev = event->dev; - - ftl_process_media_event(dev, event->event); - spdk_mempool_put(dev->media_events_pool, event); -} - -static void -ftl_process_media_event(struct spdk_ftl_dev *dev, struct spdk_bdev_media_event event) -{ - struct ftl_band *band; - struct ftl_addr addr = { .offset = event.offset }; - size_t block_off; - - if (!ftl_check_core_thread(dev)) { - struct ftl_media_event *media_event; - - media_event = spdk_mempool_get(dev->media_events_pool); - if (!media_event) { - SPDK_ERRLOG("Media event lost due to lack of memory"); - return; - } - - media_event->dev = dev; - media_event->event = event; - spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_process_media_event, - media_event); - return; - } - - band = ftl_band_from_addr(dev, addr); - block_off = ftl_band_block_offset_from_addr(band, addr); - - ftl_reloc_add(dev->reloc, band, block_off, event.num_blocks, 0, false); -} - -void -ftl_get_media_events(struct spdk_ftl_dev *dev) -{ -#define FTL_MAX_MEDIA_EVENTS 128 - struct spdk_bdev_media_event events[FTL_MAX_MEDIA_EVENTS]; - size_t num_events, i; - - if (!dev->initialized) { - return; - } - - do { - num_events = spdk_bdev_get_media_events(dev->base_bdev_desc, - events, FTL_MAX_MEDIA_EVENTS); - - for (i = 0; i < num_events; ++i) { - ftl_process_media_event(dev, events[i]); - } - - } while (num_events); -} - -int -ftl_io_channel_poll(void *arg) -{ - struct ftl_io_channel *ch = arg; - struct ftl_io *io; - TAILQ_HEAD(, ftl_io) retry_queue; - - if (TAILQ_EMPTY(&ch->write_cmpl_queue) && TAILQ_EMPTY(&ch->retry_queue)) { - return SPDK_POLLER_IDLE; - } - - while (!TAILQ_EMPTY(&ch->write_cmpl_queue)) { - io = TAILQ_FIRST(&ch->write_cmpl_queue); - TAILQ_REMOVE(&ch->write_cmpl_queue, io, ioch_entry); - ftl_io_complete(io); - } - - /* - * Create local copy of the retry queue to prevent from infinite retrying if IO will be - * inserted to the retry queue again - */ - TAILQ_INIT(&retry_queue); - TAILQ_SWAP(&ch->retry_queue, &retry_queue, ftl_io, ioch_entry); - - while (!TAILQ_EMPTY(&retry_queue)) { - io = TAILQ_FIRST(&retry_queue); - TAILQ_REMOVE(&retry_queue, io, ioch_entry); - if (io->type == FTL_IO_WRITE) { - ftl_io_write(io); - } else { - ftl_io_read(io); - } - } - - return SPDK_POLLER_BUSY; -} - -int -ftl_task_core(void *ctx) -{ - struct spdk_ftl_dev *dev = ctx; - bool busy; - - if (dev->halt) { - if (ftl_shutdown_complete(dev)) { - spdk_poller_unregister(&dev->core_poller); - return SPDK_POLLER_IDLE; - } - } - - busy = ftl_process_writes(dev) || ftl_process_relocs(dev); - - return busy ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; -} - -SPDK_LOG_REGISTER_COMPONENT(ftl_core) diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h deleted file mode 100644 index 1fb72e38b..000000000 --- a/lib/ftl/ftl_core.h +++ /dev/null @@ -1,524 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef FTL_CORE_H -#define FTL_CORE_H - -#include "spdk/stdinc.h" -#include "spdk/uuid.h" -#include "spdk/thread.h" -#include "spdk/util.h" -#include "spdk/log.h" -#include "spdk/likely.h" -#include "spdk/queue.h" -#include "spdk/ftl.h" -#include "spdk/bdev.h" -#include "spdk/bdev_zone.h" - -#include "ftl_addr.h" -#include "ftl_io.h" -#include "ftl_trace.h" - -#ifdef SPDK_CONFIG_PMDK -#include "libpmem.h" -#endif /* SPDK_CONFIG_PMDK */ - -struct spdk_ftl_dev; -struct ftl_band; -struct ftl_zone; -struct ftl_io; -struct ftl_restore; -struct ftl_wptr; -struct ftl_flush; -struct ftl_reloc; -struct ftl_anm_event; -struct ftl_band_flush; - -struct ftl_stats { - /* Number of writes scheduled directly by the user */ - uint64_t write_user; - - /* Total number of writes */ - uint64_t write_total; - - /* Traces */ - struct ftl_trace trace; - - /* Number of limits applied */ - uint64_t limits[SPDK_FTL_LIMIT_MAX]; -}; - -struct ftl_global_md { - /* Device instance */ - struct spdk_uuid uuid; - /* Size of the l2p table */ - uint64_t num_lbas; -}; - -struct ftl_nv_cache { - /* Write buffer cache bdev */ - struct spdk_bdev_desc *bdev_desc; - /* Write pointer */ - uint64_t current_addr; - /* Number of available blocks left */ - uint64_t num_available; - /* Maximum number of blocks */ - uint64_t num_data_blocks; - /* - * Phase of the current cycle of writes. Each time whole cache area is filled, the phase is - * advanced. Current phase is saved in every IO's metadata, as well as in the header saved - * in the first sector. By looking at the phase of each block, it's possible to find the - * oldest block and replay the order of the writes when recovering the data from the cache. - */ - unsigned int phase; - /* Indicates that the data can be written to the cache */ - bool ready; - /* Metadata pool */ - struct spdk_mempool *md_pool; - /* DMA buffer for writing the header */ - void *dma_buf; - /* Cache lock */ - pthread_spinlock_t lock; -}; - -struct ftl_batch { - /* Queue of write buffer entries, can reach up to xfer_size entries */ - TAILQ_HEAD(, ftl_wbuf_entry) entries; - /* Number of entries in the queue above */ - uint32_t num_entries; - /* Index within spdk_ftl_dev.batch_array */ - uint32_t index; - struct iovec *iov; - void *metadata; - TAILQ_ENTRY(ftl_batch) tailq; -}; - -struct spdk_ftl_dev { - /* Device instance */ - struct spdk_uuid uuid; - /* Device name */ - char *name; - /* Configuration */ - struct spdk_ftl_conf conf; - - /* Indicates the device is fully initialized */ - int initialized; - /* Indicates the device is about to be stopped */ - int halt; - /* Indicates the device is about to start stopping - use to handle multiple stop request */ - bool halt_started; - - /* Underlying device */ - struct spdk_bdev_desc *base_bdev_desc; - - /* Non-volatile write buffer cache */ - struct ftl_nv_cache nv_cache; - - /* LBA map memory pool */ - struct spdk_mempool *lba_pool; - - /* LBA map requests pool */ - struct spdk_mempool *lba_request_pool; - - /* Media management events pool */ - struct spdk_mempool *media_events_pool; - - /* Statistics */ - struct ftl_stats stats; - - /* Current sequence number */ - uint64_t seq; - - /* Array of bands */ - struct ftl_band *bands; - /* Number of operational bands */ - size_t num_bands; - /* Next write band */ - struct ftl_band *next_band; - /* Free band list */ - LIST_HEAD(, ftl_band) free_bands; - /* Closed bands list */ - LIST_HEAD(, ftl_band) shut_bands; - /* Number of free bands */ - size_t num_free; - - /* List of write pointers */ - LIST_HEAD(, ftl_wptr) wptr_list; - - /* Logical -> physical table */ - void *l2p; - /* Size of the l2p table */ - uint64_t num_lbas; - /* Size of pages mmapped for l2p, valid only for mapping on persistent memory */ - size_t l2p_pmem_len; - - /* Address size */ - size_t addr_len; - - /* Flush list */ - LIST_HEAD(, ftl_flush) flush_list; - /* List of band flush requests */ - LIST_HEAD(, ftl_band_flush) band_flush_list; - - /* Device specific md buffer */ - struct ftl_global_md global_md; - - /* Metadata size */ - size_t md_size; - void *md_buf; - - /* Transfer unit size */ - size_t xfer_size; - - /* Current user write limit */ - int limit; - - /* Inflight IO operations */ - uint32_t num_inflight; - - /* Manages data relocation */ - struct ftl_reloc *reloc; - - /* Thread on which the poller is running */ - struct spdk_thread *core_thread; - /* IO channel */ - struct spdk_io_channel *ioch; - /* Poller */ - struct spdk_poller *core_poller; - - /* IO channel array provides means for retrieving write buffer entries - * from their address stored in L2P. The address is divided into two - * parts - IO channel offset poining at specific IO channel (within this - * array) and entry offset pointing at specific entry within that IO - * channel. - */ - struct ftl_io_channel **ioch_array; - TAILQ_HEAD(, ftl_io_channel) ioch_queue; - uint64_t num_io_channels; - /* Value required to shift address of a write buffer entry to retrieve - * the IO channel it's part of. The other part of the address describes - * the offset of an entry within the IO channel's entry array. - */ - uint64_t ioch_shift; - - /* Write buffer batches */ -#define FTL_BATCH_COUNT 4096 - struct ftl_batch batch_array[FTL_BATCH_COUNT]; - /* Iovec buffer used by batches */ - struct iovec *iov_buf; - /* Batch currently being filled */ - struct ftl_batch *current_batch; - /* Full and ready to be sent batches. A batch is put on this queue in - * case it's already filled, but cannot be sent. - */ - TAILQ_HEAD(, ftl_batch) pending_batches; - TAILQ_HEAD(, ftl_batch) free_batches; - - /* Devices' list */ - STAILQ_ENTRY(spdk_ftl_dev) stailq; -}; - -struct ftl_nv_cache_header { - /* Version of the header */ - uint32_t version; - /* UUID of the FTL device */ - struct spdk_uuid uuid; - /* Size of the non-volatile cache (in blocks) */ - uint64_t size; - /* Contains the next address to be written after clean shutdown, invalid LBA otherwise */ - uint64_t current_addr; - /* Current phase */ - uint8_t phase; - /* Checksum of the header, needs to be last element */ - uint32_t checksum; -} __attribute__((packed)); - -struct ftl_media_event { - /* Owner */ - struct spdk_ftl_dev *dev; - /* Media event */ - struct spdk_bdev_media_event event; -}; - -typedef void (*ftl_restore_fn)(struct ftl_restore *, int, void *cb_arg); - -void ftl_apply_limits(struct spdk_ftl_dev *dev); -void ftl_io_read(struct ftl_io *io); -void ftl_io_write(struct ftl_io *io); -int ftl_flush_wbuf(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); -int ftl_current_limit(const struct spdk_ftl_dev *dev); -int ftl_invalidate_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr); -int ftl_task_core(void *ctx); -int ftl_task_read(void *ctx); -void ftl_process_anm_event(struct ftl_anm_event *event); -size_t ftl_tail_md_num_blocks(const struct spdk_ftl_dev *dev); -size_t ftl_tail_md_hdr_num_blocks(void); -size_t ftl_vld_map_num_blocks(const struct spdk_ftl_dev *dev); -size_t ftl_lba_map_num_blocks(const struct spdk_ftl_dev *dev); -size_t ftl_head_md_num_blocks(const struct spdk_ftl_dev *dev); -int ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb, void *cb_arg); -int ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg); -void ftl_restore_nv_cache(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg); -int ftl_band_set_direct_access(struct ftl_band *band, bool access); -bool ftl_addr_is_written(struct ftl_band *band, struct ftl_addr addr); -int ftl_flush_active_bands(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); -int ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, - spdk_bdev_io_completion_cb cb_fn, void *cb_arg); -int ftl_nv_cache_scrub(struct ftl_nv_cache *nv_cache, spdk_bdev_io_completion_cb cb_fn, - void *cb_arg); -void ftl_get_media_events(struct spdk_ftl_dev *dev); -int ftl_io_channel_poll(void *arg); -void ftl_evict_cache_entry(struct spdk_ftl_dev *dev, struct ftl_wbuf_entry *entry); -struct spdk_io_channel *ftl_get_io_channel(const struct spdk_ftl_dev *dev); -struct ftl_io_channel *ftl_io_channel_get_ctx(struct spdk_io_channel *ioch); - - -#define ftl_to_addr(address) \ - (struct ftl_addr) { .offset = (uint64_t)(address) } - -#define ftl_to_addr_packed(address) \ - (struct ftl_addr) { .pack.offset = (uint32_t)(address) } - -static inline struct spdk_thread * -ftl_get_core_thread(const struct spdk_ftl_dev *dev) -{ - return dev->core_thread; -} - -static inline size_t -ftl_get_num_bands(const struct spdk_ftl_dev *dev) -{ - return dev->num_bands; -} - -static inline size_t -ftl_get_num_punits(const struct spdk_ftl_dev *dev) -{ - return spdk_bdev_get_optimal_open_zones(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); -} - -static inline size_t -ftl_get_num_zones(const struct spdk_ftl_dev *dev) -{ - return ftl_get_num_bands(dev) * ftl_get_num_punits(dev); -} - -static inline size_t -ftl_get_num_blocks_in_zone(const struct spdk_ftl_dev *dev) -{ - return spdk_bdev_get_zone_size(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); -} - -static inline uint64_t -ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev) -{ - return ftl_get_num_punits(dev) * ftl_get_num_blocks_in_zone(dev); -} - -static inline uint64_t -ftl_addr_get_zone_slba(const struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - return addr.offset -= (addr.offset % ftl_get_num_blocks_in_zone(dev)); -} - -static inline uint64_t -ftl_addr_get_band(const struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - return addr.offset / ftl_get_num_blocks_in_band(dev); -} - -static inline uint64_t -ftl_addr_get_punit(const struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - return (addr.offset / ftl_get_num_blocks_in_zone(dev)) % ftl_get_num_punits(dev); -} - -static inline uint64_t -ftl_addr_get_zone_offset(const struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - return addr.offset % ftl_get_num_blocks_in_zone(dev); -} - -static inline size_t -ftl_vld_map_size(const struct spdk_ftl_dev *dev) -{ - return (size_t)spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), CHAR_BIT); -} - -static inline int -ftl_addr_packed(const struct spdk_ftl_dev *dev) -{ - return dev->addr_len < 32; -} - -static inline void -ftl_l2p_lba_persist(const struct spdk_ftl_dev *dev, uint64_t lba) -{ -#ifdef SPDK_CONFIG_PMDK - size_t ftl_addr_size = ftl_addr_packed(dev) ? 4 : 8; - pmem_persist((char *)dev->l2p + (lba * ftl_addr_size), ftl_addr_size); -#else /* SPDK_CONFIG_PMDK */ - SPDK_ERRLOG("Libpmem not available, cannot flush l2p to pmem\n"); - assert(0); -#endif /* SPDK_CONFIG_PMDK */ -} - -static inline int -ftl_addr_invalid(struct ftl_addr addr) -{ - return addr.offset == ftl_to_addr(FTL_ADDR_INVALID).offset; -} - -static inline int -ftl_addr_cached(struct ftl_addr addr) -{ - return !ftl_addr_invalid(addr) && addr.cached; -} - -static inline struct ftl_addr -ftl_addr_to_packed(const struct spdk_ftl_dev *dev, struct ftl_addr addr) -{ - struct ftl_addr p = {}; - - if (ftl_addr_invalid(addr)) { - p = ftl_to_addr_packed(FTL_ADDR_INVALID); - } else if (ftl_addr_cached(addr)) { - p.pack.cached = 1; - p.pack.cache_offset = (uint32_t) addr.cache_offset; - } else { - p.pack.offset = (uint32_t) addr.offset; - } - - return p; -} - -static inline struct ftl_addr -ftl_addr_from_packed(const struct spdk_ftl_dev *dev, struct ftl_addr p) -{ - struct ftl_addr addr = {}; - - if (p.pack.offset == (uint32_t)FTL_ADDR_INVALID) { - addr = ftl_to_addr(FTL_ADDR_INVALID); - } else if (p.pack.cached) { - addr.cached = 1; - addr.cache_offset = p.pack.cache_offset; - } else { - addr = p; - } - - return addr; -} - -#define _ftl_l2p_set(l2p, off, val, bits) \ - __atomic_store_n(((uint##bits##_t *)(l2p)) + (off), val, __ATOMIC_SEQ_CST) - -#define _ftl_l2p_set32(l2p, off, val) \ - _ftl_l2p_set(l2p, off, val, 32) - -#define _ftl_l2p_set64(l2p, off, val) \ - _ftl_l2p_set(l2p, off, val, 64) - -#define _ftl_l2p_get(l2p, off, bits) \ - __atomic_load_n(((uint##bits##_t *)(l2p)) + (off), __ATOMIC_SEQ_CST) - -#define _ftl_l2p_get32(l2p, off) \ - _ftl_l2p_get(l2p, off, 32) - -#define _ftl_l2p_get64(l2p, off) \ - _ftl_l2p_get(l2p, off, 64) - -#define ftl_addr_cmp(p1, p2) \ - ((p1).offset == (p2).offset) - -static inline void -ftl_l2p_set(struct spdk_ftl_dev *dev, uint64_t lba, struct ftl_addr addr) -{ - assert(dev->num_lbas > lba); - - if (ftl_addr_packed(dev)) { - _ftl_l2p_set32(dev->l2p, lba, ftl_addr_to_packed(dev, addr).offset); - } else { - _ftl_l2p_set64(dev->l2p, lba, addr.offset); - } - - if (dev->l2p_pmem_len != 0) { - ftl_l2p_lba_persist(dev, lba); - } -} - -static inline struct ftl_addr -ftl_l2p_get(struct spdk_ftl_dev *dev, uint64_t lba) -{ - assert(dev->num_lbas > lba); - - if (ftl_addr_packed(dev)) { - return ftl_addr_from_packed(dev, ftl_to_addr_packed( - _ftl_l2p_get32(dev->l2p, lba))); - } else { - return ftl_to_addr(_ftl_l2p_get64(dev->l2p, lba)); - } -} - -static inline bool -ftl_dev_has_nv_cache(const struct spdk_ftl_dev *dev) -{ - return dev->nv_cache.bdev_desc != NULL; -} - -#define FTL_NV_CACHE_HEADER_VERSION (1) -#define FTL_NV_CACHE_DATA_OFFSET (1) -#define FTL_NV_CACHE_PHASE_OFFSET (62) -#define FTL_NV_CACHE_PHASE_COUNT (4) -#define FTL_NV_CACHE_PHASE_MASK (3ULL << FTL_NV_CACHE_PHASE_OFFSET) -#define FTL_NV_CACHE_LBA_INVALID (FTL_LBA_INVALID & ~FTL_NV_CACHE_PHASE_MASK) - -static inline bool -ftl_nv_cache_phase_is_valid(unsigned int phase) -{ - return phase > 0 && phase <= 3; -} - -static inline unsigned int -ftl_nv_cache_next_phase(unsigned int current) -{ - static const unsigned int phases[] = { 0, 2, 3, 1 }; - assert(ftl_nv_cache_phase_is_valid(current)); - return phases[current]; -} - -static inline unsigned int -ftl_nv_cache_prev_phase(unsigned int current) -{ - static const unsigned int phases[] = { 0, 3, 1, 2 }; - assert(ftl_nv_cache_phase_is_valid(current)); - return phases[current]; -} - -static inline uint64_t -ftl_nv_cache_pack_lba(uint64_t lba, unsigned int phase) -{ - assert(ftl_nv_cache_phase_is_valid(phase)); - return (lba & ~FTL_NV_CACHE_PHASE_MASK) | ((uint64_t)phase << FTL_NV_CACHE_PHASE_OFFSET); -} - -static inline void -ftl_nv_cache_unpack_lba(uint64_t in_lba, uint64_t *out_lba, unsigned int *phase) -{ - *out_lba = in_lba & ~FTL_NV_CACHE_PHASE_MASK; - *phase = (in_lba & FTL_NV_CACHE_PHASE_MASK) >> FTL_NV_CACHE_PHASE_OFFSET; - - /* If the phase is invalid the block wasn't written yet, so treat the LBA as invalid too */ - if (!ftl_nv_cache_phase_is_valid(*phase) || *out_lba == FTL_NV_CACHE_LBA_INVALID) { - *out_lba = FTL_LBA_INVALID; - } -} - -static inline bool -ftl_is_append_supported(const struct spdk_ftl_dev *dev) -{ - return dev->conf.use_append; -} - -#endif /* FTL_CORE_H */ diff --git a/lib/ftl/ftl_debug.c b/lib/ftl/ftl_debug.c deleted file mode 100644 index 9638d56a8..000000000 --- a/lib/ftl/ftl_debug.c +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/log.h" -#include "spdk/ftl.h" -#include "ftl_debug.h" -#include "ftl_band.h" - -#if defined(DEBUG) -#if defined(FTL_META_DEBUG) - -static const char *ftl_band_state_str[] = { - "free", - "prep", - "opening", - "open", - "full", - "closing", - "closed", - "max" -}; - -bool -ftl_band_validate_md(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_lba_map *lba_map = &band->lba_map; - struct ftl_addr addr_md, addr_l2p; - size_t i, size, seg_off; - bool valid = true; - - size = ftl_get_num_blocks_in_band(dev); - - pthread_spin_lock(&lba_map->lock); - for (i = 0; i < size; ++i) { - if (!spdk_bit_array_get(lba_map->vld, i)) { - continue; - } - - seg_off = i / FTL_NUM_LBA_IN_BLOCK; - if (lba_map->segments[seg_off] != FTL_LBA_MAP_SEG_CACHED) { - continue; - } - - addr_md = ftl_band_addr_from_block_offset(band, i); - addr_l2p = ftl_l2p_get(dev, lba_map->map[i]); - - if (addr_l2p.cached) { - continue; - } - - if (addr_l2p.offset != addr_md.offset) { - valid = false; - break; - } - - } - - pthread_spin_unlock(&lba_map->lock); - - return valid; -} - -void -ftl_dev_dump_bands(struct spdk_ftl_dev *dev) -{ - size_t i; - - if (!dev->bands) { - return; - } - - ftl_debug("Bands validity:\n"); - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - if (dev->bands[i].state == FTL_BAND_STATE_FREE && - dev->bands[i].wr_cnt == 0) { - continue; - } - - if (!dev->bands[i].num_zones) { - ftl_debug(" Band %3zu: all zones are offline\n", i + 1); - continue; - } - - ftl_debug(" Band %3zu: %8zu / %zu \tnum_zones: %zu \twr_cnt: %"PRIu64"\tmerit:" - "%10.3f\tstate: %s\n", - i + 1, dev->bands[i].lba_map.num_vld, - ftl_band_user_blocks(&dev->bands[i]), - dev->bands[i].num_zones, - dev->bands[i].wr_cnt, - dev->bands[i].merit, - ftl_band_state_str[dev->bands[i].state]); - } -} - -#endif /* defined(FTL_META_DEBUG) */ - -#if defined(FTL_DUMP_STATS) - -void -ftl_dev_dump_stats(const struct spdk_ftl_dev *dev) -{ - size_t i, total = 0; - char uuid[SPDK_UUID_STRING_LEN]; - double waf; - const char *limits[] = { - [SPDK_FTL_LIMIT_CRIT] = "crit", - [SPDK_FTL_LIMIT_HIGH] = "high", - [SPDK_FTL_LIMIT_LOW] = "low", - [SPDK_FTL_LIMIT_START] = "start" - }; - - if (!dev->bands) { - return; - } - - /* Count the number of valid LBAs */ - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - total += dev->bands[i].lba_map.num_vld; - } - - waf = (double)dev->stats.write_total / (double)dev->stats.write_user; - - spdk_uuid_fmt_lower(uuid, sizeof(uuid), &dev->uuid); - ftl_debug("\n"); - ftl_debug("device UUID: %s\n", uuid); - ftl_debug("total valid LBAs: %zu\n", total); - ftl_debug("total writes: %"PRIu64"\n", dev->stats.write_total); - ftl_debug("user writes: %"PRIu64"\n", dev->stats.write_user); - ftl_debug("WAF: %.4lf\n", waf); - ftl_debug("limits:\n"); - for (i = 0; i < SPDK_FTL_LIMIT_MAX; ++i) { - ftl_debug(" %5s: %"PRIu64"\n", limits[i], dev->stats.limits[i]); - } -} - -#endif /* defined(FTL_DUMP_STATS) */ -#endif /* defined(DEBUG) */ diff --git a/lib/ftl/ftl_debug.h b/lib/ftl/ftl_debug.h deleted file mode 100644 index 8d7dc5f64..000000000 --- a/lib/ftl/ftl_debug.h +++ /dev/null @@ -1,45 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef FTL_DEBUG_H -#define FTL_DEBUG_H - -#include "ftl_addr.h" -#include "ftl_band.h" -#include "ftl_core.h" - -#if defined(DEBUG) -/* Debug flags - enabled when defined */ -#define FTL_META_DEBUG 1 -#define FTL_DUMP_STATS 1 - -#define ftl_debug(msg, ...) \ - SPDK_ERRLOG(msg, ## __VA_ARGS__) -#else -#define ftl_debug(msg, ...) -#endif - -static inline const char * -ftl_addr2str(struct ftl_addr addr, char *buf, size_t size) -{ - snprintf(buf, size, "(%"PRIu64")", addr.offset); - return buf; -} - -#if defined(FTL_META_DEBUG) -bool ftl_band_validate_md(struct ftl_band *band); -void ftl_dev_dump_bands(struct spdk_ftl_dev *dev); -#else -#define ftl_band_validate_md(band) -#define ftl_dev_dump_bands(dev) -#endif - -#if defined(FTL_DUMP_STATS) -void ftl_dev_dump_stats(const struct spdk_ftl_dev *dev); -#else -#define ftl_dev_dump_stats(dev) -#endif - -#endif /* FTL_DEBUG_H */ diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c deleted file mode 100644 index da709e644..000000000 --- a/lib/ftl/ftl_init.c +++ /dev/null @@ -1,1660 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" -#include "spdk/nvme.h" -#include "spdk/thread.h" -#include "spdk/string.h" -#include "spdk/likely.h" -#include "spdk/log.h" -#include "spdk/ftl.h" -#include "spdk/likely.h" -#include "spdk/string.h" -#include "spdk/bdev_zone.h" -#include "spdk/bdev_module.h" -#include "spdk/config.h" - -#include "ftl_core.h" -#include "ftl_io.h" -#include "ftl_reloc.h" -#include "ftl_band.h" -#include "ftl_debug.h" - -#ifdef SPDK_CONFIG_PMDK -#include "libpmem.h" -#endif /* SPDK_CONFIG_PMDK */ - -#define FTL_CORE_RING_SIZE 4096 -#define FTL_INIT_TIMEOUT 30 -#define FTL_NSID 1 -#define FTL_ZONE_INFO_COUNT 64 - -/* Dummy bdev module used to to claim bdevs. */ -static struct spdk_bdev_module g_ftl_bdev_module = { - .name = "ftl_lib", -}; - -struct ftl_dev_init_ctx { - /* Owner */ - struct spdk_ftl_dev *dev; - /* Initial arguments */ - struct spdk_ftl_dev_init_opts opts; - /* IO channel for zone info retrieving */ - struct spdk_io_channel *ioch; - /* Buffer for reading zone info */ - struct spdk_bdev_zone_info info[FTL_ZONE_INFO_COUNT]; - /* Currently read zone */ - size_t zone_id; - /* User's callback */ - spdk_ftl_init_fn cb_fn; - /* Callback's argument */ - void *cb_arg; - /* Thread to call the callback on */ - struct spdk_thread *thread; - /* Poller to check if the device has been destroyed/initialized */ - struct spdk_poller *poller; - /* Status to return for halt completion callback */ - int halt_complete_status; -}; - -static STAILQ_HEAD(, spdk_ftl_dev) g_ftl_queue = STAILQ_HEAD_INITIALIZER(g_ftl_queue); -static pthread_mutex_t g_ftl_queue_lock = PTHREAD_MUTEX_INITIALIZER; -static const struct spdk_ftl_conf g_default_conf = { - .limits = { - /* 5 free bands / 0 % host writes */ - [SPDK_FTL_LIMIT_CRIT] = { .thld = 5, .limit = 0 }, - /* 10 free bands / 5 % host writes */ - [SPDK_FTL_LIMIT_HIGH] = { .thld = 10, .limit = 5 }, - /* 20 free bands / 40 % host writes */ - [SPDK_FTL_LIMIT_LOW] = { .thld = 20, .limit = 40 }, - /* 40 free bands / 100 % host writes - defrag starts running */ - [SPDK_FTL_LIMIT_START] = { .thld = 40, .limit = 100 }, - }, - /* 10 percent valid blocks */ - .invalid_thld = 10, - /* 20% spare blocks */ - .lba_rsvd = 20, - /* 6M write buffer per each IO channel */ - .write_buffer_size = 6 * 1024 * 1024, - /* 90% band fill threshold */ - .band_thld = 90, - /* Max 32 IO depth per band relocate */ - .max_reloc_qdepth = 32, - /* Max 3 active band relocates */ - .max_active_relocs = 3, - /* IO pool size per user thread (this should be adjusted to thread IO qdepth) */ - .user_io_pool_size = 2048, - /* - * If clear ftl will return error when restoring after a dirty shutdown - * If set, last band will be padded, ftl will restore based only on closed bands - this - * will result in lost data after recovery. - */ - .allow_open_bands = false, - .max_io_channels = 128, - .nv_cache = { - /* Maximum number of concurrent requests */ - .max_request_cnt = 2048, - /* Maximum number of blocks per request */ - .max_request_size = 16, - } -}; - -static int -ftl_band_init_md(struct ftl_band *band) -{ - struct ftl_lba_map *lba_map = &band->lba_map; - int rc; - - lba_map->vld = spdk_bit_array_create(ftl_get_num_blocks_in_band(band->dev)); - if (!lba_map->vld) { - return -ENOMEM; - } - - rc = pthread_spin_init(&lba_map->lock, PTHREAD_PROCESS_PRIVATE); - if (rc) { - spdk_bit_array_free(&lba_map->vld); - return rc; - } - ftl_band_md_clear(band); - return 0; -} - -static int -ftl_check_conf(const struct spdk_ftl_dev *dev, const struct spdk_ftl_conf *conf) -{ - size_t i; - - if (conf->invalid_thld >= 100) { - return -1; - } - if (conf->lba_rsvd >= 100) { - return -1; - } - if (conf->lba_rsvd == 0) { - return -1; - } - if (conf->write_buffer_size == 0) { - return -1; - } - if (conf->write_buffer_size % FTL_BLOCK_SIZE != 0) { - return -1; - } - - for (i = 0; i < SPDK_FTL_LIMIT_MAX; ++i) { - if (conf->limits[i].limit > 100) { - return -1; - } - } - - return 0; -} - -static int -ftl_dev_init_bands(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band, *pband; - unsigned int i; - int rc = 0; - - LIST_INIT(&dev->free_bands); - LIST_INIT(&dev->shut_bands); - - dev->num_free = 0; - dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); - if (!dev->bands) { - return -1; - } - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - band = &dev->bands[i]; - band->id = i; - band->dev = dev; - band->state = FTL_BAND_STATE_CLOSED; - - if (LIST_EMPTY(&dev->shut_bands)) { - LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); - } else { - LIST_INSERT_AFTER(pband, band, list_entry); - } - pband = band; - - CIRCLEQ_INIT(&band->zones); - band->zone_buf = calloc(ftl_get_num_punits(dev), sizeof(*band->zone_buf)); - if (!band->zone_buf) { - SPDK_ERRLOG("Failed to allocate block state table for band: [%u]\n", i); - rc = -1; - break; - } - - rc = ftl_band_init_md(band); - if (rc) { - SPDK_ERRLOG("Failed to initialize metadata structures for band [%u]\n", i); - break; - } - - band->reloc_bitmap = spdk_bit_array_create(ftl_get_num_bands(dev)); - if (!band->reloc_bitmap) { - SPDK_ERRLOG("Failed to allocate band relocation bitmap\n"); - break; - } - } - - return rc; -} - -static void -ftl_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) -{ - struct spdk_ftl_dev *dev = event_ctx; - - switch (type) { - case SPDK_BDEV_EVENT_REMOVE: - assert(0); - break; - case SPDK_BDEV_EVENT_MEDIA_MANAGEMENT: - assert(bdev == spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); - ftl_get_media_events(dev); - default: - break; - } -} - -static int -ftl_dev_init_nv_cache(struct spdk_ftl_dev *dev, const char *bdev_name) -{ - struct spdk_bdev *bdev; - struct spdk_ftl_conf *conf = &dev->conf; - struct ftl_nv_cache *nv_cache = &dev->nv_cache; - char pool_name[128]; - int rc; - - if (!bdev_name) { - return 0; - } - - bdev = spdk_bdev_get_by_name(bdev_name); - if (!bdev) { - SPDK_ERRLOG("Unable to find bdev: %s\n", bdev_name); - return -1; - } - - if (spdk_bdev_open_ext(bdev_name, true, ftl_bdev_event_cb, - dev, &nv_cache->bdev_desc)) { - SPDK_ERRLOG("Unable to open bdev: %s\n", bdev_name); - return -1; - } - - if (spdk_bdev_module_claim_bdev(bdev, nv_cache->bdev_desc, &g_ftl_bdev_module)) { - spdk_bdev_close(nv_cache->bdev_desc); - nv_cache->bdev_desc = NULL; - SPDK_ERRLOG("Unable to claim bdev %s\n", bdev_name); - return -1; - } - - SPDK_INFOLOG(ftl_init, "Using %s as write buffer cache\n", - spdk_bdev_get_name(bdev)); - - if (spdk_bdev_get_block_size(bdev) != FTL_BLOCK_SIZE) { - SPDK_ERRLOG("Unsupported block size (%d)\n", spdk_bdev_get_block_size(bdev)); - return -1; - } - - if (!spdk_bdev_is_md_separate(bdev)) { - SPDK_ERRLOG("Bdev %s doesn't support separate metadata buffer IO\n", - spdk_bdev_get_name(bdev)); - return -1; - } - - if (spdk_bdev_get_md_size(bdev) < sizeof(uint64_t)) { - SPDK_ERRLOG("Bdev's %s metadata is too small (%"PRIu32")\n", - spdk_bdev_get_name(bdev), spdk_bdev_get_md_size(bdev)); - return -1; - } - - if (spdk_bdev_get_dif_type(bdev) != SPDK_DIF_DISABLE) { - SPDK_ERRLOG("Unsupported DIF type used by bdev %s\n", - spdk_bdev_get_name(bdev)); - return -1; - } - - /* The cache needs to be capable of storing at least two full bands. This requirement comes - * from the fact that cache works as a protection against power loss, so before the data - * inside the cache can be overwritten, the band it's stored on has to be closed. Plus one - * extra block is needed to store the header. - */ - if (spdk_bdev_get_num_blocks(bdev) < ftl_get_num_blocks_in_band(dev) * 2 + 1) { - SPDK_ERRLOG("Insufficient number of blocks for write buffer cache (available: %" - PRIu64", required: %"PRIu64")\n", spdk_bdev_get_num_blocks(bdev), - ftl_get_num_blocks_in_band(dev) * 2 + 1); - return -1; - } - - rc = snprintf(pool_name, sizeof(pool_name), "ftl-nvpool-%p", dev); - if (rc < 0 || rc >= 128) { - return -1; - } - - nv_cache->md_pool = spdk_mempool_create(pool_name, conf->nv_cache.max_request_cnt, - spdk_bdev_get_md_size(bdev) * - conf->nv_cache.max_request_size, - SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, - SPDK_ENV_SOCKET_ID_ANY); - if (!nv_cache->md_pool) { - SPDK_ERRLOG("Failed to initialize non-volatile cache metadata pool\n"); - return -1; - } - - nv_cache->dma_buf = spdk_dma_zmalloc(FTL_BLOCK_SIZE, spdk_bdev_get_buf_align(bdev), NULL); - if (!nv_cache->dma_buf) { - SPDK_ERRLOG("Memory allocation failure\n"); - return -1; - } - - if (pthread_spin_init(&nv_cache->lock, PTHREAD_PROCESS_PRIVATE)) { - SPDK_ERRLOG("Failed to initialize cache lock\n"); - return -1; - } - - nv_cache->current_addr = FTL_NV_CACHE_DATA_OFFSET; - nv_cache->num_data_blocks = spdk_bdev_get_num_blocks(bdev) - 1; - nv_cache->num_available = nv_cache->num_data_blocks; - nv_cache->ready = false; - - return 0; -} - -void -spdk_ftl_conf_init_defaults(struct spdk_ftl_conf *conf) -{ - *conf = g_default_conf; -} - -static void -ftl_lba_map_request_ctor(struct spdk_mempool *mp, void *opaque, void *obj, unsigned obj_idx) -{ - struct ftl_lba_map_request *request = obj; - struct spdk_ftl_dev *dev = opaque; - - request->segments = spdk_bit_array_create(spdk_divide_round_up( - ftl_get_num_blocks_in_band(dev), FTL_NUM_LBA_IN_BLOCK)); -} - -static int -ftl_init_media_events_pool(struct spdk_ftl_dev *dev) -{ - char pool_name[128]; - int rc; - - rc = snprintf(pool_name, sizeof(pool_name), "ftl-media-%p", dev); - if (rc < 0 || rc >= (int)sizeof(pool_name)) { - SPDK_ERRLOG("Failed to create media pool name\n"); - return -1; - } - - dev->media_events_pool = spdk_mempool_create(pool_name, 1024, - sizeof(struct ftl_media_event), - SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, - SPDK_ENV_SOCKET_ID_ANY); - if (!dev->media_events_pool) { - SPDK_ERRLOG("Failed to create media events pool\n"); - return -1; - } - - return 0; -} - -static int -ftl_init_lba_map_pools(struct spdk_ftl_dev *dev) -{ -#define POOL_NAME_LEN 128 - char pool_name[POOL_NAME_LEN]; - int rc; - - rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "ftl-lba-pool"); - if (rc < 0 || rc >= POOL_NAME_LEN) { - return -ENAMETOOLONG; - } - - /* We need to reserve at least 2 buffers for band close / open sequence - * alone, plus additional (8) buffers for handling write errors. - * TODO: This memory pool is utilized only by core thread - it introduce - * unnecessary overhead and should be replaced by different data structure. - */ - dev->lba_pool = spdk_mempool_create(pool_name, 2 + 8, - ftl_lba_map_pool_elem_size(dev), - SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, - SPDK_ENV_SOCKET_ID_ANY); - if (!dev->lba_pool) { - return -ENOMEM; - } - - rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "ftl-lbareq-pool"); - if (rc < 0 || rc >= POOL_NAME_LEN) { - return -ENAMETOOLONG; - } - - dev->lba_request_pool = spdk_mempool_create_ctor(pool_name, - dev->conf.max_reloc_qdepth * dev->conf.max_active_relocs, - sizeof(struct ftl_lba_map_request), - SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, - SPDK_ENV_SOCKET_ID_ANY, - ftl_lba_map_request_ctor, - dev); - if (!dev->lba_request_pool) { - return -ENOMEM; - } - - return 0; -} - -static void -ftl_init_wptr_list(struct spdk_ftl_dev *dev) -{ - LIST_INIT(&dev->wptr_list); - LIST_INIT(&dev->flush_list); - LIST_INIT(&dev->band_flush_list); -} - -static size_t -ftl_dev_band_max_seq(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band; - size_t seq = 0; - - LIST_FOREACH(band, &dev->shut_bands, list_entry) { - if (band->seq > seq) { - seq = band->seq; - } - } - - return seq; -} - -static void -_ftl_init_bands_state(void *ctx) -{ - struct ftl_band *band, *temp_band; - struct spdk_ftl_dev *dev = ctx; - - dev->seq = ftl_dev_band_max_seq(dev); - - LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { - if (!band->lba_map.num_vld) { - ftl_band_set_state(band, FTL_BAND_STATE_FREE); - } - } - - ftl_reloc_resume(dev->reloc); - /* Clear the limit applications as they're incremented incorrectly by */ - /* the initialization code */ - memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); -} - -static int -ftl_init_num_free_bands(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band; - int cnt = 0; - - LIST_FOREACH(band, &dev->shut_bands, list_entry) { - if (band->num_zones && !band->lba_map.num_vld) { - cnt++; - } - } - return cnt; -} - -static int -ftl_init_bands_state(struct spdk_ftl_dev *dev) -{ - /* TODO: Should we abort initialization or expose read only device */ - /* if there is no free bands? */ - /* If we abort initialization should we depend on condition that */ - /* we have no free bands or should we have some minimal number of */ - /* free bands? */ - if (!ftl_init_num_free_bands(dev)) { - return -1; - } - - spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_init_bands_state, dev); - return 0; -} - -static void -_ftl_dev_init_core_thread(void *ctx) -{ - struct spdk_ftl_dev *dev = ctx; - - dev->core_poller = SPDK_POLLER_REGISTER(ftl_task_core, dev, 0); - if (!dev->core_poller) { - SPDK_ERRLOG("Unable to register core poller\n"); - assert(0); - } - - dev->ioch = spdk_get_io_channel(dev); -} - -static int -ftl_dev_init_core_thread(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) -{ - if (!opts->core_thread) { - return -1; - } - - dev->core_thread = opts->core_thread; - - spdk_thread_send_msg(opts->core_thread, _ftl_dev_init_core_thread, dev); - return 0; -} - -static int -ftl_dev_l2p_alloc_pmem(struct spdk_ftl_dev *dev, size_t l2p_size, const char *l2p_path) -{ -#ifdef SPDK_CONFIG_PMDK - int is_pmem; - - if ((dev->l2p = pmem_map_file(l2p_path, 0, - 0, 0, &dev->l2p_pmem_len, &is_pmem)) == NULL) { - SPDK_ERRLOG("Failed to mmap l2p_path\n"); - return -1; - } - - if (!is_pmem) { - SPDK_NOTICELOG("l2p_path mapped on non-pmem device\n"); - } - - if (dev->l2p_pmem_len < l2p_size) { - SPDK_ERRLOG("l2p_path file is too small\n"); - return -1; - } - - pmem_memset_persist(dev->l2p, FTL_ADDR_INVALID, l2p_size); - - return 0; -#else /* SPDK_CONFIG_PMDK */ - SPDK_ERRLOG("Libpmem not available, cannot use pmem l2p_path\n"); - return -1; -#endif /* SPDK_CONFIG_PMDK */ -} - -static int -ftl_dev_l2p_alloc_dram(struct spdk_ftl_dev *dev, size_t l2p_size) -{ - dev->l2p = malloc(l2p_size); - if (!dev->l2p) { - SPDK_ERRLOG("Failed to allocate l2p table\n"); - return -1; - } - - memset(dev->l2p, FTL_ADDR_INVALID, l2p_size); - - return 0; -} - -static int -ftl_dev_l2p_alloc(struct spdk_ftl_dev *dev) -{ - size_t addr_size = dev->addr_len >= 32 ? 8 : 4; - size_t l2p_size = dev->num_lbas * addr_size; - const char *l2p_path = dev->conf.l2p_path; - - if (dev->num_lbas == 0) { - SPDK_ERRLOG("Invalid l2p table size\n"); - return -1; - } - - if (dev->l2p) { - SPDK_ERRLOG("L2p table already allocated\n"); - return -1; - } - - dev->l2p_pmem_len = 0; - if (l2p_path) { - return ftl_dev_l2p_alloc_pmem(dev, l2p_size, l2p_path); - } else { - return ftl_dev_l2p_alloc_dram(dev, l2p_size); - } -} - -static void -ftl_dev_free_init_ctx(struct ftl_dev_init_ctx *init_ctx) -{ - if (!init_ctx) { - return; - } - - if (init_ctx->ioch) { - spdk_put_io_channel(init_ctx->ioch); - } - - free(init_ctx); -} - -static void -ftl_call_init_complete_cb(void *ctx) -{ - struct ftl_dev_init_ctx *init_ctx = ctx; - struct spdk_ftl_dev *dev = init_ctx->dev; - - if (init_ctx->cb_fn != NULL) { - init_ctx->cb_fn(dev, init_ctx->cb_arg, 0); - } - - ftl_dev_free_init_ctx(init_ctx); -} - -static void -ftl_init_complete(struct ftl_dev_init_ctx *init_ctx) -{ - struct spdk_ftl_dev *dev = init_ctx->dev; - - pthread_mutex_lock(&g_ftl_queue_lock); - STAILQ_INSERT_HEAD(&g_ftl_queue, dev, stailq); - pthread_mutex_unlock(&g_ftl_queue_lock); - - dev->initialized = 1; - - spdk_thread_send_msg(init_ctx->thread, ftl_call_init_complete_cb, init_ctx); -} - -static void -ftl_init_fail_cb(struct spdk_ftl_dev *dev, void *ctx, int status) -{ - struct ftl_dev_init_ctx *init_ctx = ctx; - - if (init_ctx->cb_fn != NULL) { - init_ctx->cb_fn(NULL, init_ctx->cb_arg, -ENODEV); - } - - ftl_dev_free_init_ctx(init_ctx); -} - -static int ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_init_fn cb_fn, void *cb_arg, - struct spdk_thread *thread); - -static void -ftl_init_fail(struct ftl_dev_init_ctx *init_ctx) -{ - if (ftl_dev_free(init_ctx->dev, ftl_init_fail_cb, init_ctx, init_ctx->thread)) { - SPDK_ERRLOG("Unable to free the device\n"); - assert(0); - } -} - -static void -ftl_write_nv_cache_md_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_dev_init_ctx *init_ctx = cb_arg; - struct spdk_ftl_dev *dev = init_ctx->dev; - - spdk_bdev_free_io(bdev_io); - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Writing non-volatile cache's metadata header failed\n"); - ftl_init_fail(init_ctx); - return; - } - - dev->nv_cache.ready = true; - ftl_init_complete(init_ctx); -} - -static void -ftl_clear_nv_cache_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_dev_init_ctx *init_ctx = cb_arg; - struct spdk_ftl_dev *dev = init_ctx->dev; - struct ftl_nv_cache *nv_cache = &dev->nv_cache; - - spdk_bdev_free_io(bdev_io); - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Unable to clear the non-volatile cache bdev\n"); - ftl_init_fail(init_ctx); - return; - } - - nv_cache->phase = 1; - if (ftl_nv_cache_write_header(nv_cache, false, ftl_write_nv_cache_md_cb, init_ctx)) { - SPDK_ERRLOG("Unable to write non-volatile cache metadata header\n"); - ftl_init_fail(init_ctx); - } -} - -static void -_ftl_nv_cache_scrub(void *ctx) -{ - struct ftl_dev_init_ctx *init_ctx = ctx; - struct spdk_ftl_dev *dev = init_ctx->dev; - int rc; - - rc = ftl_nv_cache_scrub(&dev->nv_cache, ftl_clear_nv_cache_cb, init_ctx); - - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to clear the non-volatile cache bdev: %s\n", - spdk_strerror(-rc)); - ftl_init_fail(init_ctx); - } -} - -static int -ftl_setup_initial_state(struct ftl_dev_init_ctx *init_ctx) -{ - struct spdk_ftl_dev *dev = init_ctx->dev; - struct spdk_ftl_conf *conf = &dev->conf; - size_t i; - - spdk_uuid_generate(&dev->uuid); - - dev->num_lbas = 0; - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - dev->num_lbas += ftl_band_num_usable_blocks(&dev->bands[i]); - } - - dev->num_lbas = (dev->num_lbas * (100 - conf->lba_rsvd)) / 100; - - if (ftl_dev_l2p_alloc(dev)) { - SPDK_ERRLOG("Unable to init l2p table\n"); - return -1; - } - - if (ftl_init_bands_state(dev)) { - SPDK_ERRLOG("Unable to finish the initialization\n"); - return -1; - } - - if (!ftl_dev_has_nv_cache(dev)) { - ftl_init_complete(init_ctx); - } else { - spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_nv_cache_scrub, init_ctx); - } - - return 0; -} - -static void -ftl_restore_nv_cache_cb(struct ftl_restore *restore, int status, void *cb_arg) -{ - struct ftl_dev_init_ctx *init_ctx = cb_arg; - - if (spdk_unlikely(status != 0)) { - SPDK_ERRLOG("Failed to restore the non-volatile cache state\n"); - ftl_init_fail(init_ctx); - return; - } - - ftl_init_complete(init_ctx); -} - -static void -ftl_restore_device_cb(struct ftl_restore *restore, int status, void *cb_arg) -{ - struct ftl_dev_init_ctx *init_ctx = cb_arg; - struct spdk_ftl_dev *dev = init_ctx->dev; - - if (status) { - SPDK_ERRLOG("Failed to restore the device from the SSD\n"); - ftl_init_fail(init_ctx); - return; - } - - if (ftl_init_bands_state(dev)) { - SPDK_ERRLOG("Unable to finish the initialization\n"); - ftl_init_fail(init_ctx); - return; - } - - if (!ftl_dev_has_nv_cache(dev)) { - ftl_init_complete(init_ctx); - return; - } - - ftl_restore_nv_cache(restore, ftl_restore_nv_cache_cb, init_ctx); -} - -static void -ftl_restore_md_cb(struct ftl_restore *restore, int status, void *cb_arg) -{ - struct ftl_dev_init_ctx *init_ctx = cb_arg; - - if (status) { - SPDK_ERRLOG("Failed to restore the metadata from the SSD\n"); - goto error; - } - - /* After the metadata is read it should be possible to allocate the L2P */ - if (ftl_dev_l2p_alloc(init_ctx->dev)) { - SPDK_ERRLOG("Failed to allocate the L2P\n"); - goto error; - } - - if (ftl_restore_device(restore, ftl_restore_device_cb, init_ctx)) { - SPDK_ERRLOG("Failed to start device restoration from the SSD\n"); - goto error; - } - - return; -error: - ftl_init_fail(init_ctx); -} - -static int -ftl_restore_state(struct ftl_dev_init_ctx *init_ctx) -{ - struct spdk_ftl_dev *dev = init_ctx->dev; - - dev->uuid = init_ctx->opts.uuid; - - if (ftl_restore_md(dev, ftl_restore_md_cb, init_ctx)) { - SPDK_ERRLOG("Failed to start metadata restoration from the SSD\n"); - return -1; - } - - return 0; -} - -static void -ftl_dev_update_bands(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band, *temp_band; - size_t i; - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - band = &dev->bands[i]; - band->tail_md_addr = ftl_band_tail_md_addr(band); - } - - /* Remove band from shut_bands list to prevent further processing */ - /* if all blocks on this band are bad */ - LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { - if (!band->num_zones) { - dev->num_bands--; - LIST_REMOVE(band, list_entry); - } - } -} - -static void -ftl_dev_init_state(struct ftl_dev_init_ctx *init_ctx) -{ - struct spdk_ftl_dev *dev = init_ctx->dev; - - ftl_dev_update_bands(dev); - - if (ftl_dev_init_core_thread(dev, &init_ctx->opts)) { - SPDK_ERRLOG("Unable to initialize device thread\n"); - ftl_init_fail(init_ctx); - return; - } - - if (init_ctx->opts.mode & SPDK_FTL_MODE_CREATE) { - if (ftl_setup_initial_state(init_ctx)) { - SPDK_ERRLOG("Failed to setup initial state of the device\n"); - ftl_init_fail(init_ctx); - return; - } - } else { - if (ftl_restore_state(init_ctx)) { - SPDK_ERRLOG("Unable to restore device's state from the SSD\n"); - ftl_init_fail(init_ctx); - return; - } - } -} - -static void ftl_dev_get_zone_info(struct ftl_dev_init_ctx *init_ctx); - -static void -ftl_dev_get_zone_info_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_dev_init_ctx *init_ctx = cb_arg; - struct spdk_ftl_dev *dev = init_ctx->dev; - struct ftl_band *band; - struct ftl_zone *zone; - struct ftl_addr addr; - size_t i, zones_left, num_zones; - - spdk_bdev_free_io(bdev_io); - - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Unable to read zone info for zone id: %"PRIu64"\n", init_ctx->zone_id); - ftl_init_fail(init_ctx); - return; - } - - zones_left = ftl_get_num_zones(dev) - (init_ctx->zone_id / ftl_get_num_blocks_in_zone(dev)); - num_zones = spdk_min(zones_left, FTL_ZONE_INFO_COUNT); - - for (i = 0; i < num_zones; ++i) { - addr.offset = init_ctx->info[i].zone_id; - band = &dev->bands[ftl_addr_get_band(dev, addr)]; - zone = &band->zone_buf[ftl_addr_get_punit(dev, addr)]; - zone->info = init_ctx->info[i]; - - /* TODO: add support for zone capacity less than zone size */ - if (zone->info.capacity != ftl_get_num_blocks_in_zone(dev)) { - zone->info.state = SPDK_BDEV_ZONE_STATE_OFFLINE; - SPDK_ERRLOG("Zone capacity is not equal zone size for " - "zone id: %"PRIu64"\n", init_ctx->zone_id); - } - - /* Set write pointer to the last block plus one for zone in full state */ - if (zone->info.state == SPDK_BDEV_ZONE_STATE_FULL) { - zone->info.write_pointer = zone->info.zone_id + zone->info.capacity; - } - - if (zone->info.state != SPDK_BDEV_ZONE_STATE_OFFLINE) { - band->num_zones++; - CIRCLEQ_INSERT_TAIL(&band->zones, zone, circleq); - } - } - - init_ctx->zone_id = init_ctx->zone_id + num_zones * ftl_get_num_blocks_in_zone(dev); - - ftl_dev_get_zone_info(init_ctx); -} - -static void -ftl_dev_get_zone_info(struct ftl_dev_init_ctx *init_ctx) -{ - struct spdk_ftl_dev *dev = init_ctx->dev; - size_t zones_left, num_zones; - int rc; - - zones_left = ftl_get_num_zones(dev) - (init_ctx->zone_id / ftl_get_num_blocks_in_zone(dev)); - if (zones_left == 0) { - ftl_dev_init_state(init_ctx); - return; - } - - num_zones = spdk_min(zones_left, FTL_ZONE_INFO_COUNT); - - rc = spdk_bdev_get_zone_info(dev->base_bdev_desc, init_ctx->ioch, - init_ctx->zone_id, num_zones, init_ctx->info, - ftl_dev_get_zone_info_cb, init_ctx); - - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to read zone info for zone id: %"PRIu64"\n", init_ctx->zone_id); - ftl_init_fail(init_ctx); - } -} - -static int -ftl_dev_init_zones(struct ftl_dev_init_ctx *init_ctx) -{ - struct spdk_ftl_dev *dev = init_ctx->dev; - - init_ctx->zone_id = 0; - init_ctx->ioch = spdk_bdev_get_io_channel(dev->base_bdev_desc); - if (!init_ctx->ioch) { - SPDK_ERRLOG("Failed to get base bdev IO channel\n"); - return -1; - } - - ftl_dev_get_zone_info(init_ctx); - - return 0; -} - -struct _ftl_io_channel { - struct ftl_io_channel *ioch; -}; - -struct ftl_io_channel * -ftl_io_channel_get_ctx(struct spdk_io_channel *ioch) -{ - struct _ftl_io_channel *_ioch = spdk_io_channel_get_ctx(ioch); - - return _ioch->ioch; -} - -static void -ftl_io_channel_register(void *ctx) -{ - struct ftl_io_channel *ioch = ctx; - struct spdk_ftl_dev *dev = ioch->dev; - uint32_t ioch_index; - - for (ioch_index = 0; ioch_index < dev->conf.max_io_channels; ++ioch_index) { - if (dev->ioch_array[ioch_index] == NULL) { - dev->ioch_array[ioch_index] = ioch; - ioch->index = ioch_index; - break; - } - } - - assert(ioch_index < dev->conf.max_io_channels); - TAILQ_INSERT_TAIL(&dev->ioch_queue, ioch, tailq); -} - -static int -ftl_io_channel_init_wbuf(struct ftl_io_channel *ioch) -{ - struct spdk_ftl_dev *dev = ioch->dev; - struct ftl_wbuf_entry *entry; - uint32_t i; - int rc; - - ioch->num_entries = dev->conf.write_buffer_size / FTL_BLOCK_SIZE; - ioch->wbuf_entries = calloc(ioch->num_entries, sizeof(*ioch->wbuf_entries)); - if (ioch->wbuf_entries == NULL) { - SPDK_ERRLOG("Failed to allocate write buffer entry array\n"); - return -1; - } - - ioch->qdepth_limit = ioch->num_entries; - ioch->wbuf_payload = spdk_zmalloc(dev->conf.write_buffer_size, FTL_BLOCK_SIZE, NULL, - SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); - if (ioch->wbuf_payload == NULL) { - SPDK_ERRLOG("Failed to allocate write buffer payload\n"); - goto error_entries; - } - - ioch->free_queue = spdk_ring_create(SPDK_RING_TYPE_SP_SC, - spdk_align32pow2(ioch->num_entries + 1), - SPDK_ENV_SOCKET_ID_ANY); - if (ioch->free_queue == NULL) { - SPDK_ERRLOG("Failed to allocate free queue\n"); - goto error_payload; - } - - ioch->submit_queue = spdk_ring_create(SPDK_RING_TYPE_SP_SC, - spdk_align32pow2(ioch->num_entries + 1), - SPDK_ENV_SOCKET_ID_ANY); - if (ioch->submit_queue == NULL) { - SPDK_ERRLOG("Failed to allocate submit queue\n"); - goto error_free_queue; - } - - for (i = 0; i < ioch->num_entries; ++i) { - entry = &ioch->wbuf_entries[i]; - entry->payload = (char *)ioch->wbuf_payload + i * FTL_BLOCK_SIZE; - entry->ioch = ioch; - entry->index = i; - entry->addr.offset = FTL_ADDR_INVALID; - - rc = pthread_spin_init(&entry->lock, PTHREAD_PROCESS_PRIVATE); - if (rc != 0) { - SPDK_ERRLOG("Failed to initialize spinlock\n"); - goto error_spinlock; - } - - spdk_ring_enqueue(ioch->free_queue, (void **)&entry, 1, NULL); - } - - return 0; -error_spinlock: - for (; i > 0; --i) { - pthread_spin_destroy(&ioch->wbuf_entries[i - 1].lock); - } - - spdk_ring_free(ioch->submit_queue); -error_free_queue: - spdk_ring_free(ioch->free_queue); -error_payload: - spdk_free(ioch->wbuf_payload); -error_entries: - free(ioch->wbuf_entries); - - return -1; -} - -static int -ftl_io_channel_create_cb(void *io_device, void *ctx) -{ - struct spdk_ftl_dev *dev = io_device; - struct _ftl_io_channel *_ioch = ctx; - struct ftl_io_channel *ioch; - uint32_t num_io_channels; - char mempool_name[32]; - int rc; - - num_io_channels = __atomic_fetch_add(&dev->num_io_channels, 1, __ATOMIC_SEQ_CST); - if (num_io_channels >= dev->conf.max_io_channels) { - SPDK_ERRLOG("Reached maximum number of IO channels\n"); - __atomic_fetch_sub(&dev->num_io_channels, 1, __ATOMIC_SEQ_CST); - return -1; - } - - ioch = calloc(1, sizeof(*ioch)); - if (ioch == NULL) { - SPDK_ERRLOG("Failed to allocate IO channel\n"); - return -1; - } - - rc = snprintf(mempool_name, sizeof(mempool_name), "ftl_io_%p", ioch); - if (rc < 0 || rc >= (int)sizeof(mempool_name)) { - SPDK_ERRLOG("Failed to create IO channel pool name\n"); - free(ioch); - return -1; - } - - ioch->cache_ioch = NULL; - ioch->index = FTL_IO_CHANNEL_INDEX_INVALID; - ioch->dev = dev; - ioch->elem_size = sizeof(struct ftl_md_io); - ioch->io_pool = spdk_mempool_create(mempool_name, - dev->conf.user_io_pool_size, - ioch->elem_size, - 0, - SPDK_ENV_SOCKET_ID_ANY); - if (!ioch->io_pool) { - SPDK_ERRLOG("Failed to create IO channel's IO pool\n"); - free(ioch); - return -1; - } - - ioch->base_ioch = spdk_bdev_get_io_channel(dev->base_bdev_desc); - if (!ioch->base_ioch) { - SPDK_ERRLOG("Failed to create base bdev IO channel\n"); - goto fail_ioch; - } - - if (ftl_dev_has_nv_cache(dev)) { - ioch->cache_ioch = spdk_bdev_get_io_channel(dev->nv_cache.bdev_desc); - if (!ioch->cache_ioch) { - SPDK_ERRLOG("Failed to create cache IO channel\n"); - goto fail_cache; - } - } - - TAILQ_INIT(&ioch->write_cmpl_queue); - TAILQ_INIT(&ioch->retry_queue); - ioch->poller = SPDK_POLLER_REGISTER(ftl_io_channel_poll, ioch, 0); - if (!ioch->poller) { - SPDK_ERRLOG("Failed to register IO channel poller\n"); - goto fail_poller; - } - - if (ftl_io_channel_init_wbuf(ioch)) { - SPDK_ERRLOG("Failed to initialize IO channel's write buffer\n"); - goto fail_wbuf; - } - - _ioch->ioch = ioch; - - spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_io_channel_register, ioch); - - return 0; -fail_wbuf: - spdk_poller_unregister(&ioch->poller); -fail_poller: - if (ioch->cache_ioch) { - spdk_put_io_channel(ioch->cache_ioch); - } -fail_cache: - spdk_put_io_channel(ioch->base_ioch); -fail_ioch: - spdk_mempool_free(ioch->io_pool); - free(ioch); - - return -1; -} - -static void -ftl_io_channel_unregister(void *ctx) -{ - struct ftl_io_channel *ioch = ctx; - struct spdk_ftl_dev *dev = ioch->dev; - uint32_t i, num_io_channels __attribute__((unused)); - - assert(ioch->index < dev->conf.max_io_channels); - assert(dev->ioch_array[ioch->index] == ioch); - - dev->ioch_array[ioch->index] = NULL; - TAILQ_REMOVE(&dev->ioch_queue, ioch, tailq); - - num_io_channels = __atomic_fetch_sub(&dev->num_io_channels, 1, __ATOMIC_SEQ_CST); - assert(num_io_channels > 0); - - for (i = 0; i < ioch->num_entries; ++i) { - pthread_spin_destroy(&ioch->wbuf_entries[i].lock); - } - - spdk_mempool_free(ioch->io_pool); - spdk_ring_free(ioch->free_queue); - spdk_ring_free(ioch->submit_queue); - spdk_free(ioch->wbuf_payload); - free(ioch->wbuf_entries); - free(ioch); -} - -static void -_ftl_io_channel_destroy_cb(void *ctx) -{ - struct ftl_io_channel *ioch = ctx; - struct spdk_ftl_dev *dev = ioch->dev; - uint32_t i; - - /* Do not destroy the channel if some of its entries are still in use */ - if (spdk_ring_count(ioch->free_queue) != ioch->num_entries) { - spdk_thread_send_msg(spdk_get_thread(), _ftl_io_channel_destroy_cb, ctx); - return; - } - - /* Evict all valid entries from cache */ - for (i = 0; i < ioch->num_entries; ++i) { - ftl_evict_cache_entry(dev, &ioch->wbuf_entries[i]); - } - - spdk_poller_unregister(&ioch->poller); - - spdk_put_io_channel(ioch->base_ioch); - if (ioch->cache_ioch) { - spdk_put_io_channel(ioch->cache_ioch); - } - - ioch->base_ioch = NULL; - ioch->cache_ioch = NULL; - - spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_io_channel_unregister, ioch); -} - -static void -ftl_io_channel_destroy_cb(void *io_device, void *ctx) -{ - struct _ftl_io_channel *_ioch = ctx; - struct ftl_io_channel *ioch = _ioch->ioch; - - /* Mark the IO channel as being flush to force out any unwritten entries */ - ioch->flush = true; - - _ftl_io_channel_destroy_cb(ioch); -} - -static int -ftl_dev_init_io_channel(struct spdk_ftl_dev *dev) -{ - struct ftl_batch *batch; - uint32_t i; - - /* Align the IO channels to nearest power of 2 to allow for easy addr bit shift */ - dev->conf.max_io_channels = spdk_align32pow2(dev->conf.max_io_channels); - dev->ioch_shift = spdk_u32log2(dev->conf.max_io_channels); - - dev->ioch_array = calloc(dev->conf.max_io_channels, sizeof(*dev->ioch_array)); - if (!dev->ioch_array) { - SPDK_ERRLOG("Failed to allocate IO channel array\n"); - return -1; - } - - if (dev->md_size > 0) { - dev->md_buf = spdk_zmalloc(dev->md_size * dev->xfer_size * FTL_BATCH_COUNT, - dev->md_size, NULL, SPDK_ENV_LCORE_ID_ANY, - SPDK_MALLOC_DMA); - if (dev->md_buf == NULL) { - SPDK_ERRLOG("Failed to allocate metadata buffer\n"); - return -1; - } - } - - dev->iov_buf = calloc(FTL_BATCH_COUNT, dev->xfer_size * sizeof(struct iovec)); - if (!dev->iov_buf) { - SPDK_ERRLOG("Failed to allocate iovec buffer\n"); - return -1; - } - - TAILQ_INIT(&dev->free_batches); - TAILQ_INIT(&dev->pending_batches); - TAILQ_INIT(&dev->ioch_queue); - - for (i = 0; i < FTL_BATCH_COUNT; ++i) { - batch = &dev->batch_array[i]; - batch->iov = &dev->iov_buf[i * dev->xfer_size]; - batch->num_entries = 0; - batch->index = i; - TAILQ_INIT(&batch->entries); - if (dev->md_buf != NULL) { - batch->metadata = (char *)dev->md_buf + i * dev->xfer_size * dev->md_size; - } - - TAILQ_INSERT_TAIL(&dev->free_batches, batch, tailq); - } - - dev->num_io_channels = 0; - - spdk_io_device_register(dev, ftl_io_channel_create_cb, ftl_io_channel_destroy_cb, - sizeof(struct _ftl_io_channel), - NULL); - - return 0; -} - -static int -ftl_dev_init_base_bdev(struct spdk_ftl_dev *dev, const char *bdev_name) -{ - uint32_t block_size; - uint64_t num_blocks; - struct spdk_bdev *bdev; - - bdev = spdk_bdev_get_by_name(bdev_name); - if (!bdev) { - SPDK_ERRLOG("Unable to find bdev: %s\n", bdev_name); - return -1; - } - - if (!spdk_bdev_is_zoned(bdev)) { - SPDK_ERRLOG("Bdev doesn't support zone capabilities: %s\n", - spdk_bdev_get_name(bdev)); - return -1; - } - - if (spdk_bdev_open_ext(bdev_name, true, ftl_bdev_event_cb, - dev, &dev->base_bdev_desc)) { - SPDK_ERRLOG("Unable to open bdev: %s\n", bdev_name); - return -1; - } - - if (spdk_bdev_module_claim_bdev(bdev, dev->base_bdev_desc, &g_ftl_bdev_module)) { - spdk_bdev_close(dev->base_bdev_desc); - dev->base_bdev_desc = NULL; - SPDK_ERRLOG("Unable to claim bdev %s\n", bdev_name); - return -1; - } - - dev->xfer_size = spdk_bdev_get_write_unit_size(bdev); - dev->md_size = spdk_bdev_get_md_size(bdev); - - block_size = spdk_bdev_get_block_size(bdev); - if (block_size != FTL_BLOCK_SIZE) { - SPDK_ERRLOG("Unsupported block size (%"PRIu32")\n", block_size); - return -1; - } - - num_blocks = spdk_bdev_get_num_blocks(bdev); - if (num_blocks % ftl_get_num_punits(dev)) { - SPDK_ERRLOG("Unsupported geometry. Base bdev block count must be multiple " - "of optimal number of zones.\n"); - return -1; - } - - if (ftl_is_append_supported(dev) && - !spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZONE_APPEND)) { - SPDK_ERRLOG("Bdev doesn't support append: %s\n", - spdk_bdev_get_name(bdev)); - return -1; - } - - dev->num_bands = num_blocks / (ftl_get_num_punits(dev) * ftl_get_num_blocks_in_zone(dev)); - dev->addr_len = spdk_u64log2(num_blocks) + 1; - - return 0; -} - -static void -ftl_lba_map_request_dtor(struct spdk_mempool *mp, void *opaque, void *obj, unsigned obj_idx) -{ - struct ftl_lba_map_request *request = obj; - - spdk_bit_array_free(&request->segments); -} - -static void -ftl_release_bdev(struct spdk_bdev_desc *bdev_desc) -{ - if (!bdev_desc) { - return; - } - - spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(bdev_desc)); - spdk_bdev_close(bdev_desc); -} - -static void -ftl_dev_free_sync(struct spdk_ftl_dev *dev) -{ - struct spdk_ftl_dev *iter; - size_t i; - - if (!dev) { - return; - } - - pthread_mutex_lock(&g_ftl_queue_lock); - STAILQ_FOREACH(iter, &g_ftl_queue, stailq) { - if (iter == dev) { - STAILQ_REMOVE(&g_ftl_queue, dev, spdk_ftl_dev, stailq); - break; - } - } - pthread_mutex_unlock(&g_ftl_queue_lock); - - assert(LIST_EMPTY(&dev->wptr_list)); - assert(dev->current_batch == NULL); - - ftl_dev_dump_bands(dev); - ftl_dev_dump_stats(dev); - - if (dev->bands) { - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - free(dev->bands[i].zone_buf); - spdk_bit_array_free(&dev->bands[i].lba_map.vld); - spdk_bit_array_free(&dev->bands[i].reloc_bitmap); - } - } - - spdk_dma_free(dev->nv_cache.dma_buf); - - spdk_mempool_free(dev->lba_pool); - spdk_mempool_free(dev->nv_cache.md_pool); - spdk_mempool_free(dev->media_events_pool); - if (dev->lba_request_pool) { - spdk_mempool_obj_iter(dev->lba_request_pool, ftl_lba_map_request_dtor, NULL); - } - spdk_mempool_free(dev->lba_request_pool); - - ftl_reloc_free(dev->reloc); - - ftl_release_bdev(dev->nv_cache.bdev_desc); - ftl_release_bdev(dev->base_bdev_desc); - - spdk_free(dev->md_buf); - - assert(dev->num_io_channels == 0); - free(dev->ioch_array); - free(dev->iov_buf); - free(dev->name); - free(dev->bands); - if (dev->l2p_pmem_len != 0) { -#ifdef SPDK_CONFIG_PMDK - pmem_unmap(dev->l2p, dev->l2p_pmem_len); -#endif /* SPDK_CONFIG_PMDK */ - } else { - free(dev->l2p); - } - free((char *)dev->conf.l2p_path); - free(dev); -} - -int -spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *_opts, spdk_ftl_init_fn cb_fn, void *cb_arg) -{ - struct spdk_ftl_dev *dev; - struct spdk_ftl_dev_init_opts opts = *_opts; - struct ftl_dev_init_ctx *init_ctx = NULL; - int rc = -ENOMEM; - - dev = calloc(1, sizeof(*dev)); - if (!dev) { - return -ENOMEM; - } - - init_ctx = calloc(1, sizeof(*init_ctx)); - if (!init_ctx) { - goto fail_sync; - } - - init_ctx->dev = dev; - init_ctx->opts = *_opts; - init_ctx->cb_fn = cb_fn; - init_ctx->cb_arg = cb_arg; - init_ctx->thread = spdk_get_thread(); - - if (!opts.conf) { - opts.conf = &g_default_conf; - } - - if (!opts.base_bdev) { - SPDK_ERRLOG("Lack of underlying device in configuration\n"); - rc = -EINVAL; - goto fail_sync; - } - - dev->conf = *opts.conf; - dev->limit = SPDK_FTL_LIMIT_MAX; - - dev->name = strdup(opts.name); - if (!dev->name) { - SPDK_ERRLOG("Unable to set device name\n"); - goto fail_sync; - } - - if (ftl_dev_init_base_bdev(dev, opts.base_bdev)) { - SPDK_ERRLOG("Unsupported underlying device\n"); - goto fail_sync; - } - - if (opts.conf->l2p_path) { - dev->conf.l2p_path = strdup(opts.conf->l2p_path); - if (!dev->conf.l2p_path) { - rc = -ENOMEM; - goto fail_sync; - } - } - - /* In case of errors, we free all of the memory in ftl_dev_free_sync(), */ - /* so we don't have to clean up in each of the init functions. */ - if (ftl_check_conf(dev, opts.conf)) { - SPDK_ERRLOG("Invalid device configuration\n"); - goto fail_sync; - } - - if (ftl_init_lba_map_pools(dev)) { - SPDK_ERRLOG("Unable to init LBA map pools\n"); - goto fail_sync; - } - - if (ftl_init_media_events_pool(dev)) { - SPDK_ERRLOG("Unable to init media events pools\n"); - goto fail_sync; - } - - ftl_init_wptr_list(dev); - - if (ftl_dev_init_bands(dev)) { - SPDK_ERRLOG("Unable to initialize band array\n"); - goto fail_sync; - } - - if (ftl_dev_init_nv_cache(dev, opts.cache_bdev)) { - SPDK_ERRLOG("Unable to initialize persistent cache\n"); - goto fail_sync; - } - - dev->reloc = ftl_reloc_init(dev); - if (!dev->reloc) { - SPDK_ERRLOG("Unable to initialize reloc structures\n"); - goto fail_sync; - } - - if (ftl_dev_init_io_channel(dev)) { - SPDK_ERRLOG("Unable to initialize IO channels\n"); - goto fail_sync; - } - - if (ftl_dev_init_zones(init_ctx)) { - SPDK_ERRLOG("Failed to initialize zones\n"); - goto fail_async; - } - - return 0; -fail_sync: - ftl_dev_free_sync(dev); - ftl_dev_free_init_ctx(init_ctx); - return rc; -fail_async: - ftl_init_fail(init_ctx); - return 0; -} - -static void -_ftl_halt_defrag(void *arg) -{ - ftl_reloc_halt(((struct spdk_ftl_dev *)arg)->reloc); -} - -static void -ftl_halt_complete_cb(void *ctx) -{ - struct ftl_dev_init_ctx *fini_ctx = ctx; - struct spdk_ftl_dev *dev = fini_ctx->dev; - - /* Make sure core IO channel has already been released */ - if (dev->num_io_channels > 0) { - spdk_thread_send_msg(spdk_get_thread(), ftl_halt_complete_cb, ctx); - return; - } - - spdk_io_device_unregister(fini_ctx->dev, NULL); - - ftl_dev_free_sync(fini_ctx->dev); - if (fini_ctx->cb_fn != NULL) { - fini_ctx->cb_fn(NULL, fini_ctx->cb_arg, fini_ctx->halt_complete_status); - } - - ftl_dev_free_init_ctx(fini_ctx); -} - -static void -ftl_put_io_channel_cb(void *ctx) -{ - struct ftl_dev_init_ctx *fini_ctx = ctx; - struct spdk_ftl_dev *dev = fini_ctx->dev; - - spdk_put_io_channel(dev->ioch); - spdk_thread_send_msg(spdk_get_thread(), ftl_halt_complete_cb, ctx); -} - -static void -ftl_nv_cache_header_fini_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_dev_init_ctx *fini_ctx = cb_arg; - int rc = 0; - - spdk_bdev_free_io(bdev_io); - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Failed to write non-volatile cache metadata header\n"); - rc = -EIO; - } - - fini_ctx->halt_complete_status = rc; - spdk_thread_send_msg(fini_ctx->thread, ftl_put_io_channel_cb, fini_ctx); -} - -static int -ftl_halt_poller(void *ctx) -{ - struct ftl_dev_init_ctx *fini_ctx = ctx; - struct spdk_ftl_dev *dev = fini_ctx->dev; - - if (!dev->core_poller) { - spdk_poller_unregister(&fini_ctx->poller); - - if (ftl_dev_has_nv_cache(dev)) { - ftl_nv_cache_write_header(&dev->nv_cache, true, - ftl_nv_cache_header_fini_cb, fini_ctx); - } else { - fini_ctx->halt_complete_status = 0; - spdk_thread_send_msg(fini_ctx->thread, ftl_put_io_channel_cb, fini_ctx); - } - } - - return SPDK_POLLER_BUSY; -} - -static void -ftl_add_halt_poller(void *ctx) -{ - struct ftl_dev_init_ctx *fini_ctx = ctx; - struct spdk_ftl_dev *dev = fini_ctx->dev; - - dev->halt = 1; - - _ftl_halt_defrag(dev); - - assert(!fini_ctx->poller); - fini_ctx->poller = SPDK_POLLER_REGISTER(ftl_halt_poller, fini_ctx, 100); -} - -static int -ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_init_fn cb_fn, void *cb_arg, - struct spdk_thread *thread) -{ - struct ftl_dev_init_ctx *fini_ctx; - - if (dev->halt_started) { - dev->halt_started = true; - return -EBUSY; - } - - fini_ctx = calloc(1, sizeof(*fini_ctx)); - if (!fini_ctx) { - return -ENOMEM; - } - - fini_ctx->dev = dev; - fini_ctx->cb_fn = cb_fn; - fini_ctx->cb_arg = cb_arg; - fini_ctx->thread = thread; - - spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_add_halt_poller, fini_ctx); - return 0; -} - -int -spdk_ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_init_fn cb_fn, void *cb_arg) -{ - return ftl_dev_free(dev, cb_fn, cb_arg, spdk_get_thread()); -} - -SPDK_LOG_REGISTER_COMPONENT(ftl_init) diff --git a/lib/ftl/ftl_io.c b/lib/ftl/ftl_io.c deleted file mode 100644 index 690d1db11..000000000 --- a/lib/ftl/ftl_io.c +++ /dev/null @@ -1,536 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" -#include "spdk/ftl.h" -#include "spdk/likely.h" -#include "spdk/util.h" - -#include "ftl_io.h" -#include "ftl_core.h" -#include "ftl_band.h" -#include "ftl_debug.h" - -void -ftl_io_inc_req(struct ftl_io *io) -{ - struct ftl_band *band = io->band; - - if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) { - ftl_band_acquire_lba_map(band); - } - - __atomic_fetch_add(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST); - - ++io->req_cnt; -} - -void -ftl_io_dec_req(struct ftl_io *io) -{ - struct ftl_band *band = io->band; - unsigned long num_inflight __attribute__((unused)); - - if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) { - ftl_band_release_lba_map(band); - } - - num_inflight = __atomic_fetch_sub(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST); - - assert(num_inflight > 0); - assert(io->req_cnt > 0); - - --io->req_cnt; -} - -struct iovec * -ftl_io_iovec(struct ftl_io *io) -{ - return &io->iov[0]; -} - -uint64_t -ftl_io_get_lba(const struct ftl_io *io, size_t offset) -{ - assert(offset < io->num_blocks); - - if (io->flags & FTL_IO_VECTOR_LBA) { - return io->lba.vector[offset]; - } else { - return io->lba.single + offset; - } -} - -uint64_t -ftl_io_current_lba(const struct ftl_io *io) -{ - return ftl_io_get_lba(io, io->pos); -} - -void -ftl_io_advance(struct ftl_io *io, size_t num_blocks) -{ - struct iovec *iov = ftl_io_iovec(io); - size_t iov_blocks, block_left = num_blocks; - - io->pos += num_blocks; - - if (io->iov_cnt != 0) { - while (block_left > 0) { - assert(io->iov_pos < io->iov_cnt); - iov_blocks = iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE; - - if (io->iov_off + block_left < iov_blocks) { - io->iov_off += block_left; - break; - } - - assert(iov_blocks > io->iov_off); - block_left -= (iov_blocks - io->iov_off); - io->iov_off = 0; - io->iov_pos++; - } - } - - if (io->parent) { - ftl_io_advance(io->parent, num_blocks); - } -} - -size_t -ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt) -{ - size_t num_blocks = 0, i = 0; - - for (; i < iov_cnt; ++i) { - num_blocks += iov[i].iov_len / FTL_BLOCK_SIZE; - } - - return num_blocks; -} - -void * -ftl_io_iovec_addr(struct ftl_io *io) -{ - assert(io->iov_pos < io->iov_cnt); - assert(io->iov_off * FTL_BLOCK_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len); - - return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base + - io->iov_off * FTL_BLOCK_SIZE; -} - -size_t -ftl_io_iovec_len_left(struct ftl_io *io) -{ - struct iovec *iov = ftl_io_iovec(io); - return iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE - io->iov_off; -} - -static void -ftl_io_init_iovec(struct ftl_io *io, const struct iovec *iov, size_t iov_cnt, size_t iov_off, - size_t num_blocks) -{ - size_t offset = 0, num_left; - - io->iov_pos = 0; - io->iov_cnt = 0; - io->num_blocks = num_blocks; - - while (offset < num_blocks) { - assert(io->iov_cnt < FTL_IO_MAX_IOVEC && io->iov_cnt < iov_cnt); - - num_left = spdk_min(iov[io->iov_cnt].iov_len / FTL_BLOCK_SIZE - iov_off, - num_blocks); - io->iov[io->iov_cnt].iov_base = (char *)iov[io->iov_cnt].iov_base + - iov_off * FTL_BLOCK_SIZE; - io->iov[io->iov_cnt].iov_len = num_left * FTL_BLOCK_SIZE; - - offset += num_left; - io->iov_cnt++; - iov_off = 0; - } -} - -void -ftl_io_shrink_iovec(struct ftl_io *io, size_t num_blocks) -{ - size_t iov_off = 0, block_off = 0; - - assert(io->num_blocks >= num_blocks); - assert(io->pos == 0 && io->iov_pos == 0 && io->iov_off == 0); - - for (; iov_off < io->iov_cnt; ++iov_off) { - size_t num_iov = io->iov[iov_off].iov_len / FTL_BLOCK_SIZE; - size_t num_left = num_blocks - block_off; - - if (num_iov >= num_left) { - io->iov[iov_off].iov_len = num_left * FTL_BLOCK_SIZE; - io->iov_cnt = iov_off + 1; - io->num_blocks = num_blocks; - break; - } - - block_off += num_iov; - } -} - -static void -ftl_io_init(struct ftl_io *io, struct spdk_ftl_dev *dev, - ftl_io_fn fn, void *ctx, int flags, int type) -{ - io->flags |= flags | FTL_IO_INITIALIZED; - io->type = type; - io->dev = dev; - io->lba.single = FTL_LBA_INVALID; - io->addr.offset = FTL_ADDR_INVALID; - io->cb_fn = fn; - io->cb_ctx = ctx; - io->trace = ftl_trace_alloc_id(dev); -} - -struct ftl_io * -ftl_io_init_internal(const struct ftl_io_init_opts *opts) -{ - struct ftl_io *io = opts->io; - struct ftl_io *parent = opts->parent; - struct spdk_ftl_dev *dev = opts->dev; - const struct iovec *iov; - size_t iov_cnt, iov_off; - - if (!io) { - if (parent) { - io = ftl_io_alloc_child(parent); - } else { - io = ftl_io_alloc(ftl_get_io_channel(dev)); - } - - if (!io) { - return NULL; - } - } - - ftl_io_clear(io); - ftl_io_init(io, dev, opts->cb_fn, opts->cb_ctx, opts->flags | FTL_IO_INTERNAL, opts->type); - - io->batch = opts->batch; - io->band = opts->band; - io->md = opts->md; - io->iov = &io->iov_buf[0]; - - if (parent) { - if (parent->flags & FTL_IO_VECTOR_LBA) { - io->lba.vector = parent->lba.vector + parent->pos; - } else { - io->lba.single = parent->lba.single + parent->pos; - } - - iov = &parent->iov[parent->iov_pos]; - iov_cnt = parent->iov_cnt - parent->iov_pos; - iov_off = parent->iov_off; - } else { - iov = &opts->iovs[0]; - iov_cnt = opts->iovcnt; - iov_off = 0; - } - - /* Some requests (zone resets) do not use iovecs */ - if (iov_cnt > 0) { - ftl_io_init_iovec(io, iov, iov_cnt, iov_off, opts->num_blocks); - } - - if (opts->flags & FTL_IO_VECTOR_LBA) { - io->lba.vector = calloc(io->num_blocks, sizeof(uint64_t)); - if (!io->lba.vector) { - ftl_io_free(io); - return NULL; - } - } - - return io; -} - -struct ftl_io * -ftl_io_wbuf_init(struct spdk_ftl_dev *dev, struct ftl_addr addr, struct ftl_band *band, - struct ftl_batch *batch, ftl_io_fn cb) -{ - struct ftl_io *io; - struct ftl_io_init_opts opts = { - .dev = dev, - .io = NULL, - .batch = batch, - .band = band, - .size = sizeof(struct ftl_io), - .flags = 0, - .type = FTL_IO_WRITE, - .num_blocks = dev->xfer_size, - .cb_fn = cb, - .iovcnt = dev->xfer_size, - .md = batch->metadata, - }; - - memcpy(opts.iovs, batch->iov, sizeof(struct iovec) * dev->xfer_size); - - io = ftl_io_init_internal(&opts); - if (!io) { - return NULL; - } - - io->addr = addr; - - return io; -} - -struct ftl_io * -ftl_io_erase_init(struct ftl_band *band, size_t num_blocks, ftl_io_fn cb) -{ - struct ftl_io *io; - struct ftl_io_init_opts opts = { - .dev = band->dev, - .io = NULL, - .band = band, - .size = sizeof(struct ftl_io), - .flags = FTL_IO_PHYSICAL_MODE, - .type = FTL_IO_ERASE, - .num_blocks = 1, - .cb_fn = cb, - .iovcnt = 0, - .md = NULL, - }; - - io = ftl_io_init_internal(&opts); - if (!io) { - return NULL; - } - - io->num_blocks = num_blocks; - - return io; -} - -static void -_ftl_user_cb(struct ftl_io *io, void *arg, int status) -{ - io->user_fn(arg, status); -} - -struct ftl_io * -ftl_io_user_init(struct spdk_io_channel *_ioch, uint64_t lba, size_t num_blocks, struct iovec *iov, - size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type) -{ - struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(_ioch); - struct spdk_ftl_dev *dev = ioch->dev; - struct ftl_io *io; - - io = ftl_io_alloc(_ioch); - if (spdk_unlikely(!io)) { - return NULL; - } - - ftl_io_init(io, dev, _ftl_user_cb, cb_ctx, 0, type); - io->lba.single = lba; - io->user_fn = cb_fn; - io->iov = iov; - io->iov_cnt = iov_cnt; - io->num_blocks = num_blocks; - - ftl_trace_lba_io_init(io->dev, io); - return io; -} - -static void -_ftl_io_free(struct ftl_io *io) -{ - struct ftl_io_channel *ioch; - - assert(LIST_EMPTY(&io->children)); - - if (io->flags & FTL_IO_VECTOR_LBA) { - free(io->lba.vector); - } - - if (pthread_spin_destroy(&io->lock)) { - SPDK_ERRLOG("pthread_spin_destroy failed\n"); - } - - ioch = ftl_io_channel_get_ctx(io->ioch); - spdk_mempool_put(ioch->io_pool, io); -} - -static bool -ftl_io_remove_child(struct ftl_io *io) -{ - struct ftl_io *parent = io->parent; - bool parent_done; - - pthread_spin_lock(&parent->lock); - LIST_REMOVE(io, child_entry); - parent_done = parent->done && LIST_EMPTY(&parent->children); - parent->status = parent->status ? : io->status; - pthread_spin_unlock(&parent->lock); - - return parent_done; -} - -void -ftl_io_complete(struct ftl_io *io) -{ - struct ftl_io *parent = io->parent; - bool complete; - - io->flags &= ~FTL_IO_INITIALIZED; - - pthread_spin_lock(&io->lock); - complete = LIST_EMPTY(&io->children); - io->done = true; - pthread_spin_unlock(&io->lock); - - if (complete) { - if (io->cb_fn) { - io->cb_fn(io, io->cb_ctx, io->status); - } - - if (parent && ftl_io_remove_child(io)) { - ftl_io_complete(parent); - } - - _ftl_io_free(io); - } -} - -struct ftl_io * -ftl_io_alloc_child(struct ftl_io *parent) -{ - struct ftl_io *io; - - io = ftl_io_alloc(parent->ioch); - if (spdk_unlikely(!io)) { - return NULL; - } - - ftl_io_init(io, parent->dev, NULL, NULL, parent->flags, parent->type); - io->parent = parent; - - pthread_spin_lock(&parent->lock); - LIST_INSERT_HEAD(&parent->children, io, child_entry); - pthread_spin_unlock(&parent->lock); - - return io; -} - -void -ftl_io_fail(struct ftl_io *io, int status) -{ - io->status = status; - ftl_io_advance(io, io->num_blocks - io->pos); -} - -void * -ftl_io_get_md(const struct ftl_io *io) -{ - if (!io->md) { - return NULL; - } - - return (char *)io->md + io->pos * io->dev->md_size; -} - -struct ftl_io * -ftl_io_alloc(struct spdk_io_channel *ch) -{ - struct ftl_io *io; - struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(ch); - - io = spdk_mempool_get(ioch->io_pool); - if (!io) { - return NULL; - } - - memset(io, 0, ioch->elem_size); - io->ioch = ch; - - if (pthread_spin_init(&io->lock, PTHREAD_PROCESS_PRIVATE)) { - SPDK_ERRLOG("pthread_spin_init failed\n"); - spdk_mempool_put(ioch->io_pool, io); - return NULL; - } - - return io; -} - -void -ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb, void *ctx, int flags, int type) -{ - ftl_io_clear(io); - ftl_io_init(io, io->dev, cb, ctx, flags, type); -} - -void -ftl_io_clear(struct ftl_io *io) -{ - ftl_io_reset(io); - - io->flags = 0; - io->batch = NULL; - io->band = NULL; -} - -void -ftl_io_reset(struct ftl_io *io) -{ - io->req_cnt = io->pos = io->iov_pos = io->iov_off = 0; - io->done = false; -} - -void -ftl_io_free(struct ftl_io *io) -{ - struct ftl_io *parent; - - if (!io) { - return; - } - - parent = io->parent; - if (parent && ftl_io_remove_child(io)) { - ftl_io_complete(parent); - } - - _ftl_io_free(io); -} - -void -ftl_io_call_foreach_child(struct ftl_io *io, int (*callback)(struct ftl_io *)) -{ - struct ftl_io *child, *tmp; - - assert(!io->done); - - /* - * If the IO doesn't have any children, it means that it directly describes a request (i.e. - * all of the buffers, LBAs, etc. are filled). Otherwise the IO only groups together several - * requests and may be partially filled, so the callback needs to be called on all of its - * children instead. - */ - if (LIST_EMPTY(&io->children)) { - callback(io); - return; - } - - LIST_FOREACH_SAFE(child, &io->children, child_entry, tmp) { - int rc = callback(child); - if (rc) { - assert(rc != -EAGAIN); - ftl_io_fail(io, rc); - break; - } - } - - /* - * If all the callbacks were processed or an error occurred, treat this IO as completed. - * Multiple calls to ftl_io_call_foreach_child are not supported, resubmissions are supposed - * to be handled in the callback. - */ - ftl_io_complete(io); -} diff --git a/lib/ftl/ftl_io.h b/lib/ftl/ftl_io.h deleted file mode 100644 index c64ddffd8..000000000 --- a/lib/ftl/ftl_io.h +++ /dev/null @@ -1,323 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef FTL_IO_H -#define FTL_IO_H - -#include "spdk/stdinc.h" -#include "spdk/nvme.h" -#include "spdk/ftl.h" - -#include "ftl_addr.h" -#include "ftl_trace.h" - -struct spdk_ftl_dev; -struct ftl_band; -struct ftl_batch; -struct ftl_io; - -typedef int (*ftl_md_pack_fn)(struct ftl_band *); -typedef void (*ftl_io_fn)(struct ftl_io *, void *, int); - -/* IO flags */ -enum ftl_io_flags { - /* Indicates whether IO is already initialized */ - FTL_IO_INITIALIZED = (1 << 0), - /* Internal based IO (defrag, metadata etc.) */ - FTL_IO_INTERNAL = (1 << 1), - /* Indicates that the IO should not go through if there's */ - /* already another one scheduled to the same LBA */ - FTL_IO_WEAK = (1 << 2), - /* Indicates that the IO is used for padding */ - FTL_IO_PAD = (1 << 3), - /* The IO operates on metadata */ - FTL_IO_MD = (1 << 4), - /* Using physical instead of logical address */ - FTL_IO_PHYSICAL_MODE = (1 << 5), - /* Indicates that IO contains noncontiguous LBAs */ - FTL_IO_VECTOR_LBA = (1 << 6), - /* The IO is directed to non-volatile cache */ - FTL_IO_CACHE = (1 << 7), - /* Indicates that physical address should be taken from IO struct, */ - /* not assigned by wptr, only works if wptr is also in direct mode */ - FTL_IO_DIRECT_ACCESS = (1 << 8), - /* Bypass the non-volatile cache */ - FTL_IO_BYPASS_CACHE = (1 << 9), -}; - -enum ftl_io_type { - FTL_IO_READ, - FTL_IO_WRITE, - FTL_IO_ERASE, -}; - -#define FTL_IO_MAX_IOVEC 64 - -struct ftl_io_init_opts { - struct spdk_ftl_dev *dev; - - /* IO descriptor */ - struct ftl_io *io; - - /* Parent request */ - struct ftl_io *parent; - - /* Size of IO descriptor */ - size_t size; - - /* IO flags */ - int flags; - - /* IO type */ - enum ftl_io_type type; - - /* Transfer batch, set for IO going through the write buffer */ - struct ftl_batch *batch; - - /* Band to which the IO is directed */ - struct ftl_band *band; - - /* Number of logical blocks */ - size_t num_blocks; - - /* Data */ - struct iovec iovs[FTL_IO_MAX_IOVEC]; - int iovcnt; - - /* Metadata */ - void *md; - - /* Callback's function */ - ftl_io_fn cb_fn; - - /* Callback's context */ - void *cb_ctx; -}; - -struct ftl_io_channel; - -struct ftl_wbuf_entry { - /* IO channel that owns the write buffer entry */ - struct ftl_io_channel *ioch; - /* Data payload (single block) */ - void *payload; - /* Index within the IO channel's wbuf_entries array */ - uint32_t index; - uint32_t io_flags; - /* Points at the band the data is copied from. Only valid for internal - * requests coming from reloc. - */ - struct ftl_band *band; - /* Physical address of that particular block. Valid once the data has - * been written out. - */ - struct ftl_addr addr; - /* Logical block address */ - uint64_t lba; - - /* Trace ID of the requests the entry is part of */ - uint64_t trace; - - /* Indicates that the entry was written out and is still present in the - * L2P table. - */ - bool valid; - /* Lock that protects the entry from being evicted from the L2P */ - pthread_spinlock_t lock; - TAILQ_ENTRY(ftl_wbuf_entry) tailq; -}; - -#define FTL_IO_CHANNEL_INDEX_INVALID ((uint64_t)-1) - -struct ftl_io_channel { - /* Device */ - struct spdk_ftl_dev *dev; - /* IO pool element size */ - size_t elem_size; - /* Index within the IO channel array */ - uint64_t index; - /* IO pool */ - struct spdk_mempool *io_pool; - /* Underlying device IO channel */ - struct spdk_io_channel *base_ioch; - /* Persistent cache IO channel */ - struct spdk_io_channel *cache_ioch; - /* Poller used for completing write requests and retrying IO */ - struct spdk_poller *poller; - /* Write completion queue */ - TAILQ_HEAD(, ftl_io) write_cmpl_queue; - TAILQ_HEAD(, ftl_io) retry_queue; - TAILQ_ENTRY(ftl_io_channel) tailq; - - /* Array of write buffer entries */ - struct ftl_wbuf_entry *wbuf_entries; - /* Write buffer data payload */ - void *wbuf_payload; - /* Number of write buffer entries */ - uint32_t num_entries; - /* Write buffer queues */ - struct spdk_ring *free_queue; - struct spdk_ring *submit_queue; - /* Maximum number of concurrent user writes */ - uint32_t qdepth_limit; - /* Current number of concurrent user writes */ - uint32_t qdepth_current; - /* Means that the IO channel is being flushed */ - bool flush; -}; - -/* General IO descriptor */ -struct ftl_io { - /* Device */ - struct spdk_ftl_dev *dev; - - /* IO channel */ - struct spdk_io_channel *ioch; - - union { - /* LBA table */ - uint64_t *vector; - - /* First LBA */ - uint64_t single; - } lba; - - /* First block address */ - struct ftl_addr addr; - - /* Number of processed blocks */ - size_t pos; - - /* Number of blocks */ - size_t num_blocks; - - /* IO vector pointer */ - struct iovec *iov; - - /* IO vector buffer for internal requests */ - struct iovec iov_buf[FTL_IO_MAX_IOVEC]; - - /* Metadata */ - void *md; - - /* Number of IO vectors */ - size_t iov_cnt; - - /* Position within the iovec */ - size_t iov_pos; - - /* Offset within the iovec (in blocks) */ - size_t iov_off; - - /* Transfer batch (valid only for writes going through the write buffer) */ - struct ftl_batch *batch; - - /* Band this IO is being written to */ - struct ftl_band *band; - - /* Request status */ - int status; - - /* Number of split requests */ - size_t req_cnt; - - /* Callback's function */ - ftl_io_fn cb_fn; - - /* Callback's context */ - void *cb_ctx; - - /* User callback function */ - spdk_ftl_fn user_fn; - - /* Flags */ - int flags; - - /* IO type */ - enum ftl_io_type type; - - /* Done flag */ - bool done; - - /* Parent request */ - struct ftl_io *parent; - /* Child requests list */ - LIST_HEAD(, ftl_io) children; - /* Child list link */ - LIST_ENTRY(ftl_io) child_entry; - /* Children lock */ - pthread_spinlock_t lock; - - /* Trace group id */ - uint64_t trace; - - /* Used by retry and write completion queues */ - TAILQ_ENTRY(ftl_io) ioch_entry; -}; - -/* Metadata IO */ -struct ftl_md_io { - /* Parent IO structure */ - struct ftl_io io; - - /* Serialization/deserialization callback */ - ftl_md_pack_fn pack_fn; - - /* Callback's function */ - ftl_io_fn cb_fn; - - /* Callback's context */ - void *cb_ctx; -}; - -static inline bool -ftl_io_mode_physical(const struct ftl_io *io) -{ - return io->flags & FTL_IO_PHYSICAL_MODE; -} - -static inline bool -ftl_io_mode_logical(const struct ftl_io *io) -{ - return !ftl_io_mode_physical(io); -} - -static inline bool -ftl_io_done(const struct ftl_io *io) -{ - return io->req_cnt == 0 && io->pos == io->num_blocks; -} - -struct ftl_io *ftl_io_alloc(struct spdk_io_channel *ch); -struct ftl_io *ftl_io_alloc_child(struct ftl_io *parent); -void ftl_io_fail(struct ftl_io *io, int status); -void ftl_io_free(struct ftl_io *io); -struct ftl_io *ftl_io_init_internal(const struct ftl_io_init_opts *opts); -void ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb, - void *ctx, int flags, int type); -void ftl_io_clear(struct ftl_io *io); -void ftl_io_inc_req(struct ftl_io *io); -void ftl_io_dec_req(struct ftl_io *io); -struct iovec *ftl_io_iovec(struct ftl_io *io); -uint64_t ftl_io_current_lba(const struct ftl_io *io); -uint64_t ftl_io_get_lba(const struct ftl_io *io, size_t offset); -void ftl_io_advance(struct ftl_io *io, size_t num_blocks); -size_t ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt); -void *ftl_io_iovec_addr(struct ftl_io *io); -size_t ftl_io_iovec_len_left(struct ftl_io *io); -struct ftl_io *ftl_io_wbuf_init(struct spdk_ftl_dev *dev, struct ftl_addr addr, - struct ftl_band *band, struct ftl_batch *batch, ftl_io_fn cb); -struct ftl_io *ftl_io_erase_init(struct ftl_band *band, size_t num_blocks, ftl_io_fn cb); -struct ftl_io *ftl_io_user_init(struct spdk_io_channel *ioch, uint64_t lba, size_t num_blocks, - struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, - void *cb_arg, int type); -void *ftl_io_get_md(const struct ftl_io *io); -void ftl_io_complete(struct ftl_io *io); -void ftl_io_shrink_iovec(struct ftl_io *io, size_t num_blocks); -void ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status); -void ftl_io_reset(struct ftl_io *io); -void ftl_io_call_foreach_child(struct ftl_io *io, int (*callback)(struct ftl_io *)); - -#endif /* FTL_IO_H */ diff --git a/lib/ftl/ftl_reloc.c b/lib/ftl/ftl_reloc.c deleted file mode 100644 index 35fe74c72..000000000 --- a/lib/ftl/ftl_reloc.c +++ /dev/null @@ -1,834 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/likely.h" -#include "spdk/log.h" -#include "spdk/ftl.h" - -#include "ftl_reloc.h" -#include "ftl_core.h" -#include "ftl_io.h" -#include "ftl_band.h" -#include "ftl_debug.h" - -/* Maximum active reloc moves */ -#define FTL_RELOC_MAX_MOVES 256 - -struct ftl_reloc; -struct ftl_band_reloc; - -enum ftl_reloc_move_state { - FTL_RELOC_STATE_READ_LBA_MAP, - FTL_RELOC_STATE_READ, - FTL_RELOC_STATE_WRITE, -}; - -enum ftl_band_reloc_state { - FTL_BAND_RELOC_STATE_INACTIVE, - FTL_BAND_RELOC_STATE_PENDING, - FTL_BAND_RELOC_STATE_ACTIVE, - FTL_BAND_RELOC_STATE_HIGH_PRIO -}; - -struct ftl_reloc_move { - struct ftl_band_reloc *breloc; - - /* Start addr */ - struct ftl_addr addr; - - /* Number of logical blocks */ - size_t num_blocks; - - /* Data buffer */ - void *data; - - /* Move state (read lba_map, read, write) */ - enum ftl_reloc_move_state state; - - /* IO associated with move */ - struct ftl_io *io; - - STAILQ_ENTRY(ftl_reloc_move) entry; -}; - -struct ftl_band_reloc { - struct ftl_reloc *parent; - - /* Band being relocated */ - struct ftl_band *band; - - /* Number of logical blocks to be relocated */ - size_t num_blocks; - - /* Bitmap of logical blocks to be relocated */ - struct spdk_bit_array *reloc_map; - - /* State of the band reloc */ - enum ftl_band_reloc_state state; - - /* The band is being defragged */ - bool defrag; - - /* Reloc map iterator */ - struct { - /* Array of zone offsets */ - size_t *zone_offset; - - /* Current zone */ - size_t zone_current; - } iter; - - /* Number of outstanding moves */ - size_t num_outstanding; - - /* Pool of move objects */ - struct ftl_reloc_move *moves; - - /* Move queue */ - STAILQ_HEAD(, ftl_reloc_move) move_queue; - - TAILQ_ENTRY(ftl_band_reloc) entry; -}; - -struct ftl_reloc { - /* Device associated with relocate */ - struct spdk_ftl_dev *dev; - - /* Indicates relocate is about to halt */ - bool halt; - - /* Maximum number of IOs per band */ - size_t max_qdepth; - - /* Maximum number of active band relocates */ - size_t max_active; - - /* Maximum transfer size (in logical blocks) per single IO */ - size_t xfer_size; - /* Number of bands being defragged */ - size_t num_defrag_bands; - - /* Array of band relocates */ - struct ftl_band_reloc *brelocs; - - /* Number of active/priority band relocates */ - size_t num_active; - - /* Priority band relocates queue */ - TAILQ_HEAD(, ftl_band_reloc) prio_queue; - - /* Active band relocates queue */ - TAILQ_HEAD(, ftl_band_reloc) active_queue; - - /* Pending band relocates queue */ - TAILQ_HEAD(, ftl_band_reloc) pending_queue; -}; - -bool -ftl_reloc_is_defrag_active(const struct ftl_reloc *reloc) -{ - return reloc->num_defrag_bands > 0; -} - -static size_t -ftl_reloc_iter_zone_offset(struct ftl_band_reloc *breloc) -{ - size_t zone = breloc->iter.zone_current; - - return breloc->iter.zone_offset[zone]; -} - -static size_t -ftl_reloc_iter_zone_done(struct ftl_band_reloc *breloc) -{ - size_t num_blocks = ftl_get_num_blocks_in_zone(breloc->parent->dev); - - return ftl_reloc_iter_zone_offset(breloc) == num_blocks; -} - -static void -ftl_reloc_clr_block(struct ftl_band_reloc *breloc, size_t block_off) -{ - if (!spdk_bit_array_get(breloc->reloc_map, block_off)) { - return; - } - - spdk_bit_array_clear(breloc->reloc_map, block_off); - assert(breloc->num_blocks); - breloc->num_blocks--; -} - -static void -ftl_reloc_read_lba_map_cb(struct ftl_io *io, void *arg, int status) -{ - struct ftl_reloc_move *move = arg; - struct ftl_band_reloc *breloc = move->breloc; - - breloc->num_outstanding--; - assert(status == 0); - move->state = FTL_RELOC_STATE_WRITE; - STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); -} - -static int -ftl_reloc_read_lba_map(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) -{ - struct ftl_band *band = breloc->band; - - breloc->num_outstanding++; - return ftl_band_read_lba_map(band, ftl_band_block_offset_from_addr(band, move->addr), - move->num_blocks, ftl_reloc_read_lba_map_cb, move); -} - -static void -ftl_reloc_prep(struct ftl_band_reloc *breloc) -{ - struct ftl_band *band = breloc->band; - struct ftl_reloc *reloc = breloc->parent; - struct ftl_reloc_move *move; - size_t i; - - reloc->num_active++; - - if (!band->high_prio) { - if (ftl_band_alloc_lba_map(band)) { - SPDK_ERRLOG("Failed to allocate lba map\n"); - assert(false); - } - } else { - ftl_band_acquire_lba_map(band); - } - - for (i = 0; i < reloc->max_qdepth; ++i) { - move = &breloc->moves[i]; - move->state = FTL_RELOC_STATE_READ; - STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); - } -} - -static void -ftl_reloc_free_move(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) -{ - assert(move); - spdk_dma_free(move->data); - memset(move, 0, sizeof(*move)); - move->state = FTL_RELOC_STATE_READ; -} - -static void -ftl_reloc_write_cb(struct ftl_io *io, void *arg, int status) -{ - struct ftl_reloc_move *move = arg; - struct ftl_addr addr = move->addr; - struct ftl_band_reloc *breloc = move->breloc; - size_t i; - - breloc->num_outstanding--; - - if (status) { - SPDK_ERRLOG("Reloc write failed with status: %d\n", status); - assert(false); - return; - } - - for (i = 0; i < move->num_blocks; ++i) { - addr.offset = move->addr.offset + i; - size_t block_off = ftl_band_block_offset_from_addr(breloc->band, addr); - ftl_reloc_clr_block(breloc, block_off); - } - - ftl_reloc_free_move(breloc, move); - STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); -} - -static void -ftl_reloc_read_cb(struct ftl_io *io, void *arg, int status) -{ - struct ftl_reloc_move *move = arg; - struct ftl_band_reloc *breloc = move->breloc; - - breloc->num_outstanding--; - - /* TODO: We should handle fail on relocation read. We need to inform */ - /* user that this group of blocks is bad (update l2p with bad block address and */ - /* put it to lba_map/sector_lba). Maybe we could also retry read with smaller granularity? */ - if (status) { - SPDK_ERRLOG("Reloc read failed with status: %d\n", status); - assert(false); - return; - } - - move->state = FTL_RELOC_STATE_READ_LBA_MAP; - move->io = NULL; - STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); -} - -static void -ftl_reloc_iter_reset(struct ftl_band_reloc *breloc) -{ - memset(breloc->iter.zone_offset, 0, ftl_get_num_punits(breloc->band->dev) * - sizeof(*breloc->iter.zone_offset)); - breloc->iter.zone_current = 0; -} - -static size_t -ftl_reloc_iter_block_offset(struct ftl_band_reloc *breloc) -{ - size_t zone_offset = breloc->iter.zone_current * ftl_get_num_blocks_in_zone(breloc->parent->dev); - - return breloc->iter.zone_offset[breloc->iter.zone_current] + zone_offset; -} - -static void -ftl_reloc_iter_next_zone(struct ftl_band_reloc *breloc) -{ - size_t num_zones = ftl_get_num_punits(breloc->band->dev); - - breloc->iter.zone_current = (breloc->iter.zone_current + 1) % num_zones; -} - -static int -ftl_reloc_block_valid(struct ftl_band_reloc *breloc, size_t block_off) -{ - struct ftl_addr addr = ftl_band_addr_from_block_offset(breloc->band, block_off); - - return ftl_addr_is_written(breloc->band, addr) && - spdk_bit_array_get(breloc->reloc_map, block_off) && - ftl_band_block_offset_valid(breloc->band, block_off); -} - -static int -ftl_reloc_iter_next(struct ftl_band_reloc *breloc, size_t *block_off) -{ - size_t zone = breloc->iter.zone_current; - - *block_off = ftl_reloc_iter_block_offset(breloc); - - if (ftl_reloc_iter_zone_done(breloc)) { - return 0; - } - - breloc->iter.zone_offset[zone]++; - - if (!ftl_reloc_block_valid(breloc, *block_off)) { - ftl_reloc_clr_block(breloc, *block_off); - return 0; - } - - return 1; -} - -static int -ftl_reloc_first_valid_block(struct ftl_band_reloc *breloc, size_t *block_off) -{ - size_t i, num_blocks = ftl_get_num_blocks_in_zone(breloc->parent->dev); - - for (i = ftl_reloc_iter_zone_offset(breloc); i < num_blocks; ++i) { - if (ftl_reloc_iter_next(breloc, block_off)) { - return 1; - } - } - - return 0; -} - -static int -ftl_reloc_iter_done(struct ftl_band_reloc *breloc) -{ - size_t i; - size_t num_zones = ftl_get_num_punits(breloc->band->dev); - size_t num_blocks = ftl_get_num_blocks_in_zone(breloc->parent->dev); - - for (i = 0; i < num_zones; ++i) { - if (breloc->iter.zone_offset[i] != num_blocks) { - return 0; - } - } - - return 1; -} - -static size_t -ftl_reloc_find_valid_blocks(struct ftl_band_reloc *breloc, - size_t _num_blocks, struct ftl_addr *addr) -{ - size_t block_off, num_blocks = 0; - - if (!ftl_reloc_first_valid_block(breloc, &block_off)) { - return 0; - } - - *addr = ftl_band_addr_from_block_offset(breloc->band, block_off); - - for (num_blocks = 1; num_blocks < _num_blocks; num_blocks++) { - if (!ftl_reloc_iter_next(breloc, &block_off)) { - break; - } - } - - return num_blocks; -} - -static size_t -ftl_reloc_next_blocks(struct ftl_band_reloc *breloc, struct ftl_addr *addr) -{ - size_t i, num_blocks = 0; - struct spdk_ftl_dev *dev = breloc->parent->dev; - - for (i = 0; i < ftl_get_num_punits(dev); ++i) { - num_blocks = ftl_reloc_find_valid_blocks(breloc, breloc->parent->xfer_size, addr); - ftl_reloc_iter_next_zone(breloc); - - if (num_blocks || ftl_reloc_iter_done(breloc)) { - break; - } - } - - return num_blocks; -} - -static struct ftl_io * -ftl_reloc_io_init(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move, - ftl_io_fn fn, enum ftl_io_type io_type, int flags) -{ - size_t block_off, i; - struct ftl_addr addr = move->addr; - struct ftl_io *io = NULL; - struct ftl_io_init_opts opts = { - .dev = breloc->parent->dev, - .band = breloc->band, - .size = sizeof(*io), - .flags = flags | FTL_IO_INTERNAL | FTL_IO_PHYSICAL_MODE, - .type = io_type, - .num_blocks = move->num_blocks, - .iovs = { - { - .iov_base = move->data, - .iov_len = move->num_blocks * FTL_BLOCK_SIZE, - } - }, - .iovcnt = 1, - .cb_fn = fn, - }; - - io = ftl_io_init_internal(&opts); - if (!io) { - return NULL; - } - - io->cb_ctx = move; - io->addr = move->addr; - - if (flags & FTL_IO_VECTOR_LBA) { - for (i = 0; i < io->num_blocks; ++i, ++addr.offset) { - block_off = ftl_band_block_offset_from_addr(breloc->band, addr); - - if (!ftl_band_block_offset_valid(breloc->band, block_off)) { - io->lba.vector[i] = FTL_LBA_INVALID; - continue; - } - - io->lba.vector[i] = breloc->band->lba_map.map[block_off]; - } - } - - ftl_trace_lba_io_init(io->dev, io); - - return io; -} - -static int -ftl_reloc_write(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) -{ - int io_flags = FTL_IO_WEAK | FTL_IO_VECTOR_LBA | FTL_IO_BYPASS_CACHE; - - if (spdk_likely(!move->io)) { - move->io = ftl_reloc_io_init(breloc, move, ftl_reloc_write_cb, - FTL_IO_WRITE, io_flags); - if (!move->io) { - ftl_reloc_free_move(breloc, move); - STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); - return -ENOMEM; - } - } - - breloc->num_outstanding++; - ftl_io_write(move->io); - return 0; -} - -static int -ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) -{ - struct ftl_addr addr = {}; - - move->num_blocks = ftl_reloc_next_blocks(breloc, &addr); - move->breloc = breloc; - move->addr = addr; - - if (!move->num_blocks) { - return 0; - } - - move->data = spdk_dma_malloc(FTL_BLOCK_SIZE * move->num_blocks, 4096, NULL); - if (!move->data) { - return -1; - } - - move->io = ftl_reloc_io_init(breloc, move, ftl_reloc_read_cb, FTL_IO_READ, 0); - if (!move->io) { - ftl_reloc_free_move(breloc, move); - STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); - SPDK_ERRLOG("Failed to initialize io for relocation."); - return -1; - } - - breloc->num_outstanding++; - ftl_io_read(move->io); - return 0; -} - -static void -ftl_reloc_process_moves(struct ftl_band_reloc *breloc) -{ - struct ftl_reloc_move *move; - STAILQ_HEAD(, ftl_reloc_move) move_queue; - int rc = 0; - - /* - * When IO allocation fails, we do not want to retry immediately so keep moves on - * temporary queue - */ - STAILQ_INIT(&move_queue); - STAILQ_SWAP(&breloc->move_queue, &move_queue, ftl_reloc_move); - - while (!STAILQ_EMPTY(&move_queue)) { - move = STAILQ_FIRST(&move_queue); - STAILQ_REMOVE_HEAD(&move_queue, entry); - - switch (move->state) { - case FTL_RELOC_STATE_READ_LBA_MAP: - rc = ftl_reloc_read_lba_map(breloc, move); - break; - case FTL_RELOC_STATE_READ: - rc = ftl_reloc_read(breloc, move); - break; - case FTL_RELOC_STATE_WRITE: - rc = ftl_reloc_write(breloc, move); - break; - default: - assert(false); - break; - } - - if (rc) { - SPDK_ERRLOG("Move queue processing failed\n"); - assert(false); - } - } -} - -static bool -ftl_reloc_done(struct ftl_band_reloc *breloc) -{ - return !breloc->num_outstanding && STAILQ_EMPTY(&breloc->move_queue); -} - -static void -ftl_reloc_release(struct ftl_band_reloc *breloc) -{ - struct ftl_reloc *reloc = breloc->parent; - struct ftl_band *band = breloc->band; - - ftl_reloc_iter_reset(breloc); - ftl_band_release_lba_map(band); - reloc->num_active--; - - if (breloc->state == FTL_BAND_RELOC_STATE_HIGH_PRIO) { - /* High prio band must be relocated as a whole and ANM events will be ignored */ - assert(breloc->num_blocks == 0 && ftl_band_empty(band)); - TAILQ_REMOVE(&reloc->prio_queue, breloc, entry); - band->high_prio = 0; - breloc->state = FTL_BAND_RELOC_STATE_INACTIVE; - } else { - assert(breloc->state == FTL_BAND_RELOC_STATE_ACTIVE); - TAILQ_REMOVE(&reloc->active_queue, breloc, entry); - breloc->state = FTL_BAND_RELOC_STATE_INACTIVE; - - /* If we got ANM event during relocation put such band back to pending queue */ - if (breloc->num_blocks != 0) { - breloc->state = FTL_BAND_RELOC_STATE_PENDING; - TAILQ_INSERT_TAIL(&reloc->pending_queue, breloc, entry); - return; - } - } - - if (ftl_band_empty(band) && band->state == FTL_BAND_STATE_CLOSED) { - ftl_band_set_state(breloc->band, FTL_BAND_STATE_FREE); - - if (breloc->defrag) { - breloc->defrag = false; - assert(reloc->num_defrag_bands > 0); - reloc->num_defrag_bands--; - } - } -} - -static void -ftl_process_reloc(struct ftl_band_reloc *breloc) -{ - ftl_reloc_process_moves(breloc); - - if (ftl_reloc_done(breloc)) { - ftl_reloc_release(breloc); - } -} - -static int -ftl_band_reloc_init(struct ftl_reloc *reloc, struct ftl_band_reloc *breloc, - struct ftl_band *band) -{ - breloc->band = band; - breloc->parent = reloc; - - breloc->reloc_map = spdk_bit_array_create(ftl_get_num_blocks_in_band(reloc->dev)); - if (!breloc->reloc_map) { - SPDK_ERRLOG("Failed to initialize reloc map"); - return -1; - } - - breloc->iter.zone_offset = calloc(ftl_get_num_punits(band->dev), - sizeof(*breloc->iter.zone_offset)); - if (!breloc->iter.zone_offset) { - SPDK_ERRLOG("Failed to initialize reloc iterator"); - return -1; - } - - STAILQ_INIT(&breloc->move_queue); - - breloc->moves = calloc(reloc->max_qdepth, sizeof(*breloc->moves)); - if (!breloc->moves) { - return -1; - } - - return 0; -} - -static void -ftl_band_reloc_free(struct ftl_band_reloc *breloc) -{ - struct ftl_reloc_move *move; - - if (!breloc) { - return; - } - - assert(breloc->num_outstanding == 0); - - /* Drain write queue if there is active band relocation during shutdown */ - if (breloc->state == FTL_BAND_RELOC_STATE_ACTIVE || - breloc->state == FTL_BAND_RELOC_STATE_HIGH_PRIO) { - assert(breloc->parent->halt); - STAILQ_FOREACH(move, &breloc->move_queue, entry) { - ftl_reloc_free_move(breloc, move); - } - } - - spdk_bit_array_free(&breloc->reloc_map); - free(breloc->iter.zone_offset); - free(breloc->moves); -} - -struct ftl_reloc * -ftl_reloc_init(struct spdk_ftl_dev *dev) -{ - struct ftl_reloc *reloc; - size_t i; - - reloc = calloc(1, sizeof(*reloc)); - if (!reloc) { - return NULL; - } - - reloc->dev = dev; - reloc->halt = true; - reloc->max_qdepth = dev->conf.max_reloc_qdepth; - reloc->max_active = dev->conf.max_active_relocs; - reloc->xfer_size = dev->xfer_size; - reloc->num_defrag_bands = 0; - - if (reloc->max_qdepth > FTL_RELOC_MAX_MOVES) { - goto error; - } - - reloc->brelocs = calloc(ftl_get_num_bands(dev), sizeof(*reloc->brelocs)); - if (!reloc->brelocs) { - goto error; - } - - for (i = 0; i < ftl_get_num_bands(reloc->dev); ++i) { - if (ftl_band_reloc_init(reloc, &reloc->brelocs[i], &dev->bands[i])) { - goto error; - } - } - - TAILQ_INIT(&reloc->pending_queue); - TAILQ_INIT(&reloc->active_queue); - TAILQ_INIT(&reloc->prio_queue); - - return reloc; -error: - ftl_reloc_free(reloc); - return NULL; -} - -void -ftl_reloc_free(struct ftl_reloc *reloc) -{ - size_t i; - - if (!reloc) { - return; - } - - for (i = 0; i < ftl_get_num_bands(reloc->dev); ++i) { - ftl_band_reloc_free(&reloc->brelocs[i]); - } - - free(reloc->brelocs); - free(reloc); -} - -bool -ftl_reloc_is_halted(const struct ftl_reloc *reloc) -{ - return reloc->halt; -} - -void -ftl_reloc_halt(struct ftl_reloc *reloc) -{ - reloc->halt = true; -} - -void -ftl_reloc_resume(struct ftl_reloc *reloc) -{ - reloc->halt = false; -} - -bool -ftl_reloc(struct ftl_reloc *reloc) -{ - struct ftl_band_reloc *breloc, *tbreloc; - - if (ftl_reloc_is_halted(reloc)) { - return false; - } - - /* Process first band from priority queue and return */ - breloc = TAILQ_FIRST(&reloc->prio_queue); - if (breloc) { - ftl_process_reloc(breloc); - return true; - } - - TAILQ_FOREACH_SAFE(breloc, &reloc->pending_queue, entry, tbreloc) { - if (reloc->num_active == reloc->max_active) { - break; - } - - /* Wait for band to close before relocating */ - if (breloc->band->state != FTL_BAND_STATE_CLOSED) { - continue; - } - - ftl_reloc_prep(breloc); - assert(breloc->state == FTL_BAND_RELOC_STATE_PENDING); - TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); - breloc->state = FTL_BAND_RELOC_STATE_ACTIVE; - TAILQ_INSERT_HEAD(&reloc->active_queue, breloc, entry); - } - - TAILQ_FOREACH_SAFE(breloc, &reloc->active_queue, entry, tbreloc) { - assert(breloc->state == FTL_BAND_RELOC_STATE_ACTIVE); - ftl_process_reloc(breloc); - } - - return reloc->num_active != 0; -} - -void -ftl_reloc_add(struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, - size_t num_blocks, int prio, bool is_defrag) -{ - struct ftl_band_reloc *breloc = &reloc->brelocs[band->id]; - size_t i; - - /* No need to add anything if already at high prio - whole band should be relocated */ - if (!prio && band->high_prio) { - return; - } - - pthread_spin_lock(&band->lba_map.lock); - if (band->lba_map.num_vld == 0) { - pthread_spin_unlock(&band->lba_map.lock); - - /* If the band is closed and has no valid blocks, free it */ - if (band->state == FTL_BAND_STATE_CLOSED) { - ftl_band_set_state(band, FTL_BAND_STATE_FREE); - } - - return; - } - pthread_spin_unlock(&band->lba_map.lock); - - for (i = offset; i < offset + num_blocks; ++i) { - if (spdk_bit_array_get(breloc->reloc_map, i)) { - continue; - } - spdk_bit_array_set(breloc->reloc_map, i); - breloc->num_blocks++; - } - - /* If the band is coming from the defrag process, mark it appropriately */ - if (is_defrag) { - assert(offset == 0 && num_blocks == ftl_get_num_blocks_in_band(band->dev)); - reloc->num_defrag_bands++; - breloc->defrag = true; - } - - if (!prio) { - if (breloc->state == FTL_BAND_RELOC_STATE_INACTIVE) { - breloc->state = FTL_BAND_RELOC_STATE_PENDING; - TAILQ_INSERT_HEAD(&reloc->pending_queue, breloc, entry); - } - } else { - bool active = false; - /* If priority band is already on pending or active queue, remove it from it */ - switch (breloc->state) { - case FTL_BAND_RELOC_STATE_PENDING: - TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); - break; - case FTL_BAND_RELOC_STATE_ACTIVE: - active = true; - TAILQ_REMOVE(&reloc->active_queue, breloc, entry); - break; - default: - break; - } - - breloc->state = FTL_BAND_RELOC_STATE_HIGH_PRIO; - TAILQ_INSERT_TAIL(&reloc->prio_queue, breloc, entry); - - /* - * If band has been already on active queue it doesn't need any additional - * resources - */ - if (!active) { - ftl_reloc_prep(breloc); - } - } -} diff --git a/lib/ftl/ftl_reloc.h b/lib/ftl/ftl_reloc.h deleted file mode 100644 index 8419ea64e..000000000 --- a/lib/ftl/ftl_reloc.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef FTL_RELOC_H -#define FTL_RELOC_H - -#include "spdk/stdinc.h" -#include "spdk/ftl.h" - -struct ftl_reloc; -struct ftl_band; - -struct ftl_reloc *ftl_reloc_init(struct spdk_ftl_dev *dev); -void ftl_reloc_free(struct ftl_reloc *reloc); -void ftl_reloc_add(struct ftl_reloc *reloc, struct ftl_band *band, - size_t offset, size_t num_blocks, int prio, bool is_defrag); -bool ftl_reloc(struct ftl_reloc *reloc); -void ftl_reloc_halt(struct ftl_reloc *reloc); -void ftl_reloc_resume(struct ftl_reloc *reloc); -bool ftl_reloc_is_halted(const struct ftl_reloc *reloc); -bool ftl_reloc_is_defrag_active(const struct ftl_reloc *reloc); - -#endif /* FTL_RELOC_H */ diff --git a/lib/ftl/ftl_restore.c b/lib/ftl/ftl_restore.c deleted file mode 100644 index b89deae17..000000000 --- a/lib/ftl/ftl_restore.c +++ /dev/null @@ -1,1319 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" -#include "spdk/ftl.h" -#include "spdk/util.h" -#include "spdk/likely.h" -#include "spdk/string.h" -#include "spdk/crc32.h" - -#include "ftl_core.h" -#include "ftl_band.h" -#include "ftl_io.h" - -struct ftl_restore_band { - struct ftl_restore *parent; - /* Associated band */ - struct ftl_band *band; - /* Status of retrieving this band's metadata */ - enum ftl_md_status md_status; - /* Padded queue link */ - STAILQ_ENTRY(ftl_restore_band) stailq; -}; - -struct ftl_nv_cache_restore; - -/* Describes single phase to be restored from non-volatile cache */ -struct ftl_nv_cache_range { - struct ftl_nv_cache_restore *parent; - /* Start offset */ - uint64_t start_addr; - /* Last block's address */ - uint64_t last_addr; - /* - * Number of blocks (can be smaller than the difference between the last - * and the starting block due to range overlap) - */ - uint64_t num_blocks; - /* Number of blocks already recovered */ - uint64_t num_recovered; - /* Current address during recovery */ - uint64_t current_addr; - /* Phase of the range */ - unsigned int phase; - /* Indicates whether the data from this range needs to be recovered */ - bool recovery; -}; - -struct ftl_nv_cache_block { - struct ftl_nv_cache_restore *parent; - /* Data buffer */ - void *buf; - /* Metadata buffer */ - void *md_buf; - /* Block offset within the cache */ - uint64_t offset; -}; - -struct ftl_nv_cache_restore { - struct ftl_nv_cache *nv_cache; - /* IO channel to use */ - struct spdk_io_channel *ioch; - /* - * Non-volatile cache ranges. The ranges can overlap, as we have no - * control over the order of completions. The phase of the range is the - * index within the table. The range with index 0 marks blocks that were - * never written. - */ - struct ftl_nv_cache_range range[FTL_NV_CACHE_PHASE_COUNT]; -#define FTL_NV_CACHE_RESTORE_DEPTH 128 - /* Non-volatile cache buffers */ - struct ftl_nv_cache_block block[FTL_NV_CACHE_RESTORE_DEPTH]; - /* Current address */ - uint64_t current_addr; - /* Number of outstanding requests */ - size_t num_outstanding; - /* Recovery/scan status */ - int status; - /* Current phase of the recovery */ - unsigned int phase; -}; - -struct ftl_restore { - struct spdk_ftl_dev *dev; - /* Completion callback (called for each phase of the restoration) */ - ftl_restore_fn cb; - /* Completion callback context */ - void *cb_arg; - /* Number of inflight IOs */ - unsigned int num_ios; - /* Current band number (index in the below bands array) */ - unsigned int current; - /* Array of bands */ - struct ftl_restore_band *bands; - /* Queue of bands to be padded (due to unsafe shutdown) */ - STAILQ_HEAD(, ftl_restore_band) pad_bands; - /* Status of the padding */ - int pad_status; - /* Metadata buffer */ - void *md_buf; - /* LBA map buffer */ - void *lba_map; - /* Indicates we're in the final phase of the restoration */ - bool final_phase; - /* Non-volatile cache recovery */ - struct ftl_nv_cache_restore nv_cache; -}; - -static int ftl_restore_tail_md(struct ftl_restore_band *rband); -static void ftl_pad_zone_cb(struct ftl_io *io, void *arg, int status); -static void ftl_restore_pad_band(struct ftl_restore_band *rband); - -static void -ftl_restore_free(struct ftl_restore *restore) -{ - unsigned int i; - - if (!restore) { - return; - } - - for (i = 0; i < FTL_NV_CACHE_RESTORE_DEPTH; ++i) { - spdk_dma_free(restore->nv_cache.block[i].buf); - } - - spdk_dma_free(restore->md_buf); - free(restore->bands); - free(restore); -} - -static struct ftl_restore * -ftl_restore_init(struct spdk_ftl_dev *dev, ftl_restore_fn cb, void *cb_arg) -{ - struct ftl_restore *restore; - struct ftl_restore_band *rband; - size_t i; - - restore = calloc(1, sizeof(*restore)); - if (!restore) { - goto error; - } - - restore->dev = dev; - restore->cb = cb; - restore->cb_arg = cb_arg; - restore->final_phase = false; - - restore->bands = calloc(ftl_get_num_bands(dev), sizeof(*restore->bands)); - if (!restore->bands) { - goto error; - } - - STAILQ_INIT(&restore->pad_bands); - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - rband = &restore->bands[i]; - rband->band = &dev->bands[i]; - rband->parent = restore; - rband->md_status = FTL_MD_NO_MD; - } - - /* Allocate buffer capable of holding head mds of all bands */ - restore->md_buf = spdk_dma_zmalloc(ftl_get_num_bands(dev) * ftl_head_md_num_blocks(dev) * - FTL_BLOCK_SIZE, 0, NULL); - if (!restore->md_buf) { - goto error; - } - - return restore; -error: - ftl_restore_free(restore); - return NULL; -} - -static void -ftl_restore_complete(struct ftl_restore *restore, int status) -{ - struct ftl_restore *ctx = status ? NULL : restore; - bool final_phase = restore->final_phase; - - restore->cb(ctx, status, restore->cb_arg); - if (status || final_phase) { - ftl_restore_free(restore); - } -} - -static int -ftl_band_cmp(const void *lband, const void *rband) -{ - uint64_t lseq = ((struct ftl_restore_band *)lband)->band->seq; - uint64_t rseq = ((struct ftl_restore_band *)rband)->band->seq; - - if (lseq < rseq) { - return -1; - } else { - return 1; - } -} - -static int -ftl_restore_check_seq(const struct ftl_restore *restore) -{ - const struct spdk_ftl_dev *dev = restore->dev; - const struct ftl_restore_band *rband; - const struct ftl_band *next_band; - size_t i; - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - rband = &restore->bands[i]; - if (rband->md_status != FTL_MD_SUCCESS) { - continue; - } - - next_band = LIST_NEXT(rband->band, list_entry); - if (next_band && rband->band->seq == next_band->seq) { - return -1; - } - } - - return 0; -} - -static bool -ftl_restore_head_valid(struct spdk_ftl_dev *dev, struct ftl_restore *restore, size_t *num_valid) -{ - struct ftl_restore_band *rband; - size_t i; - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - rband = &restore->bands[i]; - - if (rband->md_status != FTL_MD_SUCCESS && - rband->md_status != FTL_MD_NO_MD && - rband->md_status != FTL_MD_IO_FAILURE) { - SPDK_ERRLOG("Inconsistent head metadata found on band %u\n", - rband->band->id); - return false; - } - - if (rband->md_status == FTL_MD_SUCCESS) { - (*num_valid)++; - } - } - - return true; -} - -static void -ftl_restore_head_complete(struct ftl_restore *restore) -{ - struct spdk_ftl_dev *dev = restore->dev; - size_t num_valid = 0; - int status = -EIO; - - if (!ftl_restore_head_valid(dev, restore, &num_valid)) { - goto out; - } - - if (num_valid == 0) { - SPDK_ERRLOG("Couldn't find any valid bands\n"); - goto out; - } - - /* Sort bands in sequence number ascending order */ - qsort(restore->bands, ftl_get_num_bands(dev), sizeof(struct ftl_restore_band), - ftl_band_cmp); - - if (ftl_restore_check_seq(restore)) { - SPDK_ERRLOG("Band sequence consistency failed\n"); - goto out; - } - - dev->num_lbas = dev->global_md.num_lbas; - status = 0; -out: - ftl_restore_complete(restore, status); -} - -static void -ftl_restore_head_cb(struct ftl_io *io, void *ctx, int status) -{ - struct ftl_restore_band *rband = ctx; - struct ftl_restore *restore = rband->parent; - unsigned int num_ios; - - rband->md_status = status; - num_ios = __atomic_fetch_sub(&restore->num_ios, 1, __ATOMIC_SEQ_CST); - assert(num_ios > 0); - - if (num_ios == 1) { - ftl_restore_head_complete(restore); - } -} - -static void -ftl_restore_head_md(void *ctx) -{ - struct ftl_restore *restore = ctx; - struct spdk_ftl_dev *dev = restore->dev; - struct ftl_restore_band *rband; - struct ftl_lba_map *lba_map; - unsigned int num_failed = 0, num_ios; - size_t i; - - restore->num_ios = ftl_get_num_bands(dev); - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - rband = &restore->bands[i]; - lba_map = &rband->band->lba_map; - - lba_map->dma_buf = restore->md_buf + i * ftl_head_md_num_blocks(dev) * FTL_BLOCK_SIZE; - - if (ftl_band_read_head_md(rband->band, ftl_restore_head_cb, rband)) { - if (spdk_likely(rband->band->num_zones)) { - SPDK_ERRLOG("Failed to read metadata on band %zu\n", i); - - rband->md_status = FTL_MD_INVALID_CRC; - - /* If the first IO fails, don't bother sending anything else */ - if (i == 0) { - ftl_restore_complete(restore, -EIO); - } - } - - num_failed++; - } - } - - if (spdk_unlikely(num_failed > 0)) { - num_ios = __atomic_fetch_sub(&restore->num_ios, num_failed, __ATOMIC_SEQ_CST); - if (num_ios == num_failed) { - ftl_restore_complete(restore, -EIO); - } - } -} - -int -ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb, void *cb_arg) -{ - struct ftl_restore *restore; - - restore = ftl_restore_init(dev, cb, cb_arg); - if (!restore) { - return -ENOMEM; - } - - spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_restore_head_md, restore); - - return 0; -} - -static int -ftl_restore_l2p(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - struct ftl_addr addr; - uint64_t lba; - size_t i; - - for (i = 0; i < ftl_get_num_blocks_in_band(band->dev); ++i) { - if (!spdk_bit_array_get(band->lba_map.vld, i)) { - continue; - } - - lba = band->lba_map.map[i]; - if (lba >= dev->num_lbas) { - return -1; - } - - addr = ftl_l2p_get(dev, lba); - if (!ftl_addr_invalid(addr)) { - ftl_invalidate_addr(dev, addr); - } - - addr = ftl_band_addr_from_block_offset(band, i); - - ftl_band_set_addr(band, lba, addr); - ftl_l2p_set(dev, lba, addr); - } - - return 0; -} - -static struct ftl_restore_band * -ftl_restore_next_band(struct ftl_restore *restore) -{ - struct ftl_restore_band *rband; - - for (; restore->current < ftl_get_num_bands(restore->dev); ++restore->current) { - rband = &restore->bands[restore->current]; - - if (spdk_likely(rband->band->num_zones) && - rband->md_status == FTL_MD_SUCCESS) { - restore->current++; - return rband; - } - } - - return NULL; -} - -static void -ftl_nv_cache_restore_complete(struct ftl_nv_cache_restore *restore, int status) -{ - struct ftl_restore *ftl_restore = SPDK_CONTAINEROF(restore, struct ftl_restore, nv_cache); - - restore->status = restore->status ? : status; - if (restore->num_outstanding == 0) { - ftl_restore_complete(ftl_restore, restore->status); - } -} - -static void ftl_nv_cache_block_read_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg); - -static void -ftl_nv_cache_restore_done(struct ftl_nv_cache_restore *restore, uint64_t current_addr) -{ - struct ftl_nv_cache *nv_cache = restore->nv_cache; - - pthread_spin_lock(&nv_cache->lock); - nv_cache->current_addr = current_addr; - nv_cache->ready = true; - pthread_spin_unlock(&nv_cache->lock); - - SPDK_DEBUGLOG(ftl_init, "Enabling non-volatile cache (phase: %u, addr: %" - PRIu64")\n", nv_cache->phase, current_addr); - - ftl_nv_cache_restore_complete(restore, 0); -} - -static void -ftl_nv_cache_write_header_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_nv_cache_restore *restore = cb_arg; - - spdk_bdev_free_io(bdev_io); - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Unable to write the non-volatile cache metadata header\n"); - ftl_nv_cache_restore_complete(restore, -EIO); - return; - } - - ftl_nv_cache_restore_done(restore, FTL_NV_CACHE_DATA_OFFSET); -} - -static void -ftl_nv_cache_scrub_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_nv_cache_restore *restore = cb_arg; - struct ftl_nv_cache *nv_cache = restore->nv_cache; - int rc; - - spdk_bdev_free_io(bdev_io); - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Scrubbing non-volatile cache failed\n"); - ftl_nv_cache_restore_complete(restore, -EIO); - return; - } - - nv_cache->phase = 1; - rc = ftl_nv_cache_write_header(nv_cache, false, ftl_nv_cache_write_header_cb, restore); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to write the non-volatile cache metadata header: %s\n", - spdk_strerror(-rc)); - ftl_nv_cache_restore_complete(restore, -EIO); - } -} - -static void -ftl_nv_cache_scrub_header_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_nv_cache_restore *restore = cb_arg; - struct ftl_nv_cache *nv_cache = restore->nv_cache; - int rc; - - spdk_bdev_free_io(bdev_io); - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Unable to write non-volatile cache metadata header\n"); - ftl_nv_cache_restore_complete(restore, -EIO); - return; - } - - rc = ftl_nv_cache_scrub(nv_cache, ftl_nv_cache_scrub_cb, restore); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to scrub the non-volatile cache: %s\n", spdk_strerror(-rc)); - ftl_nv_cache_restore_complete(restore, rc); - } -} - -static void -ftl_nv_cache_band_flush_cb(void *ctx, int status) -{ - struct ftl_nv_cache_restore *restore = ctx; - struct ftl_nv_cache *nv_cache = restore->nv_cache; - int rc; - - if (spdk_unlikely(status != 0)) { - SPDK_ERRLOG("Flushing active bands failed: %s\n", spdk_strerror(-status)); - ftl_nv_cache_restore_complete(restore, status); - return; - } - - /* - * Use phase 0 to indicate that the cache is being scrubbed. If the power is lost during - * this process, we'll know it needs to be resumed. - */ - nv_cache->phase = 0; - rc = ftl_nv_cache_write_header(nv_cache, false, ftl_nv_cache_scrub_header_cb, restore); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to write non-volatile cache metadata header: %s\n", - spdk_strerror(-rc)); - ftl_nv_cache_restore_complete(restore, rc); - } -} - -static void -ftl_nv_cache_wbuf_flush_cb(void *ctx, int status) -{ - struct ftl_nv_cache_restore *restore = ctx; - struct ftl_nv_cache *nv_cache = restore->nv_cache; - struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); - int rc; - - if (spdk_unlikely(status != 0)) { - SPDK_ERRLOG("Flushing the write buffer failed: %s\n", spdk_strerror(-status)); - ftl_nv_cache_restore_complete(restore, status); - return; - } - - rc = ftl_flush_active_bands(dev, ftl_nv_cache_band_flush_cb, restore); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to flush active bands: %s\n", spdk_strerror(-rc)); - ftl_nv_cache_restore_complete(restore, rc); - } -} - -static void -ftl_nv_cache_recovery_done(struct ftl_nv_cache_restore *restore) -{ - struct ftl_nv_cache *nv_cache = restore->nv_cache; - struct ftl_nv_cache_range *range_prev, *range_current; - struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); - struct spdk_bdev *bdev; - uint64_t current_addr; - int rc; - - range_prev = &restore->range[ftl_nv_cache_prev_phase(nv_cache->phase)]; - range_current = &restore->range[nv_cache->phase]; - bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - - /* - * If there are more than two ranges or the ranges overlap, scrub the non-volatile cache to - * make sure that any subsequent power loss will find the cache in usable state - */ - if ((range_prev->num_blocks + range_current->num_blocks < nv_cache->num_data_blocks) || - (range_prev->start_addr < range_current->last_addr && - range_current->start_addr < range_prev->last_addr)) { - SPDK_DEBUGLOG(ftl_init, "Non-volatile cache inconsistency detected\n"); - - rc = ftl_flush_wbuf(dev, ftl_nv_cache_wbuf_flush_cb, restore); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to flush the write buffer: %s\n", spdk_strerror(-rc)); - ftl_nv_cache_restore_complete(restore, rc); - } - - return; - } - - /* The latest phase is the one written in the header (set in nvc_cache->phase) */ - current_addr = range_current->last_addr + 1; - - /* - * The first range might be empty (only the header was written) or the range might - * end at the last available address, in which case set current address to the - * beginning of the device. - */ - if (range_current->num_blocks == 0 || current_addr >= spdk_bdev_get_num_blocks(bdev)) { - current_addr = FTL_NV_CACHE_DATA_OFFSET; - } - - ftl_nv_cache_restore_done(restore, current_addr); -} - -static void -ftl_nv_cache_recover_block(struct ftl_nv_cache_block *block) -{ - struct ftl_nv_cache_restore *restore = block->parent; - struct ftl_nv_cache *nv_cache = restore->nv_cache; - struct ftl_nv_cache_range *range = &restore->range[restore->phase]; - int rc; - - assert(range->current_addr <= range->last_addr); - - restore->num_outstanding++; - block->offset = range->current_addr++; - rc = spdk_bdev_read_blocks_with_md(nv_cache->bdev_desc, restore->ioch, - block->buf, block->md_buf, - block->offset, 1, ftl_nv_cache_block_read_cb, - block); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Non-volatile cache restoration failed on block %"PRIu64" (%s)\n", - block->offset, spdk_strerror(-rc)); - restore->num_outstanding--; - ftl_nv_cache_restore_complete(restore, rc); - } -} - -static void -ftl_nv_cache_recover_range(struct ftl_nv_cache_restore *restore) -{ - struct ftl_nv_cache_range *range; - unsigned int phase = restore->phase; - - do { - /* Find first range with non-zero number of blocks that is marked for recovery */ - range = &restore->range[phase]; - if (range->recovery && range->num_recovered < range->num_blocks) { - break; - } - - phase = ftl_nv_cache_next_phase(phase); - } while (phase != restore->phase); - - /* There are no ranges to be recovered, we're done */ - if (range->num_recovered == range->num_blocks || !range->recovery) { - SPDK_DEBUGLOG(ftl_init, "Non-volatile cache recovery done\n"); - ftl_nv_cache_recovery_done(restore); - return; - } - - range->current_addr = range->start_addr; - restore->phase = phase; - - SPDK_DEBUGLOG(ftl_init, "Recovering range %u %"PRIu64"-%"PRIu64" (%"PRIu64")\n", - phase, range->start_addr, range->last_addr, range->num_blocks); - - ftl_nv_cache_recover_block(&restore->block[0]); -} - -static void -ftl_nv_cache_write_cb(struct ftl_io *io, void *cb_arg, int status) -{ - struct ftl_nv_cache_block *block = cb_arg; - struct ftl_nv_cache_restore *restore = block->parent; - struct ftl_nv_cache_range *range = &restore->range[restore->phase]; - - restore->num_outstanding--; - if (status != 0) { - SPDK_ERRLOG("Non-volatile cache restoration failed on block %"PRIu64" (%s)\n", - block->offset, spdk_strerror(-status)); - ftl_nv_cache_restore_complete(restore, -ENOMEM); - return; - } - - range->num_recovered++; - if (range->current_addr <= range->last_addr) { - ftl_nv_cache_recover_block(block); - } else if (restore->num_outstanding == 0) { - assert(range->num_recovered == range->num_blocks); - ftl_nv_cache_recover_range(restore); - } -} - -static struct ftl_io * -ftl_nv_cache_alloc_io(struct ftl_nv_cache_block *block, uint64_t lba) -{ - struct ftl_restore *restore = SPDK_CONTAINEROF(block->parent, struct ftl_restore, nv_cache); - struct ftl_io_init_opts opts = { - .dev = restore->dev, - .io = NULL, - .flags = FTL_IO_BYPASS_CACHE, - .type = FTL_IO_WRITE, - .num_blocks = 1, - .cb_fn = ftl_nv_cache_write_cb, - .cb_ctx = block, - .iovs = { - { - .iov_base = block->buf, - .iov_len = FTL_BLOCK_SIZE, - } - }, - .iovcnt = 1, - }; - struct ftl_io *io; - - io = ftl_io_init_internal(&opts); - if (spdk_unlikely(!io)) { - return NULL; - } - - io->lba.single = lba; - return io; -} - -static void -ftl_nv_cache_block_read_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_nv_cache_block *block = cb_arg; - struct ftl_nv_cache_restore *restore = block->parent; - struct ftl_nv_cache_range *range = &restore->range[restore->phase]; - struct ftl_io *io; - unsigned int phase; - uint64_t lba; - - spdk_bdev_free_io(bdev_io); - restore->num_outstanding--; - - if (!success) { - SPDK_ERRLOG("Non-volatile cache restoration failed on block %"PRIu64"\n", - block->offset); - ftl_nv_cache_restore_complete(restore, -EIO); - return; - } - - ftl_nv_cache_unpack_lba(*(uint64_t *)block->md_buf, &lba, &phase); - if (spdk_unlikely(phase != restore->phase)) { - if (range->current_addr < range->last_addr) { - ftl_nv_cache_recover_block(block); - } else if (restore->num_outstanding == 0) { - ftl_nv_cache_recover_range(restore); - } - - return; - } - - io = ftl_nv_cache_alloc_io(block, lba); - if (spdk_unlikely(!io)) { - SPDK_ERRLOG("Failed to allocate ftl_io during non-volatile cache recovery\n"); - ftl_nv_cache_restore_complete(restore, -ENOMEM); - return; - } - - restore->num_outstanding++; - ftl_io_write(io); -} - -/* - * Since we have no control over the order in which the requests complete in regards to their - * submission, the cache can be in either of the following states: - * - [1 1 1 1 1 1 1 1 1 1]: simplest case, whole cache contains single phase (although it should be - * very rare), - * - [1 1 1 1 3 3 3 3 3 3]: two phases, changing somewhere in the middle with no overlap. This is - * the state left by clean shutdown, - * - [1 1 1 1 3 1 3 3 3 3]: similar to the above, but this time the two ranges overlap. This - * happens when completions are reordered during unsafe shutdown, - * - [2 1 2 1 1 1 1 3 1 3]: three different phases, each one of which can overlap with - * previous/next one. The data from the oldest phase doesn't need to be - * recovered, as it was already being written to, which means it's - * already on the main storage. - */ -static void -ftl_nv_cache_scan_done(struct ftl_nv_cache_restore *restore) -{ - struct ftl_nv_cache *nv_cache = restore->nv_cache; -#if defined(DEBUG) - struct ftl_nv_cache_range *range; - uint64_t i, num_blocks = 0; - - for (i = 0; i < FTL_NV_CACHE_PHASE_COUNT; ++i) { - range = &restore->range[i]; - SPDK_DEBUGLOG(ftl_init, "Range %"PRIu64": %"PRIu64"-%"PRIu64" (%" PRIu64 - ")\n", i, range->start_addr, range->last_addr, range->num_blocks); - num_blocks += range->num_blocks; - } - assert(num_blocks == nv_cache->num_data_blocks); -#endif - restore->phase = ftl_nv_cache_prev_phase(nv_cache->phase); - - /* - * Only the latest two phases need to be recovered. The third one, even if present, - * already has to be stored on the main storage, as it's already started to be - * overwritten (only present here because of reordering of requests' completions). - */ - restore->range[nv_cache->phase].recovery = true; - restore->range[restore->phase].recovery = true; - - ftl_nv_cache_recover_range(restore); -} - -static int ftl_nv_cache_scan_block(struct ftl_nv_cache_block *block); - -static void -ftl_nv_cache_scan_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_nv_cache_block *block = cb_arg; - struct ftl_nv_cache_restore *restore = block->parent; - struct ftl_nv_cache_range *range; - struct spdk_bdev *bdev; - unsigned int phase; - uint64_t lba; - - restore->num_outstanding--; - bdev = spdk_bdev_desc_get_bdev(restore->nv_cache->bdev_desc); - spdk_bdev_free_io(bdev_io); - - if (!success) { - SPDK_ERRLOG("Non-volatile cache scan failed on block %"PRIu64"\n", - block->offset); - ftl_nv_cache_restore_complete(restore, -EIO); - return; - } - - /* If we've already hit an error, don't bother with scanning anything else */ - if (spdk_unlikely(restore->status != 0)) { - ftl_nv_cache_restore_complete(restore, restore->status); - return; - } - - ftl_nv_cache_unpack_lba(*(uint64_t *)block->md_buf, &lba, &phase); - range = &restore->range[phase]; - range->num_blocks++; - - if (range->start_addr == FTL_LBA_INVALID || range->start_addr > block->offset) { - range->start_addr = block->offset; - } - - if (range->last_addr == FTL_LBA_INVALID || range->last_addr < block->offset) { - range->last_addr = block->offset; - } - - /* All the blocks were read, once they're all completed and we're finished */ - if (restore->current_addr == spdk_bdev_get_num_blocks(bdev)) { - if (restore->num_outstanding == 0) { - ftl_nv_cache_scan_done(restore); - } - - return; - } - - ftl_nv_cache_scan_block(block); -} - -static int -ftl_nv_cache_scan_block(struct ftl_nv_cache_block *block) -{ - struct ftl_nv_cache_restore *restore = block->parent; - struct ftl_nv_cache *nv_cache = restore->nv_cache; - int rc; - - restore->num_outstanding++; - block->offset = restore->current_addr++; - rc = spdk_bdev_read_blocks_with_md(nv_cache->bdev_desc, restore->ioch, - block->buf, block->md_buf, - block->offset, 1, ftl_nv_cache_scan_cb, - block); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Non-volatile cache scan failed on block %"PRIu64" (%s)\n", - block->offset, spdk_strerror(-rc)); - restore->num_outstanding--; - ftl_nv_cache_restore_complete(restore, rc); - return rc; - } - - return 0; -} - -static void -ftl_nv_cache_clean_header_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_nv_cache_restore *restore = cb_arg; - - spdk_bdev_free_io(bdev_io); - if (spdk_unlikely(!success)) { - SPDK_ERRLOG("Unable to write the non-volatile cache metadata header\n"); - ftl_nv_cache_restore_complete(restore, -EIO); - return; - } - - ftl_nv_cache_restore_done(restore, restore->current_addr); -} - -static bool -ftl_nv_cache_header_valid(struct spdk_ftl_dev *dev, const struct ftl_nv_cache_header *hdr) -{ - struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); - uint32_t checksum; - - checksum = spdk_crc32c_update(hdr, offsetof(struct ftl_nv_cache_header, checksum), 0); - if (checksum != hdr->checksum) { - SPDK_ERRLOG("Invalid header checksum (found: %"PRIu32", expected: %"PRIu32")\n", - checksum, hdr->checksum); - return false; - } - - if (hdr->version != FTL_NV_CACHE_HEADER_VERSION) { - SPDK_ERRLOG("Invalid header version (found: %"PRIu32", expected: %"PRIu32")\n", - hdr->version, FTL_NV_CACHE_HEADER_VERSION); - return false; - } - - if (hdr->size != spdk_bdev_get_num_blocks(bdev)) { - SPDK_ERRLOG("Unexpected size of the non-volatile cache bdev (%"PRIu64", expected: %" - PRIu64")\n", hdr->size, spdk_bdev_get_num_blocks(bdev)); - return false; - } - - if (spdk_uuid_compare(&hdr->uuid, &dev->uuid)) { - SPDK_ERRLOG("Invalid device UUID\n"); - return false; - } - - if (!ftl_nv_cache_phase_is_valid(hdr->phase) && hdr->phase != 0) { - return false; - } - - if ((hdr->current_addr >= spdk_bdev_get_num_blocks(bdev) || - hdr->current_addr < FTL_NV_CACHE_DATA_OFFSET) && - (hdr->current_addr != FTL_LBA_INVALID)) { - SPDK_ERRLOG("Unexpected value of non-volatile cache's current address: %"PRIu64"\n", - hdr->current_addr); - return false; - } - - return true; -} - -static void -ftl_nv_cache_read_header_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) -{ - struct ftl_restore *restore = cb_arg; - struct spdk_ftl_dev *dev = restore->dev; - struct ftl_nv_cache *nv_cache = &dev->nv_cache; - struct ftl_nv_cache_header *hdr; - struct iovec *iov = NULL; - int iov_cnt = 0, i, rc; - - if (!success) { - SPDK_ERRLOG("Unable to read non-volatile cache metadata header\n"); - ftl_restore_complete(restore, -ENOTRECOVERABLE); - goto out; - } - - spdk_bdev_io_get_iovec(bdev_io, &iov, &iov_cnt); - assert(iov != NULL); - hdr = iov[0].iov_base; - - if (!ftl_nv_cache_header_valid(dev, hdr)) { - ftl_restore_complete(restore, -ENOTRECOVERABLE); - goto out; - } - - /* Remember the latest phase */ - nv_cache->phase = hdr->phase; - - /* If the phase equals zero, we lost power during recovery. We need to finish it up - * by scrubbing the device once again. - */ - if (hdr->phase == 0) { - SPDK_DEBUGLOG(ftl_init, "Detected phase 0, restarting scrub\n"); - rc = ftl_nv_cache_scrub(nv_cache, ftl_nv_cache_scrub_cb, restore); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Unable to scrub the non-volatile cache: %s\n", - spdk_strerror(-rc)); - ftl_restore_complete(restore, -ENOTRECOVERABLE); - } - - goto out; - } - - /* Valid current_addr means that the shutdown was clean, so we just need to overwrite the - * header to make sure that any power loss occurring before the cache is wrapped won't be - * mistaken for a clean shutdown. - */ - if (hdr->current_addr != FTL_LBA_INVALID) { - restore->nv_cache.current_addr = hdr->current_addr; - - rc = ftl_nv_cache_write_header(nv_cache, false, ftl_nv_cache_clean_header_cb, - &restore->nv_cache); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Failed to overwrite the non-volatile cache header: %s\n", - spdk_strerror(-rc)); - ftl_restore_complete(restore, -ENOTRECOVERABLE); - } - - goto out; - } - - /* Otherwise the shutdown was unexpected, so we need to recover the data from the cache */ - restore->nv_cache.current_addr = FTL_NV_CACHE_DATA_OFFSET; - - for (i = 0; i < FTL_NV_CACHE_RESTORE_DEPTH; ++i) { - if (ftl_nv_cache_scan_block(&restore->nv_cache.block[i])) { - break; - } - } -out: - spdk_bdev_free_io(bdev_io); -} - -void -ftl_restore_nv_cache(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg) -{ - struct spdk_ftl_dev *dev = restore->dev; - struct spdk_bdev *bdev; - struct ftl_nv_cache *nv_cache = &dev->nv_cache; - struct ftl_io_channel *ioch; - struct ftl_nv_cache_restore *nvc_restore = &restore->nv_cache; - struct ftl_nv_cache_block *block; - size_t alignment; - int rc, i; - - ioch = ftl_io_channel_get_ctx(ftl_get_io_channel(dev)); - bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); - alignment = spdk_max(spdk_bdev_get_buf_align(bdev), sizeof(uint64_t)); - - nvc_restore->nv_cache = nv_cache; - nvc_restore->ioch = ioch->cache_ioch; - - restore->final_phase = true; - restore->cb = cb; - restore->cb_arg = cb_arg; - - for (i = 0; i < FTL_NV_CACHE_RESTORE_DEPTH; ++i) { - block = &nvc_restore->block[i]; - block->parent = nvc_restore; - block->buf = spdk_dma_zmalloc(spdk_bdev_get_block_size(bdev) + - spdk_bdev_get_md_size(bdev), - alignment, NULL); - if (!block->buf) { - /* The memory will be freed in ftl_restore_free */ - SPDK_ERRLOG("Unable to allocate memory\n"); - ftl_restore_complete(restore, -ENOMEM); - return; - } - - block->md_buf = (char *)block->buf + spdk_bdev_get_block_size(bdev); - } - - for (i = 0; i < FTL_NV_CACHE_PHASE_COUNT; ++i) { - nvc_restore->range[i].parent = nvc_restore; - nvc_restore->range[i].start_addr = FTL_LBA_INVALID; - nvc_restore->range[i].last_addr = FTL_LBA_INVALID; - nvc_restore->range[i].num_blocks = 0; - nvc_restore->range[i].recovery = false; - nvc_restore->range[i].phase = i; - } - - rc = spdk_bdev_read_blocks(nv_cache->bdev_desc, ioch->cache_ioch, nv_cache->dma_buf, - 0, FTL_NV_CACHE_DATA_OFFSET, ftl_nv_cache_read_header_cb, restore); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("Failed to read non-volatile cache metadata header: %s\n", - spdk_strerror(-rc)); - ftl_restore_complete(restore, rc); - } -} - -static bool -ftl_pad_zone_pad_finish(struct ftl_restore_band *rband, bool direct_access) -{ - struct ftl_restore *restore = rband->parent; - struct ftl_restore_band *next_band; - size_t i, num_pad_zones = 0; - - if (spdk_unlikely(restore->pad_status && !restore->num_ios)) { - if (direct_access) { - /* In case of any errors found we want to clear direct access. */ - /* Direct access bands have their own allocated md, which would be lost */ - /* on restore complete otherwise. */ - rband->band->state = FTL_BAND_STATE_CLOSED; - ftl_band_set_direct_access(rband->band, false); - } - ftl_restore_complete(restore, restore->pad_status); - return true; - } - - for (i = 0; i < rband->band->num_zones; ++i) { - if (rband->band->zone_buf[i].info.state != SPDK_BDEV_ZONE_STATE_FULL) { - num_pad_zones++; - } - } - - /* Finished all zones in a band, check if all bands are done */ - if (num_pad_zones == 0) { - if (direct_access) { - rband->band->state = FTL_BAND_STATE_CLOSED; - ftl_band_set_direct_access(rband->band, false); - } - - next_band = STAILQ_NEXT(rband, stailq); - if (!next_band) { - ftl_restore_complete(restore, restore->pad_status); - return true; - } else { - /* Start off padding in the next band */ - ftl_restore_pad_band(next_band); - return true; - } - } - - return false; -} - -static struct ftl_io * -ftl_restore_init_pad_io(struct ftl_restore_band *rband, void *buffer, - struct ftl_addr addr) -{ - struct ftl_band *band = rband->band; - struct spdk_ftl_dev *dev = band->dev; - int flags = FTL_IO_PAD | FTL_IO_INTERNAL | FTL_IO_PHYSICAL_MODE | FTL_IO_MD | - FTL_IO_DIRECT_ACCESS; - struct ftl_io_init_opts opts = { - .dev = dev, - .io = NULL, - .band = band, - .size = sizeof(struct ftl_io), - .flags = flags, - .type = FTL_IO_WRITE, - .num_blocks = dev->xfer_size, - .cb_fn = ftl_pad_zone_cb, - .cb_ctx = rband, - .iovs = { - { - .iov_base = buffer, - .iov_len = dev->xfer_size * FTL_BLOCK_SIZE, - } - }, - .iovcnt = 1, - .parent = NULL, - }; - struct ftl_io *io; - - io = ftl_io_init_internal(&opts); - if (spdk_unlikely(!io)) { - return NULL; - } - - io->addr = addr; - rband->parent->num_ios++; - - return io; -} - -static void -ftl_pad_zone_cb(struct ftl_io *io, void *arg, int status) -{ - struct ftl_restore_band *rband = arg; - struct ftl_restore *restore = rband->parent; - struct ftl_band *band = io->band; - struct ftl_zone *zone; - struct ftl_io *new_io; - uint64_t offset; - - restore->num_ios--; - /* TODO check for next unit error vs early close error */ - if (status) { - restore->pad_status = status; - goto end; - } - - offset = io->addr.offset % ftl_get_num_blocks_in_zone(restore->dev); - if (offset + io->num_blocks == ftl_get_num_blocks_in_zone(restore->dev)) { - zone = ftl_band_zone_from_addr(band, io->addr); - zone->info.state = SPDK_BDEV_ZONE_STATE_FULL; - } else { - struct ftl_addr addr = io->addr; - addr.offset += io->num_blocks; - new_io = ftl_restore_init_pad_io(rband, io->iov[0].iov_base, addr); - if (spdk_unlikely(!new_io)) { - restore->pad_status = -ENOMEM; - goto end; - } - - ftl_io_write(new_io); - return; - } - -end: - spdk_dma_free(io->iov[0].iov_base); - ftl_pad_zone_pad_finish(rband, true); -} - -static void -ftl_restore_pad_band(struct ftl_restore_band *rband) -{ - struct ftl_restore *restore = rband->parent; - struct ftl_band *band = rband->band; - struct spdk_ftl_dev *dev = band->dev; - void *buffer = NULL; - struct ftl_io *io; - struct ftl_addr addr; - size_t i; - int rc = 0; - - /* Check if some zones are not closed */ - if (ftl_pad_zone_pad_finish(rband, false)) { - /* - * If we're here, end meta wasn't recognized, but the whole band is written - * Assume the band was padded and ignore it - */ - return; - } - - band->state = FTL_BAND_STATE_OPEN; - rc = ftl_band_set_direct_access(band, true); - if (rc) { - ftl_restore_complete(restore, rc); - return; - } - - for (i = 0; i < band->num_zones; ++i) { - if (band->zone_buf[i].info.state == SPDK_BDEV_ZONE_STATE_FULL) { - continue; - } - - addr.offset = band->zone_buf[i].info.write_pointer; - - buffer = spdk_dma_zmalloc(FTL_BLOCK_SIZE * dev->xfer_size, 0, NULL); - if (spdk_unlikely(!buffer)) { - rc = -ENOMEM; - goto error; - } - - io = ftl_restore_init_pad_io(rband, buffer, addr); - if (spdk_unlikely(!io)) { - rc = -ENOMEM; - spdk_dma_free(buffer); - goto error; - } - - ftl_io_write(io); - } - - return; - -error: - restore->pad_status = rc; - ftl_pad_zone_pad_finish(rband, true); -} - -static void -ftl_restore_pad_open_bands(void *ctx) -{ - struct ftl_restore *restore = ctx; - - ftl_restore_pad_band(STAILQ_FIRST(&restore->pad_bands)); -} - -static void -ftl_restore_tail_md_cb(struct ftl_io *io, void *ctx, int status) -{ - struct ftl_restore_band *rband = ctx; - struct ftl_restore *restore = rband->parent; - struct spdk_ftl_dev *dev = restore->dev; - - if (status) { - if (!dev->conf.allow_open_bands) { - SPDK_ERRLOG("%s while restoring tail md in band %u.\n", - spdk_strerror(-status), rband->band->id); - ftl_band_release_lba_map(rband->band); - ftl_restore_complete(restore, status); - return; - } else { - SPDK_ERRLOG("%s while restoring tail md. Will attempt to pad band %u.\n", - spdk_strerror(-status), rband->band->id); - STAILQ_INSERT_TAIL(&restore->pad_bands, rband, stailq); - } - } - - if (!status && ftl_restore_l2p(rband->band)) { - ftl_band_release_lba_map(rband->band); - ftl_restore_complete(restore, -ENOTRECOVERABLE); - return; - } - ftl_band_release_lba_map(rband->band); - - rband = ftl_restore_next_band(restore); - if (!rband) { - if (!STAILQ_EMPTY(&restore->pad_bands)) { - spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_restore_pad_open_bands, - restore); - } else { - ftl_restore_complete(restore, 0); - } - - return; - } - - ftl_restore_tail_md(rband); -} - -static int -ftl_restore_tail_md(struct ftl_restore_band *rband) -{ - struct ftl_restore *restore = rband->parent; - struct ftl_band *band = rband->band; - - if (ftl_band_alloc_lba_map(band)) { - SPDK_ERRLOG("Failed to allocate lba map\n"); - ftl_restore_complete(restore, -ENOMEM); - return -ENOMEM; - } - - if (ftl_band_read_tail_md(band, band->tail_md_addr, ftl_restore_tail_md_cb, rband)) { - SPDK_ERRLOG("Failed to send tail metadata read\n"); - ftl_restore_complete(restore, -EIO); - return -EIO; - } - - return 0; -} - -int -ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg) -{ - struct spdk_ftl_dev *dev = restore->dev; - struct ftl_restore_band *rband; - - restore->current = 0; - restore->cb = cb; - restore->cb_arg = cb_arg; - restore->final_phase = dev->nv_cache.bdev_desc == NULL; - - /* If restore_device is called, there must be at least one valid band */ - rband = ftl_restore_next_band(restore); - assert(rband); - return ftl_restore_tail_md(rband); -} diff --git a/lib/ftl/ftl_trace.c b/lib/ftl/ftl_trace.c deleted file mode 100644 index e629e6d99..000000000 --- a/lib/ftl/ftl_trace.c +++ /dev/null @@ -1,353 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/trace.h" - -#include "ftl_core.h" -#include "ftl_trace.h" -#include "ftl_io.h" -#include "ftl_band.h" - -#include "spdk_internal/trace_defs.h" - -#if defined(DEBUG) - -enum ftl_trace_source { - FTL_TRACE_SOURCE_INTERNAL, - FTL_TRACE_SOURCE_USER, - FTL_TRACE_SOURCE_MAX, -}; - -#define FTL_TPOINT_ID(id, src) SPDK_TPOINT_ID(TRACE_GROUP_FTL, (((id) << 1) | (!!(src)))) - -#define FTL_TRACE_BAND_DEFRAG(src) FTL_TPOINT_ID(0, src) -#define FTL_TRACE_BAND_WRITE(src) FTL_TPOINT_ID(1, src) -#define FTL_TRACE_LIMITS(src) FTL_TPOINT_ID(2, src) -#define FTL_TRACE_WBUF_POP(src) FTL_TPOINT_ID(3, src) - -#define FTL_TRACE_READ_SCHEDULE(src) FTL_TPOINT_ID(4, src) -#define FTL_TRACE_READ_SUBMISSION(src) FTL_TPOINT_ID(5, src) -#define FTL_TRACE_READ_COMPLETION_INVALID(src) FTL_TPOINT_ID(6, src) -#define FTL_TRACE_READ_COMPLETION_CACHE(src) FTL_TPOINT_ID(7, src) -#define FTL_TRACE_READ_COMPLETION_DISK(src) FTL_TPOINT_ID(8, src) - -#define FTL_TRACE_MD_READ_SCHEDULE(src) FTL_TPOINT_ID(9, src) -#define FTL_TRACE_MD_READ_SUBMISSION(src) FTL_TPOINT_ID(10, src) -#define FTL_TRACE_MD_READ_COMPLETION(src) FTL_TPOINT_ID(11, src) - -#define FTL_TRACE_WRITE_SCHEDULE(src) FTL_TPOINT_ID(12, src) -#define FTL_TRACE_WRITE_WBUF_FILL(src) FTL_TPOINT_ID(13, src) -#define FTL_TRACE_WRITE_SUBMISSION(src) FTL_TPOINT_ID(14, src) -#define FTL_TRACE_WRITE_COMPLETION(src) FTL_TPOINT_ID(15, src) - -#define FTL_TRACE_MD_WRITE_SCHEDULE(src) FTL_TPOINT_ID(16, src) -#define FTL_TRACE_MD_WRITE_SUBMISSION(src) FTL_TPOINT_ID(17, src) -#define FTL_TRACE_MD_WRITE_COMPLETION(src) FTL_TPOINT_ID(18, src) - -#define FTL_TRACE_ERASE_SUBMISSION(src) FTL_TPOINT_ID(19, src) -#define FTL_TRACE_ERASE_COMPLETION(src) FTL_TPOINT_ID(20, src) - -SPDK_TRACE_REGISTER_FN(ftl_trace_func, "ftl", TRACE_GROUP_FTL) -{ - const char source[] = { 'i', 'u' }; - char descbuf[128]; - int i; - - spdk_trace_register_owner(OWNER_FTL, 'f'); - - for (i = 0; i < FTL_TRACE_SOURCE_MAX; ++i) { - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "band_defrag"); - spdk_trace_register_description(descbuf, FTL_TRACE_BAND_DEFRAG(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "band"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "band_write"); - spdk_trace_register_description(descbuf, FTL_TRACE_BAND_WRITE(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "band"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "limits"); - spdk_trace_register_description(descbuf, FTL_TRACE_LIMITS(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "limits"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "rwb_pop"); - spdk_trace_register_description(descbuf, FTL_TRACE_WBUF_POP(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "md_read_sched"); - spdk_trace_register_description(descbuf, FTL_TRACE_MD_READ_SCHEDULE(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "md_read_submit"); - spdk_trace_register_description(descbuf, FTL_TRACE_MD_READ_SUBMISSION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "md_read_cmpl"); - spdk_trace_register_description(descbuf, FTL_TRACE_MD_READ_COMPLETION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "md_write_sched"); - spdk_trace_register_description(descbuf, FTL_TRACE_MD_WRITE_SCHEDULE(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "md_write_submit"); - spdk_trace_register_description(descbuf, FTL_TRACE_MD_WRITE_SUBMISSION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "md_write_cmpl"); - spdk_trace_register_description(descbuf, FTL_TRACE_MD_WRITE_COMPLETION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "read_sched"); - spdk_trace_register_description(descbuf, FTL_TRACE_READ_SCHEDULE(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "read_submit"); - spdk_trace_register_description(descbuf, FTL_TRACE_READ_SUBMISSION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "read_cmpl_invld"); - spdk_trace_register_description(descbuf, FTL_TRACE_READ_COMPLETION_INVALID(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "read_cmpl_cache"); - spdk_trace_register_description(descbuf, FTL_TRACE_READ_COMPLETION_CACHE(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "read_cmpl_ssd"); - spdk_trace_register_description(descbuf, FTL_TRACE_READ_COMPLETION_DISK(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "write_sched"); - spdk_trace_register_description(descbuf, FTL_TRACE_WRITE_SCHEDULE(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "rwb_fill"); - spdk_trace_register_description(descbuf, FTL_TRACE_WRITE_WBUF_FILL(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "write_submit"); - spdk_trace_register_description(descbuf, FTL_TRACE_WRITE_SUBMISSION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "write_cmpl"); - spdk_trace_register_description(descbuf, FTL_TRACE_WRITE_COMPLETION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "lba"); - - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "erase_submit"); - spdk_trace_register_description(descbuf, FTL_TRACE_ERASE_SUBMISSION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - snprintf(descbuf, sizeof(descbuf), "%c %s", source[i], "erase_cmpl"); - spdk_trace_register_description(descbuf, FTL_TRACE_ERASE_COMPLETION(i), - OWNER_FTL, OBJECT_NONE, 0, - SPDK_TRACE_ARG_TYPE_INT, "addr"); - } -} - -static uint16_t -ftl_trace_io_source(const struct ftl_io *io) -{ - if (io->flags & FTL_IO_INTERNAL) { - return FTL_TRACE_SOURCE_INTERNAL; - } else { - return FTL_TRACE_SOURCE_USER; - } -} - -static uint64_t -ftl_trace_next_id(struct ftl_trace *trace) -{ - assert(trace->id != FTL_TRACE_INVALID_ID); - return __atomic_fetch_add(&trace->id, 1, __ATOMIC_SEQ_CST); -} - -void -ftl_trace_defrag_band(struct spdk_ftl_dev *dev, const struct ftl_band *band) -{ - struct ftl_trace *trace = &dev->stats.trace; - - spdk_trace_record(FTL_TRACE_BAND_DEFRAG(FTL_TRACE_SOURCE_INTERNAL), - ftl_trace_next_id(trace), 0, band->lba_map.num_vld, band->id); -} - -void -ftl_trace_write_band(struct spdk_ftl_dev *dev, const struct ftl_band *band) -{ - struct ftl_trace *trace = &dev->stats.trace; - - spdk_trace_record(FTL_TRACE_BAND_WRITE(FTL_TRACE_SOURCE_INTERNAL), - ftl_trace_next_id(trace), 0, 0, band->id); -} - -void -ftl_trace_lba_io_init(struct spdk_ftl_dev *dev, const struct ftl_io *io) -{ - uint16_t tpoint_id = 0, source; - - assert(io->trace != FTL_TRACE_INVALID_ID); - source = ftl_trace_io_source(io); - - if (io->flags & FTL_IO_MD) { - switch (io->type) { - case FTL_IO_READ: - tpoint_id = FTL_TRACE_MD_READ_SCHEDULE(source); - break; - case FTL_IO_WRITE: - tpoint_id = FTL_TRACE_MD_WRITE_SCHEDULE(source); - break; - default: - assert(0); - } - } else { - switch (io->type) { - case FTL_IO_READ: - tpoint_id = FTL_TRACE_READ_SCHEDULE(source); - break; - case FTL_IO_WRITE: - tpoint_id = FTL_TRACE_WRITE_SCHEDULE(source); - break; - default: - assert(0); - } - } - - spdk_trace_record(tpoint_id, io->trace, io->num_blocks, 0, ftl_io_get_lba(io, 0)); -} - -void -ftl_trace_wbuf_fill(struct spdk_ftl_dev *dev, const struct ftl_io *io) -{ - assert(io->trace != FTL_TRACE_INVALID_ID); - - spdk_trace_record(FTL_TRACE_WRITE_WBUF_FILL(ftl_trace_io_source(io)), io->trace, - 0, 0, ftl_io_current_lba(io)); -} - -void -ftl_trace_wbuf_pop(struct spdk_ftl_dev *dev, const struct ftl_wbuf_entry *entry) -{ - uint16_t tpoint_id; - - assert(entry->trace != FTL_TRACE_INVALID_ID); - - if (entry->io_flags & FTL_IO_INTERNAL) { - tpoint_id = FTL_TRACE_WBUF_POP(FTL_TRACE_SOURCE_INTERNAL); - } else { - tpoint_id = FTL_TRACE_WBUF_POP(FTL_TRACE_SOURCE_USER); - } - - spdk_trace_record(tpoint_id, entry->trace, 0, entry->addr.offset, entry->lba); -} - -void -ftl_trace_completion(struct spdk_ftl_dev *dev, const struct ftl_io *io, - enum ftl_trace_completion completion) -{ - uint16_t tpoint_id = 0, source; - - assert(io->trace != FTL_TRACE_INVALID_ID); - source = ftl_trace_io_source(io); - - if (io->flags & FTL_IO_MD) { - switch (io->type) { - case FTL_IO_READ: - tpoint_id = FTL_TRACE_MD_READ_COMPLETION(source); - break; - case FTL_IO_WRITE: - tpoint_id = FTL_TRACE_MD_WRITE_COMPLETION(source); - break; - default: - assert(0); - } - } else { - switch (io->type) { - case FTL_IO_READ: - switch (completion) { - case FTL_TRACE_COMPLETION_INVALID: - tpoint_id = FTL_TRACE_READ_COMPLETION_INVALID(source); - break; - case FTL_TRACE_COMPLETION_CACHE: - tpoint_id = FTL_TRACE_READ_COMPLETION_CACHE(source); - break; - case FTL_TRACE_COMPLETION_DISK: - tpoint_id = FTL_TRACE_READ_COMPLETION_DISK(source); - break; - } - break; - case FTL_IO_WRITE: - tpoint_id = FTL_TRACE_WRITE_COMPLETION(source); - break; - case FTL_IO_ERASE: - tpoint_id = FTL_TRACE_ERASE_COMPLETION(source); - break; - default: - assert(0); - } - } - - spdk_trace_record(tpoint_id, io->trace, 0, 0, ftl_io_get_lba(io, io->pos - 1)); -} - -void -ftl_trace_submission(struct spdk_ftl_dev *dev, const struct ftl_io *io, struct ftl_addr addr, - size_t addr_cnt) -{ - uint16_t tpoint_id = 0, source; - - assert(io->trace != FTL_TRACE_INVALID_ID); - source = ftl_trace_io_source(io); - - if (io->flags & FTL_IO_MD) { - switch (io->type) { - case FTL_IO_READ: - tpoint_id = FTL_TRACE_MD_READ_SUBMISSION(source); - break; - case FTL_IO_WRITE: - tpoint_id = FTL_TRACE_MD_WRITE_SUBMISSION(source); - break; - default: - assert(0); - } - } else { - switch (io->type) { - case FTL_IO_READ: - tpoint_id = FTL_TRACE_READ_SUBMISSION(source); - break; - case FTL_IO_WRITE: - tpoint_id = FTL_TRACE_WRITE_SUBMISSION(source); - break; - case FTL_IO_ERASE: - tpoint_id = FTL_TRACE_ERASE_SUBMISSION(source); - break; - default: - assert(0); - } - } - - spdk_trace_record(tpoint_id, io->trace, addr_cnt, 0, addr.offset); -} - -void -ftl_trace_limits(struct spdk_ftl_dev *dev, int limit, size_t num_free) -{ - struct ftl_trace *trace = &dev->stats.trace; - - spdk_trace_record(FTL_TRACE_LIMITS(FTL_TRACE_SOURCE_INTERNAL), ftl_trace_next_id(trace), - num_free, limit, 0); -} - -uint64_t -ftl_trace_alloc_id(struct spdk_ftl_dev *dev) -{ - struct ftl_trace *trace = &dev->stats.trace; - - return ftl_trace_next_id(trace); -} - -#endif /* defined(DEBUG) */ diff --git a/lib/ftl/ftl_trace.h b/lib/ftl/ftl_trace.h deleted file mode 100644 index ee1839645..000000000 --- a/lib/ftl/ftl_trace.h +++ /dev/null @@ -1,56 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef FTL_TRACE_H -#define FTL_TRACE_H - -#include "ftl_addr.h" - -#define FTL_TRACE_INVALID_ID ((uint64_t) -1) - -enum ftl_trace_completion { - FTL_TRACE_COMPLETION_INVALID, - FTL_TRACE_COMPLETION_CACHE, - FTL_TRACE_COMPLETION_DISK, -}; - -struct ftl_trace { - /* Monotonically incrementing event id */ - uint64_t id; -}; - -struct spdk_ftl_dev; -struct ftl_trace; -struct ftl_io; -struct ftl_wbuf_entry; -struct ftl_band; - -#if defined(DEBUG) -uint64_t ftl_trace_alloc_id(struct spdk_ftl_dev *dev); -void ftl_trace_defrag_band(struct spdk_ftl_dev *dev, const struct ftl_band *band); -void ftl_trace_write_band(struct spdk_ftl_dev *dev, const struct ftl_band *band); -void ftl_trace_lba_io_init(struct spdk_ftl_dev *dev, const struct ftl_io *io); -void ftl_trace_wbuf_fill(struct spdk_ftl_dev *dev, const struct ftl_io *io); -void ftl_trace_wbuf_pop(struct spdk_ftl_dev *dev, const struct ftl_wbuf_entry *entry); -void ftl_trace_submission(struct spdk_ftl_dev *dev, - const struct ftl_io *io, - struct ftl_addr addr, size_t addr_cnt); -void ftl_trace_completion(struct spdk_ftl_dev *dev, - const struct ftl_io *io, - enum ftl_trace_completion type); -void ftl_trace_limits(struct spdk_ftl_dev *dev, int limit, size_t num_free); -#else /* defined(DEBUG) */ -#define ftl_trace_alloc_id(dev) FTL_TRACE_INVALID_ID -#define ftl_trace_defrag_band(dev, band) -#define ftl_trace_write_band(dev, band) -#define ftl_trace_lba_io_init(dev, io) -#define ftl_trace_wbuf_fill(dev, io) -#define ftl_trace_wbuf_pop(dev, entry) -#define ftl_trace_submission(dev, io, addr, addr_cnt) -#define ftl_trace_completion(dev, io, type) -#define ftl_trace_limits(dev, limits, num_free) -#endif - -#endif /* FTL_TRACE_H */ diff --git a/lib/ftl/spdk_ftl.map b/lib/ftl/spdk_ftl.map deleted file mode 100644 index 141fd01e0..000000000 --- a/lib/ftl/spdk_ftl.map +++ /dev/null @@ -1,14 +0,0 @@ -{ - global: - - # public functions - spdk_ftl_dev_init; - spdk_ftl_dev_free; - spdk_ftl_conf_init_defaults; - spdk_ftl_dev_get_attrs; - spdk_ftl_read; - spdk_ftl_write; - spdk_ftl_flush; - - local: *; -}; diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index 4573cdca0..117218527 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -132,9 +132,6 @@ DEPDIRS-bdev_rbd := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_uring := $(BDEV_DEPS_THREAD) DEPDIRS-bdev_virtio := $(BDEV_DEPS_THREAD) virtio DEPDIRS-bdev_zone_block := $(BDEV_DEPS_THREAD) -ifeq ($(OS),Linux) -DEPDIRS-bdev_ftl := $(BDEV_DEPS_THREAD) ftl -endif # module/event diff --git a/mk/spdk.modules.mk b/mk/spdk.modules.mk index 284bf16a4..51f35a8e2 100644 --- a/mk/spdk.modules.mk +++ b/mk/spdk.modules.mk @@ -47,7 +47,6 @@ endif endif ifeq ($(OS),Linux) -BLOCKDEV_MODULES_LIST += bdev_ftl ftl BLOCKDEV_MODULES_LIST += bdev_aio BLOCKDEV_MODULES_PRIVATE_LIBS += -laio INTR_BLOCKDEV_MODULES_LIST += bdev_aio diff --git a/module/bdev/Makefile b/module/bdev/Makefile index 3618f2a54..7de4abb0b 100644 --- a/module/bdev/Makefile +++ b/module/bdev/Makefile @@ -17,7 +17,7 @@ DIRS-$(CONFIG_REDUCE) += compress DIRS-$(CONFIG_URING) += uring ifeq ($(OS),Linux) -DIRS-y += aio ftl +DIRS-y += aio DIRS-$(CONFIG_ISCSI_INITIATOR) += iscsi DIRS-$(CONFIG_VIRTIO) += virtio DIRS-$(CONFIG_PMDK) += pmem diff --git a/module/bdev/ftl/Makefile b/module/bdev/ftl/Makefile deleted file mode 100644 index d0d7cf4e1..000000000 --- a/module/bdev/ftl/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..) -include $(SPDK_ROOT_DIR)/mk/spdk.common.mk - -SO_VER := 4 -SO_MINOR := 0 - -C_SRCS += bdev_ftl.c bdev_ftl_rpc.c -LIBNAME = bdev_ftl - -SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map - -include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/module/bdev/ftl/bdev_ftl.c b/module/bdev/ftl/bdev_ftl.c deleted file mode 100644 index 15d7bbbc3..000000000 --- a/module/bdev/ftl/bdev_ftl.c +++ /dev/null @@ -1,485 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" -#include "spdk/bdev.h" -#include "spdk/env.h" -#include "spdk/thread.h" -#include "spdk/json.h" -#include "spdk/string.h" -#include "spdk/likely.h" -#include "spdk/util.h" -#include "spdk/string.h" -#include "spdk/ftl.h" -#include "spdk/log.h" - -#include "bdev_ftl.h" - -struct ftl_bdev { - struct spdk_bdev bdev; - - struct spdk_ftl_dev *dev; - - ftl_bdev_init_fn init_cb; - - void *init_arg; -}; - -struct ftl_deferred_init { - struct ftl_bdev_init_opts opts; - - LIST_ENTRY(ftl_deferred_init) entry; -}; - -static LIST_HEAD(, ftl_deferred_init) g_deferred_init = LIST_HEAD_INITIALIZER(g_deferred_init); - -static int bdev_ftl_initialize(void); -static void bdev_ftl_finish(void); -static void bdev_ftl_examine(struct spdk_bdev *bdev); - -static struct spdk_bdev_module g_ftl_if = { - .name = "ftl", - .module_init = bdev_ftl_initialize, - .module_fini = bdev_ftl_finish, - .examine_disk = bdev_ftl_examine, -}; - -SPDK_BDEV_MODULE_REGISTER(ftl, &g_ftl_if) - -static void -bdev_ftl_free_cb(struct spdk_ftl_dev *dev, void *ctx, int status) -{ - struct ftl_bdev *ftl_bdev = ctx; - - spdk_bdev_destruct_done(&ftl_bdev->bdev, status); - free(ftl_bdev->bdev.name); - free(ftl_bdev); -} - -static int -bdev_ftl_destruct(void *ctx) -{ - struct ftl_bdev *ftl_bdev = ctx; - spdk_ftl_dev_free(ftl_bdev->dev, bdev_ftl_free_cb, ftl_bdev); - - /* return 1 to indicate that the destruction is asynchronous */ - return 1; -} - -static void -bdev_ftl_cb(void *arg, int rc) -{ - struct spdk_bdev_io *bdev_io = arg; - enum spdk_bdev_io_status status; - - switch (rc) { - case 0: - status = SPDK_BDEV_IO_STATUS_SUCCESS; - break; - case -ENOMEM: - status = SPDK_BDEV_IO_STATUS_NOMEM; - break; - default: - status = SPDK_BDEV_IO_STATUS_FAILED; - break; - } - - spdk_bdev_io_complete(bdev_io, status); -} - -static void -bdev_ftl_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, - bool success) -{ - struct ftl_bdev *ftl_bdev; - int rc; - - ftl_bdev = bdev_io->bdev->ctxt; - - if (!success) { - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); - return; - } - - rc = spdk_ftl_read(ftl_bdev->dev, - ch, - bdev_io->u.bdev.offset_blocks, - bdev_io->u.bdev.num_blocks, - bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); - - if (spdk_unlikely(rc != 0)) { - spdk_bdev_io_complete(bdev_io, rc); - } -} - -static int -_bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) -{ - struct ftl_bdev *ftl_bdev = (struct ftl_bdev *)bdev_io->bdev->ctxt; - - switch (bdev_io->type) { - case SPDK_BDEV_IO_TYPE_READ: - spdk_bdev_io_get_buf(bdev_io, bdev_ftl_get_buf_cb, - bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); - return 0; - - case SPDK_BDEV_IO_TYPE_WRITE: - return spdk_ftl_write(ftl_bdev->dev, ch, bdev_io->u.bdev.offset_blocks, - bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.iovs, - bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); - - case SPDK_BDEV_IO_TYPE_FLUSH: - return spdk_ftl_flush(ftl_bdev->dev, bdev_ftl_cb, bdev_io); - - case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: - case SPDK_BDEV_IO_TYPE_RESET: - case SPDK_BDEV_IO_TYPE_UNMAP: - default: - return -ENOTSUP; - break; - } -} - -static void -bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) -{ - int rc = _bdev_ftl_submit_request(ch, bdev_io); - - if (spdk_unlikely(rc != 0)) { - spdk_bdev_io_complete(bdev_io, rc); - } -} - -static bool -bdev_ftl_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) -{ - switch (io_type) { - case SPDK_BDEV_IO_TYPE_READ: - case SPDK_BDEV_IO_TYPE_WRITE: - case SPDK_BDEV_IO_TYPE_FLUSH: - return true; - case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: - case SPDK_BDEV_IO_TYPE_RESET: - case SPDK_BDEV_IO_TYPE_UNMAP: - default: - return false; - } -} - -static struct spdk_io_channel * -bdev_ftl_get_io_channel(void *ctx) -{ - struct ftl_bdev *ftl_bdev = ctx; - - return spdk_get_io_channel(ftl_bdev->dev); -} - -static void -_bdev_ftl_write_config_info(struct ftl_bdev *ftl_bdev, struct spdk_json_write_ctx *w) -{ - struct spdk_ftl_attrs attrs = {}; - - spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs); - - spdk_json_write_named_string(w, "base_bdev", attrs.base_bdev); - - if (attrs.cache_bdev) { - spdk_json_write_named_string(w, "cache", attrs.cache_bdev); - } -} - -static void -bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) -{ - struct ftl_bdev *ftl_bdev = bdev->ctxt; - struct spdk_ftl_attrs attrs; - struct spdk_ftl_conf *conf = &attrs.conf; - char uuid[SPDK_UUID_STRING_LEN]; - - spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs); - - spdk_json_write_object_begin(w); - - spdk_json_write_named_string(w, "method", "bdev_ftl_create"); - - spdk_json_write_named_object_begin(w, "params"); - spdk_json_write_named_string(w, "name", ftl_bdev->bdev.name); - - spdk_json_write_named_bool(w, "allow_open_bands", conf->allow_open_bands); - spdk_json_write_named_uint64(w, "overprovisioning", conf->lba_rsvd); - spdk_json_write_named_uint64(w, "limit_crit", conf->limits[SPDK_FTL_LIMIT_CRIT].limit); - spdk_json_write_named_uint64(w, "limit_crit_threshold", conf->limits[SPDK_FTL_LIMIT_CRIT].thld); - spdk_json_write_named_uint64(w, "limit_high", conf->limits[SPDK_FTL_LIMIT_HIGH].limit); - spdk_json_write_named_uint64(w, "limit_high_threshold", conf->limits[SPDK_FTL_LIMIT_HIGH].thld); - spdk_json_write_named_uint64(w, "limit_low", conf->limits[SPDK_FTL_LIMIT_LOW].limit); - spdk_json_write_named_uint64(w, "limit_low_threshold", conf->limits[SPDK_FTL_LIMIT_LOW].thld); - spdk_json_write_named_uint64(w, "limit_start", conf->limits[SPDK_FTL_LIMIT_START].limit); - spdk_json_write_named_uint64(w, "limit_start_threshold", conf->limits[SPDK_FTL_LIMIT_START].thld); - if (conf->l2p_path) { - spdk_json_write_named_string(w, "l2p_path", conf->l2p_path); - } - - spdk_uuid_fmt_lower(uuid, sizeof(uuid), &attrs.uuid); - spdk_json_write_named_string(w, "uuid", uuid); - - _bdev_ftl_write_config_info(ftl_bdev, w); - - spdk_json_write_object_end(w); - spdk_json_write_object_end(w); -} - -static int -bdev_ftl_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) -{ - struct ftl_bdev *ftl_bdev = ctx; - struct spdk_ftl_attrs attrs; - - spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs); - - spdk_json_write_named_object_begin(w, "ftl"); - - _bdev_ftl_write_config_info(ftl_bdev, w); - spdk_json_write_named_string_fmt(w, "num_zones", "%zu", attrs.num_zones); - spdk_json_write_named_string_fmt(w, "zone_size", "%zu", attrs.zone_size); - - /* ftl */ - spdk_json_write_object_end(w); - - return 0; -} - -static const struct spdk_bdev_fn_table ftl_fn_table = { - .destruct = bdev_ftl_destruct, - .submit_request = bdev_ftl_submit_request, - .io_type_supported = bdev_ftl_io_type_supported, - .get_io_channel = bdev_ftl_get_io_channel, - .write_config_json = bdev_ftl_write_config_json, - .dump_info_json = bdev_ftl_dump_info_json, -}; - -static void -bdev_ftl_create_cb(struct spdk_ftl_dev *dev, void *ctx, int status) -{ - struct ftl_bdev *ftl_bdev = ctx; - struct ftl_bdev_info info = {}; - struct spdk_ftl_attrs attrs; - ftl_bdev_init_fn init_cb = ftl_bdev->init_cb; - void *init_arg = ftl_bdev->init_arg; - int rc = -ENODEV; - - if (status) { - SPDK_ERRLOG("Failed to create FTL device (%d)\n", status); - rc = status; - goto error; - } - - spdk_ftl_dev_get_attrs(dev, &attrs); - - ftl_bdev->dev = dev; - ftl_bdev->bdev.product_name = "FTL disk"; - ftl_bdev->bdev.write_cache = 0; - ftl_bdev->bdev.blocklen = attrs.block_size; - ftl_bdev->bdev.blockcnt = attrs.num_blocks; - ftl_bdev->bdev.uuid = attrs.uuid; - - SPDK_DEBUGLOG(bdev_ftl, "Creating bdev %s:\n", ftl_bdev->bdev.name); - SPDK_DEBUGLOG(bdev_ftl, "\tblock_len:\t%zu\n", attrs.block_size); - SPDK_DEBUGLOG(bdev_ftl, "\tnum_blocks:\t%"PRIu64"\n", attrs.num_blocks); - - ftl_bdev->bdev.ctxt = ftl_bdev; - ftl_bdev->bdev.fn_table = &ftl_fn_table; - ftl_bdev->bdev.module = &g_ftl_if; - - if (spdk_bdev_register(&ftl_bdev->bdev)) { - goto error; - } - - info.name = ftl_bdev->bdev.name; - info.uuid = ftl_bdev->bdev.uuid; - - init_cb(&info, init_arg, 0); - return; - -error: - free(ftl_bdev->bdev.name); - free(ftl_bdev); - - init_cb(NULL, init_arg, rc); -} - -static void -bdev_ftl_defer_free(struct ftl_deferred_init *init) -{ - free((char *)init->opts.name); - free((char *)init->opts.base_bdev); - free((char *)init->opts.cache_bdev); - free(init); -} - -static int -bdev_ftl_defer_init(const struct ftl_bdev_init_opts *opts) -{ - struct ftl_deferred_init *init; - - init = calloc(1, sizeof(*init)); - if (!init) { - return -ENOMEM; - } - - init->opts.mode = opts->mode; - init->opts.uuid = opts->uuid; - init->opts.ftl_conf = opts->ftl_conf; - - init->opts.name = strdup(opts->name); - if (!init->opts.name) { - SPDK_ERRLOG("Could not allocate bdev name\n"); - goto error; - } - - init->opts.base_bdev = strdup(opts->base_bdev); - if (!init->opts.base_bdev) { - SPDK_ERRLOG("Could not allocate base bdev name\n"); - goto error; - } - - if (opts->cache_bdev) { - init->opts.cache_bdev = strdup(opts->cache_bdev); - if (!init->opts.cache_bdev) { - SPDK_ERRLOG("Could not allocate cache bdev name\n"); - goto error; - } - } - - LIST_INSERT_HEAD(&g_deferred_init, init, entry); - - return 0; - -error: - bdev_ftl_defer_free(init); - return -ENOMEM; -} - -int -bdev_ftl_create_bdev(const struct ftl_bdev_init_opts *bdev_opts, - ftl_bdev_init_fn cb, void *cb_arg) -{ - struct ftl_bdev *ftl_bdev = NULL; - struct spdk_ftl_dev_init_opts opts = {}; - int rc; - - ftl_bdev = calloc(1, sizeof(*ftl_bdev)); - if (!ftl_bdev) { - SPDK_ERRLOG("Could not allocate ftl_bdev\n"); - return -ENOMEM; - } - - ftl_bdev->bdev.name = strdup(bdev_opts->name); - if (!ftl_bdev->bdev.name) { - rc = -ENOMEM; - goto error_bdev; - } - - if (spdk_bdev_get_by_name(bdev_opts->base_bdev) == NULL || - (bdev_opts->cache_bdev && spdk_bdev_get_by_name(bdev_opts->cache_bdev) == NULL)) { - rc = bdev_ftl_defer_init(bdev_opts); - if (rc == 0) { - rc = -ENODEV; - } - goto error_name; - } - - ftl_bdev->init_cb = cb; - ftl_bdev->init_arg = cb_arg; - - opts.mode = bdev_opts->mode; - opts.uuid = bdev_opts->uuid; - opts.name = ftl_bdev->bdev.name; - opts.base_bdev = bdev_opts->base_bdev; - opts.cache_bdev = bdev_opts->cache_bdev; - opts.conf = &bdev_opts->ftl_conf; - - /* TODO: set threads based on config */ - opts.core_thread = spdk_get_thread(); - - rc = spdk_ftl_dev_init(&opts, bdev_ftl_create_cb, ftl_bdev); - if (rc) { - SPDK_ERRLOG("Could not create FTL device\n"); - goto error_name; - } - - return 0; - -error_name: - free(ftl_bdev->bdev.name); -error_bdev: - free(ftl_bdev); - return rc; -} - -static int -bdev_ftl_initialize(void) -{ - return 0; -} - -void -bdev_ftl_delete_bdev(const char *name, spdk_bdev_unregister_cb cb_fn, void *cb_arg) -{ - int rc; - - rc = spdk_bdev_unregister_by_name(name, &g_ftl_if, cb_fn, cb_arg); - if (rc != 0) { - cb_fn(cb_arg, rc); - } -} - -static void -bdev_ftl_finish(void) -{ -} - -static void -bdev_ftl_create_deferred_cb(const struct ftl_bdev_info *info, void *ctx, int status) -{ - struct ftl_deferred_init *opts = ctx; - - if (status) { - SPDK_ERRLOG("Failed to initialize FTL bdev '%s'\n", opts->opts.name); - } - - bdev_ftl_defer_free(opts); - - spdk_bdev_module_examine_done(&g_ftl_if); -} - -static void -bdev_ftl_examine(struct spdk_bdev *bdev) -{ - struct ftl_deferred_init *opts; - - LIST_FOREACH(opts, &g_deferred_init, entry) { - if (spdk_bdev_get_by_name(opts->opts.base_bdev) == NULL) { - continue; - } - - if (opts->opts.cache_bdev && spdk_bdev_get_by_name(opts->opts.base_bdev) == NULL) { - continue; - } - - LIST_REMOVE(opts, entry); - - /* spdk_bdev_module_examine_done will be called by bdev_ftl_create_deferred_cb */ - if (bdev_ftl_create_bdev(&opts->opts, bdev_ftl_create_deferred_cb, opts)) { - SPDK_ERRLOG("Failed to initialize FTL bdev '%s'\n", opts->opts.name); - bdev_ftl_defer_free(opts); - break; - } - return; - } - - spdk_bdev_module_examine_done(&g_ftl_if); -} - -SPDK_LOG_REGISTER_COMPONENT(bdev_ftl) diff --git a/module/bdev/ftl/bdev_ftl.h b/module/bdev/ftl/bdev_ftl.h deleted file mode 100644 index 9cd929de0..000000000 --- a/module/bdev/ftl/bdev_ftl.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#ifndef SPDK_BDEV_FTL_H -#define SPDK_BDEV_FTL_H - -#include "spdk/stdinc.h" -#include "spdk/bdev_module.h" -#include "spdk/ftl.h" - -struct spdk_bdev; -struct spdk_uuid; - -struct ftl_bdev_info { - const char *name; - struct spdk_uuid uuid; -}; - -struct ftl_bdev_init_opts { - /* Bdev's name */ - const char *name; - /* Base bdev's name */ - const char *base_bdev; - /* Write buffer bdev's name */ - const char *cache_bdev; - /* Bdev's mode */ - uint32_t mode; - /* UUID if device is restored from SSD */ - struct spdk_uuid uuid; - /* FTL library configuration */ - struct spdk_ftl_conf ftl_conf; -}; - -typedef void (*ftl_bdev_init_fn)(const struct ftl_bdev_info *, void *, int); - -int bdev_ftl_create_bdev(const struct ftl_bdev_init_opts *bdev_opts, - ftl_bdev_init_fn cb, void *cb_arg); -void bdev_ftl_delete_bdev(const char *name, spdk_bdev_unregister_cb cb_fn, void *cb_arg); - -#endif /* SPDK_BDEV_FTL_H */ diff --git a/module/bdev/ftl/bdev_ftl_rpc.c b/module/bdev/ftl/bdev_ftl_rpc.c deleted file mode 100644 index 8943f762c..000000000 --- a/module/bdev/ftl/bdev_ftl_rpc.c +++ /dev/null @@ -1,230 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/rpc.h" -#include "spdk/util.h" -#include "spdk/bdev_module.h" -#include "spdk/string.h" -#include "spdk/log.h" - -#include "bdev_ftl.h" - -struct rpc_bdev_ftl_create { - char *name; - char *base_bdev; - char *uuid; - char *cache_bdev; - struct spdk_ftl_conf ftl_conf; -}; - -static void -free_rpc_bdev_ftl_create(struct rpc_bdev_ftl_create *req) -{ - free(req->name); - free(req->base_bdev); - free(req->uuid); - free(req->cache_bdev); - free((char *)req->ftl_conf.l2p_path); -} - -static const struct spdk_json_object_decoder rpc_bdev_ftl_create_decoders[] = { - {"name", offsetof(struct rpc_bdev_ftl_create, name), spdk_json_decode_string}, - {"base_bdev", offsetof(struct rpc_bdev_ftl_create, base_bdev), spdk_json_decode_string}, - {"uuid", offsetof(struct rpc_bdev_ftl_create, uuid), spdk_json_decode_string, true}, - {"cache", offsetof(struct rpc_bdev_ftl_create, cache_bdev), spdk_json_decode_string, true}, - { - "allow_open_bands", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, allow_open_bands), spdk_json_decode_bool, true - }, - { - "overprovisioning", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, lba_rsvd), spdk_json_decode_uint64, true - }, - { - "use_append", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, use_append), spdk_json_decode_bool, true - }, - { - "l2p_path", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, l2p_path), - spdk_json_decode_string, true - }, - { - "limit_crit", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_CRIT]) + - offsetof(struct spdk_ftl_limit, limit), - spdk_json_decode_uint64, true - }, - { - "limit_crit_threshold", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_CRIT]) + - offsetof(struct spdk_ftl_limit, thld), - spdk_json_decode_uint64, true - }, - { - "limit_high", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_HIGH]) + - offsetof(struct spdk_ftl_limit, limit), - spdk_json_decode_uint64, true - }, - { - "limit_high_threshold", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_HIGH]) + - offsetof(struct spdk_ftl_limit, thld), - spdk_json_decode_uint64, true - }, - { - "limit_low", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_LOW]) + - offsetof(struct spdk_ftl_limit, limit), - spdk_json_decode_uint64, true - }, - { - "limit_low_threshold", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_LOW]) + - offsetof(struct spdk_ftl_limit, thld), - spdk_json_decode_uint64, true - }, - { - "limit_start", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_START]) + - offsetof(struct spdk_ftl_limit, limit), - spdk_json_decode_uint64, true - }, - { - "limit_start_threshold", offsetof(struct rpc_bdev_ftl_create, ftl_conf) + - offsetof(struct spdk_ftl_conf, limits[SPDK_FTL_LIMIT_START]) + - offsetof(struct spdk_ftl_limit, thld), - spdk_json_decode_uint64, true - }, -}; - -static void -rpc_bdev_ftl_create_cb(const struct ftl_bdev_info *bdev_info, void *ctx, int status) -{ - struct spdk_jsonrpc_request *request = ctx; - char bdev_uuid[SPDK_UUID_STRING_LEN]; - struct spdk_json_write_ctx *w; - - if (status) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Failed to create FTL bdev: %s", - spdk_strerror(-status)); - return; - } - - w = spdk_jsonrpc_begin_result(request); - spdk_uuid_fmt_lower(bdev_uuid, sizeof(bdev_uuid), &bdev_info->uuid); - spdk_json_write_object_begin(w); - spdk_json_write_named_string(w, "name", bdev_info->name); - spdk_json_write_named_string(w, "uuid", bdev_uuid); - spdk_json_write_object_end(w); - spdk_jsonrpc_end_result(request, w); -} - -static void -rpc_bdev_ftl_create(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_bdev_ftl_create req = {}; - struct ftl_bdev_init_opts opts = {}; - struct spdk_json_write_ctx *w; - int rc; - - spdk_ftl_conf_init_defaults(&req.ftl_conf); - - if (spdk_json_decode_object(params, rpc_bdev_ftl_create_decoders, - SPDK_COUNTOF(rpc_bdev_ftl_create_decoders), - &req)) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - goto invalid; - } - - if (req.cache_bdev && !spdk_bdev_get_by_name(req.cache_bdev)) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "No such bdev: %s", req.cache_bdev); - goto invalid; - } - - opts.name = req.name; - opts.mode = SPDK_FTL_MODE_CREATE; - opts.base_bdev = req.base_bdev; - opts.cache_bdev = req.cache_bdev; - opts.ftl_conf = req.ftl_conf; - - if (req.uuid) { - if (spdk_uuid_parse(&opts.uuid, req.uuid) < 0) { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Failed to parse uuid: %s", - req.uuid); - goto invalid; - } - - if (!spdk_mem_all_zero(&opts.uuid, sizeof(opts.uuid))) { - opts.mode &= ~SPDK_FTL_MODE_CREATE; - } - } - - rc = bdev_ftl_create_bdev(&opts, rpc_bdev_ftl_create_cb, request); - if (rc) { - if (rc == -ENODEV) { - w = spdk_jsonrpc_begin_result(request); - spdk_json_write_string_fmt(w, "FTL bdev: %s creation deferred", req.name); - spdk_jsonrpc_end_result(request, w); - } else { - spdk_jsonrpc_send_error_response_fmt(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, - "Failed to create FTL bdev: %s", - spdk_strerror(-rc)); - } - goto invalid; - } - -invalid: - free_rpc_bdev_ftl_create(&req); -} - -SPDK_RPC_REGISTER("bdev_ftl_create", rpc_bdev_ftl_create, SPDK_RPC_RUNTIME) - -struct rpc_delete_ftl { - char *name; -}; - -static const struct spdk_json_object_decoder rpc_delete_ftl_decoders[] = { - {"name", offsetof(struct rpc_bdev_ftl_create, name), spdk_json_decode_string}, -}; - -static void -rpc_bdev_ftl_delete_cb(void *cb_arg, int bdeverrno) -{ - struct spdk_jsonrpc_request *request = cb_arg; - - if (bdeverrno == 0) { - spdk_jsonrpc_send_bool_response(request, true); - } else { - spdk_jsonrpc_send_error_response(request, bdeverrno, spdk_strerror(-bdeverrno)); - } -} - -static void -rpc_bdev_ftl_delete(struct spdk_jsonrpc_request *request, - const struct spdk_json_val *params) -{ - struct rpc_delete_ftl attrs = {}; - - if (spdk_json_decode_object(params, rpc_delete_ftl_decoders, - SPDK_COUNTOF(rpc_delete_ftl_decoders), - &attrs)) { - spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, - "Invalid parameters"); - goto invalid; - } - - bdev_ftl_delete_bdev(attrs.name, rpc_bdev_ftl_delete_cb, request); -invalid: - free(attrs.name); -} - -SPDK_RPC_REGISTER("bdev_ftl_delete", rpc_bdev_ftl_delete, SPDK_RPC_RUNTIME) diff --git a/test/ftl/bdevperf.sh b/test/ftl/bdevperf.sh deleted file mode 100755 index ca091bffa..000000000 --- a/test/ftl/bdevperf.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -testdir=$(readlink -f $(dirname $0)) -rootdir=$(readlink -f $testdir/../..) -source $rootdir/test/common/autotest_common.sh -source $testdir/common.sh - -tests=('-q 1 -w randwrite -t 4 -o 69632' '-q 128 -w randwrite -t 4 -o 4096' '-q 128 -w verify -t 4 -o 4096') -device=$1 -use_append=$2 - -for ((i = 0; i < ${#tests[@]}; i++)); do - timing_enter "${tests[$i]}" - "$rootdir/test/bdev/bdevperf/bdevperf" -z -T ftl0 ${tests[$i]} --json <(gen_ftl_nvme_conf) & - bdevperf_pid=$! - - trap 'killprocess $bdevperf_pid; exit 1' SIGINT SIGTERM EXIT - waitforlisten $bdevperf_pid - $rpc_py bdev_nvme_attach_controller -b nvme0 -a $device -t pcie - bdev_create_zone nvme0n1 - $rpc_py bdev_ftl_create -b ftl0 -d "$ZONE_DEV" $use_append - - $rootdir/test/bdev/bdevperf/bdevperf.py perform_tests - $rpc_py bdev_ftl_delete -b ftl0 - bdev_delete_zone "$ZONE_DEV" - $rpc_py bdev_nvme_detach_controller nvme0 - killprocess $bdevperf_pid - trap - SIGINT SIGTERM EXIT - timing_exit "${tests[$i]}" -done diff --git a/test/ftl/common.sh b/test/ftl/common.sh deleted file mode 100644 index 7128dd859..000000000 --- a/test/ftl/common.sh +++ /dev/null @@ -1,82 +0,0 @@ -# Common utility functions to be sourced by the libftl test scripts - -function get_chunk_size() { - $SPDK_EXAMPLE_DIR/identify -r "trtype:PCIe traddr:$1" \ - | grep 'Logical blks per chunk' | sed 's/[^0-9]//g' -} - -function get_num_group() { - $SPDK_EXAMPLE_DIR/identify -r "trtype:PCIe traddr:$1" \ - | grep 'Groups' | sed 's/[^0-9]//g' -} - -function get_num_pu() { - $SPDK_EXAMPLE_DIR/identify -r "trtype:PCIe traddr:$1" \ - | grep 'PUs' | sed 's/[^0-9]//g' -} - -function gen_ftl_nvme_conf() { - jq . <<- JSON - { - "subsystems": [ - { - "subsystem": "bdev", - "config": [ - { - "params": { - "nvme_adminq_poll_period_us": 100 - }, - "method": "bdev_nvme_set_options" - } - ] - } - ] - } - JSON -} - -get_ftl_nvme_dev() { - # Find device with LBA matching the FTL_BLOCK_SIZE - local nvmes nvme identify lba - - for nvme in $(nvme_in_userspace); do - identify=$("$SPDK_EXAMPLE_DIR/identify" -r trtype:pcie -r "traddr:$nvme") - # TODO: Skip zoned nvme devices - such setup for FTL is currently not - # supported. See https://github.com/spdk/spdk/issues/1992 for details. - [[ $identity =~ "NVMe ZNS Zone Report" ]] && continue - [[ $identify =~ "Current LBA Format:"\ +"LBA Format #"([0-9]+) ]] - [[ $identify =~ "LBA Format #${BASH_REMATCH[1]}: Data Size:"\ +([0-9]+) ]] - lba=${BASH_REMATCH[1]} - ((lba && lba % FTL_BLOCK_SIZE == 0)) && nvmes+=("$nvme") - done - ((${#nvmes[@]} > 0)) || return 1 - printf '%s\n' "${nvmes[@]}" -} - -bdev_create_zone() { - local base_bdev=$1 - - # TODO: Consider use of ZNSed nvme controllers - "$rpc_py" bdev_zone_block_create \ - -b "$ZONE_DEV" \ - -o "$OPTIMAL_OPEN_ZONES" \ - -z "$ZONE_CAPACITY" \ - -n "$base_bdev" -} - -bdev_delete_zone() { - local zone_dev=$1 - - # TODO: Consider use of ZNSed nvme controllers - "$rpc_py" bdev_zone_block_delete "$zone_dev" -} - -# Optimal number of zones refers to the number of zones that need to be written at the same -# time in order to maximize drive's write bandwidth. -# ZONE_CAPACITY * FTL_BLOCK_SIZE * OPTIMAL_OPEN_ZONES should be <= size of the drive. -FTL_BLOCK_SIZE=4096 -ZONE_CAPACITY=4096 -OPTIMAL_OPEN_ZONES=32 -ZONE_DEV=zone0 - -rpc_py=$rootdir/scripts/rpc.py diff --git a/test/ftl/config/.gitignore b/test/ftl/config/.gitignore deleted file mode 100644 index 5523f29b3..000000000 --- a/test/ftl/config/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -ftl.conf -fio/*.fio diff --git a/test/ftl/config/fio/randw-verify-depth128.fio b/test/ftl/config/fio/randw-verify-depth128.fio deleted file mode 100644 index 9adee6cab..000000000 --- a/test/ftl/config/fio/randw-verify-depth128.fio +++ /dev/null @@ -1,20 +0,0 @@ -[global] -ioengine=spdk_bdev -spdk_json_conf=${FTL_JSON_CONF} -filename=${FTL_BDEV_NAME} -thread=1 -direct=1 -iodepth=128 -rw=randwrite -verify=crc32c -do_verify=1 -verify_dump=0 -verify_state_save=0 -verify_fatal=1 -bs=4k -random_distribution=normal -serialize_overlap=1 -io_size=256M - -[test] -numjobs=1 diff --git a/test/ftl/config/fio/randw-verify-j2.fio b/test/ftl/config/fio/randw-verify-j2.fio deleted file mode 100644 index 4610efa3f..000000000 --- a/test/ftl/config/fio/randw-verify-j2.fio +++ /dev/null @@ -1,25 +0,0 @@ -[global] -ioengine=spdk_bdev -spdk_json_conf=${FTL_JSON_CONF} -filename=${FTL_BDEV_NAME} -thread=1 -direct=1 -iodepth=128 -rw=randwrite -verify=crc32c -do_verify=1 -verify_dump=0 -verify_state_save=0 -verify_backlog=5000 -verify_fatal=1 -bs=4k -random_distribution=normal -serialize_overlap=1 -io_size=256M - -[first_half] -offset=0% -size=50% - -[second_half] -offset=50% diff --git a/test/ftl/config/fio/randw-verify.fio b/test/ftl/config/fio/randw-verify.fio deleted file mode 100644 index edca6c618..000000000 --- a/test/ftl/config/fio/randw-verify.fio +++ /dev/null @@ -1,20 +0,0 @@ -[global] -ioengine=spdk_bdev -spdk_json_conf=${FTL_JSON_CONF} -filename=${FTL_BDEV_NAME} -thread=1 -direct=1 -iodepth=1 -rw=randwrite -size=256M -verify=crc32c -do_verify=1 -verify_dump=0 -verify_state_save=0 -verify_backlog=16 -verify_fatal=1 -bs=68k -random_distribution=normal - -[test] -numjobs=1 diff --git a/test/ftl/config/fio/randw.fio b/test/ftl/config/fio/randw.fio deleted file mode 100644 index b4a16b5b0..000000000 --- a/test/ftl/config/fio/randw.fio +++ /dev/null @@ -1,18 +0,0 @@ -[global] -ioengine=spdk_bdev -spdk_json_conf=${FTL_JSON_CONF} -filename=${FTL_BDEV_NAME} -direct=1 -thread=1 -buffered=0 -size=256M -randrepeat=0 -time_based -norandommap - -[test] -bs=4k -numjobs=1 -rw=randwrite -iodepth=64 -runtime=10 diff --git a/test/ftl/dirty_shutdown.sh b/test/ftl/dirty_shutdown.sh deleted file mode 100755 index 28d7245a7..000000000 --- a/test/ftl/dirty_shutdown.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -testdir=$(readlink -f $(dirname $0)) -rootdir=$(readlink -f $testdir/../..) -source $rootdir/test/common/autotest_common.sh -source $testdir/common.sh - -device=$1 - -restore_kill() { - rm -f "$config" - rm -f "$SPDK_TEST_STORAGE/testfile.md5" - rm -f "$SPDK_TEST_STORAGE/testfile2.md5" - - killprocess $svcpid || true - rmmod nbd || true -} - -trap "restore_kill; exit 1" SIGINT SIGTERM EXIT - -chunk_size=$(get_chunk_size $device) -num_group=$(get_num_group $device) -num_pu=$(get_num_pu $device) -pu_count=$((num_group * num_pu)) -config=$SPDK_TEST_STORAGE/ftl.json - -# Write one band worth of data + one extra chunk -data_size=$((chunk_size * (pu_count + 1))) - -"$SPDK_BIN_DIR/spdk_tgt" --json <(gen_ftl_nvme_conf) & -svcpid=$! -waitforlisten $svcpid - -$rpc_py bdev_nvme_attach_controller -b nvme0 -a $device -t pcie -bdev_create_zone nvme0n1 -ftl_construct_args="bdev_ftl_create -b ftl0 -d $ZONE_DEV -o" - -$rpc_py $ftl_construct_args - -# Load the nbd driver -modprobe nbd -$rpc_py nbd_start_disk ftl0 /dev/nbd0 -waitfornbd nbd0 - -$rpc_py save_config > "$config" - -dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$data_size oflag=dsync -# Calculate checksum of the data written -dd if=/dev/nbd0 bs=4K count=$data_size | md5sum > "$SPDK_TEST_STORAGE/testfile.md5" -$rpc_py nbd_stop_disk /dev/nbd0 - -# Force kill bdev service (dirty shutdown) and start it again -kill -9 $svcpid -rm -f /dev/shm/spdk_tgt_trace.pid$svcpid - -"$SPDK_BIN_DIR/spdk_tgt" --json <(gen_ftl_nvme_conf) -L ftl_init & -svcpid=$! -waitforlisten $svcpid - -$rpc_py load_config < "$config" -waitfornbd nbd0 - -# Write extra data after restore -dd if=/dev/urandom of=/dev/nbd0 bs=4K count=$chunk_size seek=$data_size oflag=dsync -# Save md5 data -dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$data_size | md5sum > "$SPDK_TEST_STORAGE/testfile2.md5" - -# Make sure all data will be read from disk -echo 3 > /proc/sys/vm/drop_caches - -# Verify that the checksum matches and the data is consistent -dd if=/dev/nbd0 bs=4K count=$data_size | md5sum -c "$SPDK_TEST_STORAGE/testfile.md5" -dd if=/dev/nbd0 bs=4K count=$chunk_size skip=$data_size | md5sum -c "$SPDK_TEST_STORAGE/testfile2.md5" - -trap - SIGINT SIGTERM EXIT -restore_kill diff --git a/test/ftl/fio.sh b/test/ftl/fio.sh deleted file mode 100755 index bd77100ce..000000000 --- a/test/ftl/fio.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -testdir=$(readlink -f $(dirname $0)) -rootdir=$(readlink -f $testdir/../..) -source $rootdir/test/common/autotest_common.sh -source $testdir/common.sh - -tests=(randw randw-verify randw-verify-j2 randw-verify-depth128) - -rpc_py=$rootdir/scripts/rpc.py - -fio_kill() { - killprocess $svcpid - rm -f $FTL_JSON_CONF -} - -device=$1 - -if [[ $CONFIG_FIO_PLUGIN != y ]]; then - echo "FIO not available" - exit 1 -fi - -export FTL_BDEV_NAME=ftl0 -export FTL_JSON_CONF=$testdir/config/ftl.json - -trap "fio_kill; exit 1" SIGINT SIGTERM EXIT - -"$SPDK_BIN_DIR/spdk_tgt" --json <(gen_ftl_nvme_conf) & -svcpid=$! -waitforlisten $svcpid - -$rpc_py bdev_nvme_attach_controller -b nvme0 -a $device -t pcie -bdev_create_zone nvme0n1 -$rpc_py bdev_ftl_create -b ftl0 -d "$ZONE_DEV" - -waitforbdev ftl0 - -( - echo '{"subsystems": [' - $rpc_py save_subsystem_config -n bdev - echo ']}' -) > $FTL_JSON_CONF - -killprocess $svcpid -trap - SIGINT SIGTERM EXIT - -for test in "${tests[@]}"; do - timing_enter $test - fio_bdev $testdir/config/fio/$test.fio - timing_exit $test -done - -rm -f $FTL_JSON_CONF diff --git a/test/ftl/ftl.sh b/test/ftl/ftl.sh deleted file mode 100755 index 89afa6e9d..000000000 --- a/test/ftl/ftl.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -testdir=$(readlink -f $(dirname $0)) -rootdir=$(readlink -f $testdir/../..) -source $rootdir/test/common/autotest_common.sh -source $testdir/common.sh - -devices=($(get_ftl_nvme_dev)) device=${devices[0]} - -run_test "ftl_bdevperf" $testdir/bdevperf.sh $device -run_test "ftl_bdevperf_append" $testdir/bdevperf.sh $device --use_append -run_test "ftl_restore" $testdir/restore.sh $device -run_test "ftl_json" $testdir/json.sh $device -run_test "ftl_fio" "$testdir/fio.sh" "$device" diff --git a/test/ftl/json.sh b/test/ftl/json.sh deleted file mode 100755 index 93edf96bc..000000000 --- a/test/ftl/json.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -testdir=$(readlink -f $(dirname $0)) -rootdir=$(readlink -f $testdir/../..) -source $rootdir/test/common/autotest_common.sh -source $testdir/common.sh - -device=$1 - -json_kill() { - killprocess $svcpid -} - -trap "json_kill; exit 1" SIGINT SIGTERM EXIT - -"$SPDK_BIN_DIR/spdk_tgt" --json <(gen_ftl_nvme_conf) & -svcpid=$! -waitforlisten $svcpid - -# Create new bdev from json configuration -$rpc_py bdev_nvme_attach_controller -b nvme0 -a $device -t pcie -bdev_create_zone nvme0n1 -$rootdir/scripts/gen_ftl.sh -n ftl0 -d "$ZONE_DEV" | $rpc_py load_subsystem_config - -waitforbdev ftl0 -uuid=$($rpc_py bdev_get_bdevs | jq -r ".[] | select(.name==\"ftl0\").uuid") - -$rpc_py bdev_ftl_delete -b ftl0 - -# Restore bdev from json configuration -$rootdir/scripts/gen_ftl.sh -n ftl0 -d "$ZONE_DEV" -u $uuid | $rpc_py load_subsystem_config -$rpc_py bdev_ftl_delete -b ftl0 -$rpc_py bdev_nvme_detach_controller nvme0 - -trap - SIGINT SIGTERM EXIT -json_kill diff --git a/test/ftl/restore.sh b/test/ftl/restore.sh deleted file mode 100755 index 0b3fd2744..000000000 --- a/test/ftl/restore.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -testdir=$(readlink -f $(dirname $0)) -rootdir=$(readlink -f $testdir/../..) -source $rootdir/test/common/autotest_common.sh -source $testdir/common.sh - -mount_dir=$(mktemp -d) - -device=$1 -config=$SPDK_TEST_STORAGE/ftl.json - -restore_kill() { - if mount | grep $mount_dir; then - umount $mount_dir - fi - rm -rf $mount_dir - rm -f "$SPDK_TEST_STORAGE/testfile.md5" - rm -f "$SPDK_TEST_STORAGE/testfile2.md5" - rm -f "$config" - - killprocess $svcpid - rmmod nbd || true -} - -trap "restore_kill; exit 1" SIGINT SIGTERM EXIT - -"$SPDK_BIN_DIR/spdk_tgt" --json <(gen_ftl_nvme_conf) & -svcpid=$! -# Wait until spdk_tgt starts -waitforlisten $svcpid - -$rpc_py bdev_nvme_attach_controller -b nvme0 -a $device -t pcie -bdev_create_zone nvme0n1 -ftl_construct_args="bdev_ftl_create -b ftl0 -d $ZONE_DEV" - -$rpc_py $ftl_construct_args - -# Load the nbd driver -modprobe nbd -$rpc_py nbd_start_disk ftl0 /dev/nbd0 -waitfornbd nbd0 - -$rpc_py save_config > "$config" - -# Prepare the disk by creating ext4 fs and putting a file on it -make_filesystem ext4 /dev/nbd0 -mount /dev/nbd0 $mount_dir -dd if=/dev/urandom of=$mount_dir/testfile bs=4K count=4k -sync -mount -o remount /dev/nbd0 $mount_dir -md5sum $mount_dir/testfile > "$SPDK_TEST_STORAGE/testfile.md5" - -# Kill bdev service and start it again -umount $mount_dir -killprocess $svcpid - -"$SPDK_BIN_DIR/spdk_tgt" --json <(gen_ftl_nvme_conf) -L ftl_init & -svcpid=$! -# Wait until spdk_tgt starts -waitforlisten $svcpid - -$rpc_py load_config < "$config" -waitfornbd nbd0 - -mount /dev/nbd0 $mount_dir - -# Write second file, to make sure writer thread has restored properly -dd if=/dev/urandom of=$mount_dir/testfile2 bs=4K count=4k -md5sum $mount_dir/testfile2 > "$SPDK_TEST_STORAGE/testfile2.md5" - -# Make sure second file will be read from disk -echo 3 > /proc/sys/vm/drop_caches - -# Check both files have proper data -md5sum -c "$SPDK_TEST_STORAGE/testfile.md5" -md5sum -c "$SPDK_TEST_STORAGE/testfile2.md5" - -trap - SIGINT SIGTERM EXIT -restore_kill diff --git a/test/unit/lib/Makefile b/test/unit/lib/Makefile index 5269b83ad..312f63509 100644 --- a/test/unit/lib/Makefile +++ b/test/unit/lib/Makefile @@ -12,7 +12,6 @@ DIRS-$(CONFIG_IDXD) += idxd DIRS-$(CONFIG_REDUCE) += reduce ifeq ($(OS),Linux) DIRS-$(CONFIG_VHOST) += vhost -DIRS-y += ftl endif .PHONY: all clean $(DIRS-y) diff --git a/test/unit/lib/ftl/Makefile b/test/unit/lib/ftl/Makefile deleted file mode 100644 index 8a09bca33..000000000 --- a/test/unit/lib/ftl/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../..) -include $(SPDK_ROOT_DIR)/mk/spdk.common.mk - -DIRS-y = ftl_ppa ftl_band.c ftl_reloc.c ftl_wptr ftl_md ftl_io.c - -.PHONY: all clean $(DIRS-y) - -all: $(DIRS-y) -clean: $(DIRS-y) - -include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk diff --git a/test/unit/lib/ftl/common/utils.c b/test/unit/lib/ftl/common/utils.c deleted file mode 100644 index 9dad3c2ce..000000000 --- a/test/unit/lib/ftl/common/utils.c +++ /dev/null @@ -1,144 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/ftl.h" -#include "ftl/ftl_core.h" -#include "thread/thread_internal.h" - -struct base_bdev_geometry { - size_t write_unit_size; - size_t zone_size; - size_t optimal_open_zones; - size_t blockcnt; -}; - -extern struct base_bdev_geometry g_geo; - -struct spdk_ftl_dev *test_init_ftl_dev(const struct base_bdev_geometry *geo); -struct ftl_band *test_init_ftl_band(struct spdk_ftl_dev *dev, size_t id, size_t zone_size); -void test_free_ftl_dev(struct spdk_ftl_dev *dev); -void test_free_ftl_band(struct ftl_band *band); -uint64_t test_offset_from_addr(struct ftl_addr addr, struct ftl_band *band); - -DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *desc), NULL); - -uint64_t -spdk_bdev_get_zone_size(const struct spdk_bdev *bdev) -{ - return g_geo.zone_size; -} - -uint32_t -spdk_bdev_get_optimal_open_zones(const struct spdk_bdev *bdev) -{ - return g_geo.optimal_open_zones; -} - -struct spdk_ftl_dev * -test_init_ftl_dev(const struct base_bdev_geometry *geo) -{ - struct spdk_ftl_dev *dev; - - dev = calloc(1, sizeof(*dev)); - SPDK_CU_ASSERT_FATAL(dev != NULL); - - dev->xfer_size = geo->write_unit_size; - dev->core_thread = spdk_thread_create("unit_test_thread", NULL); - spdk_set_thread(dev->core_thread); - dev->ioch = calloc(1, sizeof(*dev->ioch) - + sizeof(struct ftl_io_channel *)); - dev->num_bands = geo->blockcnt / (geo->zone_size * geo->optimal_open_zones); - dev->bands = calloc(dev->num_bands, sizeof(*dev->bands)); - SPDK_CU_ASSERT_FATAL(dev->bands != NULL); - - dev->lba_pool = spdk_mempool_create("ftl_ut", 2, 0x18000, - SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, - SPDK_ENV_SOCKET_ID_ANY); - SPDK_CU_ASSERT_FATAL(dev->lba_pool != NULL); - - LIST_INIT(&dev->free_bands); - LIST_INIT(&dev->shut_bands); - - return dev; -} - -struct ftl_band * -test_init_ftl_band(struct spdk_ftl_dev *dev, size_t id, size_t zone_size) -{ - struct ftl_band *band; - struct ftl_zone *zone; - - SPDK_CU_ASSERT_FATAL(dev != NULL); - SPDK_CU_ASSERT_FATAL(id < dev->num_bands); - - band = &dev->bands[id]; - band->dev = dev; - band->id = id; - - band->state = FTL_BAND_STATE_CLOSED; - LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); - CIRCLEQ_INIT(&band->zones); - - band->lba_map.vld = spdk_bit_array_create(ftl_get_num_blocks_in_band(dev)); - SPDK_CU_ASSERT_FATAL(band->lba_map.vld != NULL); - - band->zone_buf = calloc(ftl_get_num_punits(dev), sizeof(*band->zone_buf)); - SPDK_CU_ASSERT_FATAL(band->zone_buf != NULL); - - band->reloc_bitmap = spdk_bit_array_create(ftl_get_num_bands(dev)); - SPDK_CU_ASSERT_FATAL(band->reloc_bitmap != NULL); - - for (size_t i = 0; i < ftl_get_num_punits(dev); ++i) { - zone = &band->zone_buf[i]; - zone->info.state = SPDK_BDEV_ZONE_STATE_FULL; - zone->info.zone_id = zone_size * (id * ftl_get_num_punits(dev) + i); - CIRCLEQ_INSERT_TAIL(&band->zones, zone, circleq); - band->num_zones++; - } - - pthread_spin_init(&band->lba_map.lock, PTHREAD_PROCESS_PRIVATE); - return band; -} - -void -test_free_ftl_dev(struct spdk_ftl_dev *dev) -{ - struct spdk_thread *thread; - - SPDK_CU_ASSERT_FATAL(dev != NULL); - free(dev->ioch); - - thread = dev->core_thread; - - spdk_set_thread(thread); - spdk_thread_exit(thread); - while (!spdk_thread_is_exited(thread)) { - spdk_thread_poll(thread, 0, 0); - } - spdk_thread_destroy(thread); - spdk_mempool_free(dev->lba_pool); - free(dev->bands); - free(dev); -} - -void -test_free_ftl_band(struct ftl_band *band) -{ - SPDK_CU_ASSERT_FATAL(band != NULL); - spdk_bit_array_free(&band->lba_map.vld); - spdk_bit_array_free(&band->reloc_bitmap); - free(band->zone_buf); - spdk_dma_free(band->lba_map.dma_buf); -} - -uint64_t -test_offset_from_addr(struct ftl_addr addr, struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - CU_ASSERT_EQUAL(ftl_addr_get_band(dev, addr), band->id); - - return addr.offset - band->id * ftl_get_num_blocks_in_band(dev); -} diff --git a/test/unit/lib/ftl/ftl_band.c/Makefile b/test/unit/lib/ftl/ftl_band.c/Makefile deleted file mode 100644 index e378d3317..000000000 --- a/test/unit/lib/ftl/ftl_band.c/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) - -TEST_FILE = ftl_band_ut.c - -include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c b/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c deleted file mode 100644 index 53c097139..000000000 --- a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c +++ /dev/null @@ -1,365 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" - -#include "spdk_cunit.h" -#include "common/lib/test_env.c" - -#include "ftl/ftl_core.c" -#include "ftl/ftl_band.c" -#include "../common/utils.c" - -#define TEST_BAND_IDX 68 -#define TEST_LBA 0x68676564 - -struct base_bdev_geometry g_geo = { - .write_unit_size = 16, - .optimal_open_zones = 9, - .zone_size = 100, - .blockcnt = 1500 * 100 * 8, -}; - -static struct spdk_ftl_dev *g_dev; -static struct ftl_band *g_band; - -#if defined(DEBUG) -DEFINE_STUB(ftl_band_validate_md, bool, (struct ftl_band *band), true); -DEFINE_STUB_V(ftl_trace_limits, (struct spdk_ftl_dev *dev, int limit, size_t num_free)); -DEFINE_STUB_V(ftl_trace_completion, (struct spdk_ftl_dev *dev, const struct ftl_io *io, - enum ftl_trace_completion completion)); -DEFINE_STUB_V(ftl_trace_defrag_band, (struct spdk_ftl_dev *dev, const struct ftl_band *band)); -DEFINE_STUB_V(ftl_trace_wbuf_fill, (struct spdk_ftl_dev *dev, const struct ftl_io *io)); -DEFINE_STUB_V(ftl_trace_wbuf_pop, (struct spdk_ftl_dev *dev, const struct ftl_wbuf_entry *entry)); -DEFINE_STUB_V(ftl_trace_write_band, (struct spdk_ftl_dev *dev, const struct ftl_band *band)); -DEFINE_STUB_V(ftl_trace_submission, (struct spdk_ftl_dev *dev, const struct ftl_io *io, - struct ftl_addr addr, size_t addr_cnt)); -#endif -DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io)); -DEFINE_STUB(spdk_bdev_get_block_size, uint32_t, (const struct spdk_bdev *bdev), 512); -DEFINE_STUB(spdk_bdev_get_name, const char *, (const struct spdk_bdev *bdev), "test"); -DEFINE_STUB(spdk_bdev_get_num_blocks, uint64_t, (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_get_media_events, size_t, - (struct spdk_bdev_desc *bdev_desc, struct spdk_bdev_media_event *events, - size_t max_events), 0); -DEFINE_STUB(spdk_bdev_get_md_size, uint32_t, (const struct spdk_bdev *bdev), 8); -DEFINE_STUB(spdk_bdev_io_get_append_location, uint64_t, (struct spdk_bdev_io *bdev_io), 0); -DEFINE_STUB(spdk_bdev_write_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, - void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_write_blocks_with_md, int, (struct spdk_bdev_desc *desc, - struct spdk_io_channel *ch, void *buf, void *md, uint64_t offset_blocks, - uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_read_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_write_zeroes_blocks, int, - (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_writev_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_zone_appendv, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - struct iovec *iov, int iovcnt, uint64_t zone_id, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_zone_management, int, (struct spdk_bdev_desc *desc, - struct spdk_io_channel *ch, - uint64_t zone_id, enum spdk_bdev_zone_action action, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); - -DEFINE_STUB_V(ftl_io_advance, (struct ftl_io *io, size_t num_blocks)); -DEFINE_STUB_V(ftl_io_call_foreach_child, - (struct ftl_io *io, int (*callback)(struct ftl_io *))); -DEFINE_STUB(ftl_io_channel_get_ctx, struct ftl_io_channel *, - (struct spdk_io_channel *ioch), NULL); -DEFINE_STUB_V(ftl_io_complete, (struct ftl_io *io)); -DEFINE_STUB(ftl_io_current_lba, uint64_t, (const struct ftl_io *io), 0); -DEFINE_STUB_V(ftl_io_dec_req, (struct ftl_io *io)); -DEFINE_STUB(ftl_io_erase_init, struct ftl_io *, - (struct ftl_band *band, size_t num_blocks, ftl_io_fn cb), NULL); -DEFINE_STUB_V(ftl_io_fail, (struct ftl_io *io, int status)); -DEFINE_STUB_V(ftl_io_free, (struct ftl_io *io)); -DEFINE_STUB(ftl_io_get_lba, uint64_t, - (const struct ftl_io *io, size_t offset), 0); -DEFINE_STUB_V(ftl_io_inc_req, (struct ftl_io *io)); -DEFINE_STUB(ftl_io_init_internal, struct ftl_io *, - (const struct ftl_io_init_opts *opts), NULL); -DEFINE_STUB_V(ftl_io_reset, (struct ftl_io *io)); -DEFINE_STUB(ftl_io_iovec_addr, void *, (struct ftl_io *io), NULL); -DEFINE_STUB(ftl_io_iovec_len_left, size_t, (struct ftl_io *io), 0); -DEFINE_STUB_V(ftl_io_shrink_iovec, (struct ftl_io *io, size_t num_blocks)); -DEFINE_STUB(ftl_io_wbuf_init, struct ftl_io *, - (struct spdk_ftl_dev *dev, struct ftl_addr addr, - struct ftl_band *band, struct ftl_batch *batch, ftl_io_fn cb), NULL); -DEFINE_STUB(ftl_io_user_init, struct ftl_io *, - (struct spdk_io_channel *ioch, uint64_t lba, size_t num_blocks, - struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, - void *cb_arg, int type), NULL); - -DEFINE_STUB(ftl_iovec_num_blocks, size_t, - (struct iovec *iov, size_t iov_cnt), 0); -DEFINE_STUB(ftl_reloc, bool, (struct ftl_reloc *reloc), false); -DEFINE_STUB_V(ftl_reloc_add, (struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, - size_t num_blocks, int prio, bool defrag)); -DEFINE_STUB(ftl_reloc_is_defrag_active, bool, (const struct ftl_reloc *reloc), false); -DEFINE_STUB(ftl_reloc_is_halted, bool, (const struct ftl_reloc *reloc), false); - -#ifdef SPDK_CONFIG_PMDK -DEFINE_STUB_V(pmem_persist, (const void *addr, size_t len)); -#endif - -static void -setup_band(void) -{ - int rc; - - g_dev = test_init_ftl_dev(&g_geo); - g_band = test_init_ftl_band(g_dev, TEST_BAND_IDX, g_geo.zone_size); - rc = ftl_band_alloc_lba_map(g_band); - CU_ASSERT_EQUAL_FATAL(rc, 0); -} - -static void -cleanup_band(void) -{ - test_free_ftl_band(g_band); - test_free_ftl_dev(g_dev); -} - -static struct ftl_addr -addr_from_punit(uint64_t punit) -{ - struct ftl_addr addr = {}; - - addr.offset = punit * g_geo.zone_size; - return addr; -} - -static void -test_band_block_offset_from_addr_base(void) -{ - struct ftl_addr addr; - uint64_t offset, i, flat_lun = 0; - - setup_band(); - for (i = 0; i < ftl_get_num_punits(g_dev); ++i) { - addr = addr_from_punit(i); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - - offset = ftl_band_block_offset_from_addr(g_band, addr); - CU_ASSERT_EQUAL(offset, flat_lun * ftl_get_num_blocks_in_zone(g_dev)); - flat_lun++; - } - cleanup_band(); -} - -static void -test_band_block_offset_from_addr_offset(void) -{ - struct ftl_addr addr; - uint64_t offset, expect, i, j; - - setup_band(); - for (i = 0; i < ftl_get_num_punits(g_dev); ++i) { - for (j = 0; j < g_geo.zone_size; ++j) { - addr = addr_from_punit(i); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev) + j; - - offset = ftl_band_block_offset_from_addr(g_band, addr); - - expect = test_offset_from_addr(addr, g_band); - CU_ASSERT_EQUAL(offset, expect); - } - } - cleanup_band(); -} - -static void -test_band_addr_from_block_offset(void) -{ - struct ftl_addr addr, expect; - uint64_t offset, i, j; - - setup_band(); - for (i = 0; i < ftl_get_num_punits(g_dev); ++i) { - for (j = 0; j < g_geo.zone_size; ++j) { - expect = addr_from_punit(i); - expect.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev) + j; - - offset = ftl_band_block_offset_from_addr(g_band, expect); - addr = ftl_band_addr_from_block_offset(g_band, offset); - - CU_ASSERT_EQUAL(addr.offset, expect.offset); - } - } - cleanup_band(); -} - -static void -test_band_set_addr(void) -{ - struct ftl_lba_map *lba_map; - struct ftl_addr addr; - uint64_t offset = 0; - - setup_band(); - lba_map = &g_band->lba_map; - addr = addr_from_punit(0); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - - CU_ASSERT_EQUAL(lba_map->num_vld, 0); - - offset = test_offset_from_addr(addr, g_band); - - ftl_band_set_addr(g_band, TEST_LBA, addr); - CU_ASSERT_EQUAL(lba_map->num_vld, 1); - CU_ASSERT_EQUAL(lba_map->map[offset], TEST_LBA); - CU_ASSERT_TRUE(spdk_bit_array_get(lba_map->vld, offset)); - - addr.offset += g_geo.zone_size; - offset = test_offset_from_addr(addr, g_band); - ftl_band_set_addr(g_band, TEST_LBA + 1, addr); - CU_ASSERT_EQUAL(lba_map->num_vld, 2); - CU_ASSERT_EQUAL(lba_map->map[offset], TEST_LBA + 1); - CU_ASSERT_TRUE(spdk_bit_array_get(lba_map->vld, offset)); - addr.offset -= g_geo.zone_size; - offset = test_offset_from_addr(addr, g_band); - CU_ASSERT_TRUE(spdk_bit_array_get(lba_map->vld, offset)); - cleanup_band(); -} - -static void -test_invalidate_addr(void) -{ - struct ftl_lba_map *lba_map; - struct ftl_addr addr; - uint64_t offset[2]; - - setup_band(); - lba_map = &g_band->lba_map; - addr = addr_from_punit(0); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - offset[0] = test_offset_from_addr(addr, g_band); - - ftl_band_set_addr(g_band, TEST_LBA, addr); - CU_ASSERT_EQUAL(lba_map->num_vld, 1); - CU_ASSERT_TRUE(spdk_bit_array_get(lba_map->vld, offset[0])); - ftl_invalidate_addr(g_band->dev, addr); - CU_ASSERT_EQUAL(lba_map->num_vld, 0); - CU_ASSERT_FALSE(spdk_bit_array_get(lba_map->vld, offset[0])); - - offset[0] = test_offset_from_addr(addr, g_band); - ftl_band_set_addr(g_band, TEST_LBA, addr); - addr.offset += g_geo.zone_size; - offset[1] = test_offset_from_addr(addr, g_band); - ftl_band_set_addr(g_band, TEST_LBA + 1, addr); - CU_ASSERT_EQUAL(lba_map->num_vld, 2); - CU_ASSERT_TRUE(spdk_bit_array_get(lba_map->vld, offset[0])); - CU_ASSERT_TRUE(spdk_bit_array_get(lba_map->vld, offset[1])); - ftl_invalidate_addr(g_band->dev, addr); - CU_ASSERT_EQUAL(lba_map->num_vld, 1); - CU_ASSERT_TRUE(spdk_bit_array_get(lba_map->vld, offset[0])); - CU_ASSERT_FALSE(spdk_bit_array_get(lba_map->vld, offset[1])); - cleanup_band(); -} - -static void -test_next_xfer_addr(void) -{ - struct ftl_addr addr, result, expect; - - setup_band(); - /* Verify simple one block increment */ - addr = addr_from_punit(0); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - expect = addr; - expect.offset += 1; - - result = ftl_band_next_xfer_addr(g_band, addr, 1); - CU_ASSERT_EQUAL(result.offset, expect.offset); - - /* Verify jumping between zones */ - expect = addr_from_punit(1); - expect.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - result = ftl_band_next_xfer_addr(g_band, addr, g_dev->xfer_size); - CU_ASSERT_EQUAL(result.offset, expect.offset); - - /* Verify jumping works with unaligned offsets */ - expect = addr_from_punit(1); - expect.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev) + 3; - result = ftl_band_next_xfer_addr(g_band, addr, g_dev->xfer_size + 3); - CU_ASSERT_EQUAL(result.offset, expect.offset); - - /* Verify jumping from last zone to the first one */ - expect = addr_from_punit(0); - expect.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev) + g_dev->xfer_size; - addr = addr_from_punit(ftl_get_num_punits(g_dev) - 1); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - result = ftl_band_next_xfer_addr(g_band, addr, g_dev->xfer_size); - CU_ASSERT_EQUAL(result.offset, expect.offset); - - /* Verify jumping from last zone to the first one with unaligned offset */ - expect = addr_from_punit(0); - expect.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - expect.offset += g_dev->xfer_size + 2; - addr = addr_from_punit(ftl_get_num_punits(g_dev) - 1); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - result = ftl_band_next_xfer_addr(g_band, addr, g_dev->xfer_size + 2); - CU_ASSERT_EQUAL(result.offset, expect.offset); - - /* Verify large offset spanning across the whole band multiple times */ - expect = addr_from_punit(0); - expect.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - expect.offset += g_dev->xfer_size * 5 + 4; - addr = addr_from_punit(0); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - addr.offset += g_dev->xfer_size * 2 + 1; - result = ftl_band_next_xfer_addr(g_band, addr, 3 * g_dev->xfer_size * - ftl_get_num_punits(g_dev) + 3); - CU_ASSERT_EQUAL(result.offset, expect.offset); - - /* Remove one zone and verify it's skipped properly */ - g_band->zone_buf[1].info.state = SPDK_BDEV_ZONE_STATE_OFFLINE; - CIRCLEQ_REMOVE(&g_band->zones, &g_band->zone_buf[1], circleq); - g_band->num_zones--; - expect = addr_from_punit(2); - expect.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - expect.offset += g_dev->xfer_size * 5 + 4; - addr = addr_from_punit(0); - addr.offset += TEST_BAND_IDX * ftl_get_num_blocks_in_band(g_dev); - addr.offset += g_dev->xfer_size * 2 + 1; - result = ftl_band_next_xfer_addr(g_band, addr, 3 * g_dev->xfer_size * - (ftl_get_num_punits(g_dev) - 1) + g_dev->xfer_size + 3); - CU_ASSERT_EQUAL(result.offset, expect.offset); - cleanup_band(); -} - -int -main(int argc, char **argv) -{ - CU_pSuite suite = NULL; - unsigned int num_failures; - - CU_set_error_action(CUEA_ABORT); - CU_initialize_registry(); - - suite = CU_add_suite("ftl_band_suite", NULL, NULL); - - - CU_ADD_TEST(suite, test_band_block_offset_from_addr_base); - CU_ADD_TEST(suite, test_band_block_offset_from_addr_offset); - CU_ADD_TEST(suite, test_band_addr_from_block_offset); - CU_ADD_TEST(suite, test_band_set_addr); - CU_ADD_TEST(suite, test_invalidate_addr); - CU_ADD_TEST(suite, test_next_xfer_addr); - - CU_basic_set_mode(CU_BRM_VERBOSE); - CU_basic_run_tests(); - num_failures = CU_get_number_of_failures(); - CU_cleanup_registry(); - - return num_failures; -} diff --git a/test/unit/lib/ftl/ftl_io.c/Makefile b/test/unit/lib/ftl/ftl_io.c/Makefile deleted file mode 100644 index 769707b68..000000000 --- a/test/unit/lib/ftl/ftl_io.c/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) - -TEST_FILE = ftl_io_ut.c - -include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c b/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c deleted file mode 100644 index b386d2306..000000000 --- a/test/unit/lib/ftl/ftl_io.c/ftl_io_ut.c +++ /dev/null @@ -1,1110 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" -#include "thread/thread_internal.h" - -#include "spdk_cunit.h" -#include "common/lib/ut_multithread.c" - -#include "ftl/ftl_io.c" -#include "ftl/ftl_init.c" -#include "ftl/ftl_core.c" -#include "ftl/ftl_band.c" - -DEFINE_STUB(spdk_bdev_io_get_append_location, uint64_t, (struct spdk_bdev_io *bdev_io), 0); -DEFINE_STUB_V(spdk_bdev_close, (struct spdk_bdev_desc *desc)); -DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *desc), NULL); -DEFINE_STUB(spdk_bdev_get_optimal_open_zones, uint32_t, (const struct spdk_bdev *b), 1); -DEFINE_STUB(spdk_bdev_get_by_name, struct spdk_bdev *, (const char *bdev_name), NULL); -DEFINE_STUB(spdk_bdev_is_md_separate, bool, (const struct spdk_bdev *bdev), false); -DEFINE_STUB(spdk_bdev_is_zoned, bool, (const struct spdk_bdev *bdev), false); -DEFINE_STUB(spdk_bdev_zone_appendv, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - struct iovec *iov, int iovcnt, uint64_t zone_id, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_get_zone_size, uint64_t, (const struct spdk_bdev *b), 1024); -DEFINE_STUB(spdk_bdev_zone_management, int, (struct spdk_bdev_desc *desc, - struct spdk_io_channel *ch, uint64_t zone_id, enum spdk_bdev_zone_action action, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io)); -DEFINE_STUB(spdk_bdev_get_buf_align, size_t, (const struct spdk_bdev *bdev), 64); -DEFINE_STUB(spdk_bdev_get_dif_type, enum spdk_dif_type, - (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_get_name, const char *, (const struct spdk_bdev *bdev), "test"); -DEFINE_STUB(spdk_bdev_get_write_unit_size, uint32_t, - (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_io_type_supported, bool, (struct spdk_bdev *bdev, - enum spdk_bdev_io_type io_type), true); -DEFINE_STUB(spdk_bdev_module_claim_bdev, int, - (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, - struct spdk_bdev_module *module), 0); -DEFINE_STUB(spdk_bdev_open_ext, int, - (const char *bdev_name, bool write, spdk_bdev_event_cb_t event_cb, - void *event_ctx, struct spdk_bdev_desc **desc), 0); -DEFINE_STUB(spdk_bdev_read_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_write_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, - void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_write_blocks_with_md, int, (struct spdk_bdev_desc *desc, - struct spdk_io_channel *ch, void *buf, void *md, uint64_t offset_blocks, - uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_writev_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_get_num_blocks, uint64_t, (const struct spdk_bdev *bdev), 1024); -DEFINE_STUB(spdk_bdev_get_md_size, uint32_t, (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_get_block_size, uint32_t, (const struct spdk_bdev *bdev), 4096); -DEFINE_STUB(spdk_bdev_get_media_events, size_t, - (struct spdk_bdev_desc *bdev_desc, struct spdk_bdev_media_event *events, - size_t max_events), 0); -DEFINE_STUB_V(spdk_bdev_module_release_bdev, (struct spdk_bdev *bdev)); -DEFINE_STUB(spdk_bdev_write_zeroes_blocks, int, - (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_get_zone_info, int, - (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - uint64_t zone_id, size_t num_zones, struct spdk_bdev_zone_info *info, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_mempool_create_ctor, struct spdk_mempool *, - (const char *name, size_t count, size_t ele_size, size_t cache_size, - int socket_id, spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg), NULL); -DEFINE_STUB(spdk_mempool_obj_iter, uint32_t, - (struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, void *obj_cb_arg), 0); -DEFINE_STUB(ftl_reloc, bool, (struct ftl_reloc *reloc), false); -DEFINE_STUB_V(ftl_reloc_add, (struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, - size_t num_blocks, int prio, bool defrag)); -DEFINE_STUB_V(ftl_reloc_free, (struct ftl_reloc *reloc)); -DEFINE_STUB_V(ftl_reloc_halt, (struct ftl_reloc *reloc)); -DEFINE_STUB(ftl_reloc_init, struct ftl_reloc *, (struct spdk_ftl_dev *dev), NULL); -DEFINE_STUB(ftl_reloc_is_defrag_active, bool, (const struct ftl_reloc *reloc), false); -DEFINE_STUB(ftl_reloc_is_halted, bool, (const struct ftl_reloc *reloc), false); -DEFINE_STUB_V(ftl_reloc_resume, (struct ftl_reloc *reloc)); -DEFINE_STUB(ftl_restore_device, int, - (struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg), 0); -DEFINE_STUB(ftl_restore_md, int, - (struct spdk_ftl_dev *dev, ftl_restore_fn cb, void *cb_arg), 0); -DEFINE_STUB_V(ftl_restore_nv_cache, - (struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg)); - -#if defined(FTL_META_DEBUG) -DEFINE_STUB(ftl_band_validate_md, bool, (struct ftl_band *band), true); -#endif -#if defined(DEBUG) -DEFINE_STUB_V(ftl_trace_defrag_band, (struct spdk_ftl_dev *dev, const struct ftl_band *band)); -DEFINE_STUB_V(ftl_trace_submission, (struct spdk_ftl_dev *dev, const struct ftl_io *io, - struct ftl_addr addr, size_t addr_cnt)); -DEFINE_STUB_V(ftl_trace_lba_io_init, (struct spdk_ftl_dev *dev, const struct ftl_io *io)); -DEFINE_STUB_V(ftl_trace_limits, (struct spdk_ftl_dev *dev, int limit, size_t num_free)); -DEFINE_STUB(ftl_trace_alloc_id, uint64_t, (struct spdk_ftl_dev *dev), 0); -DEFINE_STUB_V(ftl_trace_completion, (struct spdk_ftl_dev *dev, const struct ftl_io *io, - enum ftl_trace_completion type)); -DEFINE_STUB_V(ftl_trace_wbuf_fill, (struct spdk_ftl_dev *dev, const struct ftl_io *io)); -DEFINE_STUB_V(ftl_trace_wbuf_pop, (struct spdk_ftl_dev *dev, const struct ftl_wbuf_entry *entry)); -DEFINE_STUB_V(ftl_trace_write_band, (struct spdk_ftl_dev *dev, const struct ftl_band *band)); -#endif -#if defined(FTL_META_DEBUG) -DEFINE_STUB_V(ftl_dev_dump_bands, (struct spdk_ftl_dev *dev)); -#endif -#if defined(FTL_DUMP_STATS) -DEFINE_STUB_V(ftl_dev_dump_stats, (const struct spdk_ftl_dev *dev)); -#endif - -#ifdef SPDK_CONFIG_PMDK -DEFINE_STUB(pmem_map_file, void *, - (const char *path, size_t len, int flags, mode_t mode, - size_t *mapped_lenp, int *is_pmemp), NULL); -DEFINE_STUB(pmem_unmap, int, (void *addr, size_t len), 0); -DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL); -#endif - -struct spdk_io_channel * -spdk_bdev_get_io_channel(struct spdk_bdev_desc *bdev_desc) -{ - return spdk_get_io_channel(bdev_desc); -} - -static int -channel_create_cb(void *io_device, void *ctx) -{ - return 0; -} - -static void -channel_destroy_cb(void *io_device, void *ctx) -{} - -static struct spdk_ftl_dev * -setup_device(uint32_t num_threads, uint32_t xfer_size) -{ - struct spdk_ftl_dev *dev; - struct _ftl_io_channel *_ioch; - struct ftl_io_channel *ioch; - int rc; - - allocate_threads(num_threads); - set_thread(0); - - dev = calloc(1, sizeof(*dev)); - SPDK_CU_ASSERT_FATAL(dev != NULL); - - dev->core_thread = spdk_get_thread(); - dev->ioch = calloc(1, sizeof(*_ioch) + sizeof(struct spdk_io_channel)); - SPDK_CU_ASSERT_FATAL(dev->ioch != NULL); - - _ioch = (struct _ftl_io_channel *)(dev->ioch + 1); - ioch = _ioch->ioch = calloc(1, sizeof(*ioch)); - SPDK_CU_ASSERT_FATAL(ioch != NULL); - - ioch->elem_size = sizeof(struct ftl_md_io); - ioch->io_pool = spdk_mempool_create("io-pool", 4096, ioch->elem_size, 0, 0); - - SPDK_CU_ASSERT_FATAL(ioch->io_pool != NULL); - - dev->conf = g_default_conf; - dev->xfer_size = xfer_size; - dev->base_bdev_desc = (struct spdk_bdev_desc *)0xdeadbeef; - spdk_io_device_register(dev->base_bdev_desc, channel_create_cb, channel_destroy_cb, 0, NULL); - - rc = ftl_dev_init_io_channel(dev); - CU_ASSERT_EQUAL(rc, 0); - - return dev; -} - -static void -free_device(struct spdk_ftl_dev *dev) -{ - struct ftl_io_channel *ioch; - - ioch = ftl_io_channel_get_ctx(dev->ioch); - spdk_mempool_free(ioch->io_pool); - free(ioch); - - spdk_io_device_unregister(dev, NULL); - spdk_io_device_unregister(dev->base_bdev_desc, NULL); - free_threads(); - - free(dev->ioch_array); - free(dev->iov_buf); - free(dev->ioch); - free(dev); -} - -static void -setup_io(struct ftl_io *io, struct spdk_ftl_dev *dev, ftl_io_fn cb, void *ctx) -{ - io->dev = dev; - io->cb_fn = cb; - io->cb_ctx = ctx; -} - -static struct ftl_io * -alloc_io(struct spdk_ftl_dev *dev, ftl_io_fn cb, void *ctx) -{ - struct ftl_io *io; - - io = ftl_io_alloc(dev->ioch); - SPDK_CU_ASSERT_FATAL(io != NULL); - setup_io(io, dev, cb, ctx); - - return io; -} - -static void -io_complete_cb(struct ftl_io *io, void *ctx, int status) -{ - *(int *)ctx = status; -} - -static void -test_completion(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_io_channel *ioch; - struct ftl_io *io; - int req, status = 0; - size_t pool_size; - - dev = setup_device(1, 16); - ioch = ftl_io_channel_get_ctx(dev->ioch); - pool_size = spdk_mempool_count(ioch->io_pool); - - io = alloc_io(dev, io_complete_cb, &status); - io->status = -EIO; - -#define NUM_REQUESTS 16 - for (req = 0; req < NUM_REQUESTS; ++req) { - ftl_io_inc_req(io); - CU_ASSERT_FALSE(ftl_io_done(io)); - } - - CU_ASSERT_EQUAL(io->req_cnt, NUM_REQUESTS); - - for (req = 0; req < (NUM_REQUESTS - 1); ++req) { - ftl_io_dec_req(io); - CU_ASSERT_FALSE(ftl_io_done(io)); - } - - CU_ASSERT_EQUAL(io->req_cnt, 1); - - ftl_io_dec_req(io); - CU_ASSERT_TRUE(ftl_io_done(io)); - - ftl_io_complete(io); - CU_ASSERT_EQUAL(status, -EIO); - - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - free_device(dev); -} - -static void -test_alloc_free(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_io_channel *ioch; - struct ftl_io *parent, *child; - int parent_status = -1; - size_t pool_size; - - dev = setup_device(1, 16); - ioch = ftl_io_channel_get_ctx(dev->ioch); - pool_size = spdk_mempool_count(ioch->io_pool); - - parent = alloc_io(dev, io_complete_cb, &parent_status); - SPDK_CU_ASSERT_FATAL(parent != NULL); - child = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child != NULL); - - ftl_io_free(child); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size - 1); - - child = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child != NULL); - ftl_io_complete(child); - CU_ASSERT_EQUAL(parent_status, -1); - ftl_io_complete(parent); - CU_ASSERT_EQUAL(parent_status, 0); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - parent_status = -1; - parent = alloc_io(dev, io_complete_cb, &parent_status); - SPDK_CU_ASSERT_FATAL(parent != NULL); - child = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child != NULL); - - ftl_io_free(child); - CU_ASSERT_EQUAL(parent_status, -1); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size - 1); - ftl_io_complete(parent); - CU_ASSERT_EQUAL(parent_status, 0); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - free_device(dev); -} - -static void -test_child_requests(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_io_channel *ioch; -#define MAX_CHILDREN 16 - struct ftl_io *parent, *child[MAX_CHILDREN]; - int status[MAX_CHILDREN + 1], i; - size_t pool_size; - - dev = setup_device(1, 16); - ioch = ftl_io_channel_get_ctx(dev->ioch); - pool_size = spdk_mempool_count(ioch->io_pool); - - /* Verify correct behaviour when children finish first */ - parent = alloc_io(dev, io_complete_cb, &status[0]); - parent->status = 0; - - ftl_io_inc_req(parent); - status[0] = -1; - - for (i = 0; i < MAX_CHILDREN; ++i) { - status[i + 1] = -1; - - child[i] = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child[i] != NULL); - setup_io(child[i], dev, io_complete_cb, &status[i + 1]); - child[i]->status = 0; - - ftl_io_inc_req(child[i]); - } - - CU_ASSERT_FALSE(ftl_io_done(parent)); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size - MAX_CHILDREN - 1); - - for (i = 0; i < MAX_CHILDREN; ++i) { - CU_ASSERT_FALSE(ftl_io_done(child[i])); - ftl_io_dec_req(child[i]); - CU_ASSERT_TRUE(ftl_io_done(child[i])); - CU_ASSERT_FALSE(ftl_io_done(parent)); - - ftl_io_complete(child[i]); - CU_ASSERT_FALSE(ftl_io_done(parent)); - CU_ASSERT_EQUAL(status[i + 1], 0); - } - - CU_ASSERT_EQUAL(status[0], -1); - - ftl_io_dec_req(parent); - CU_ASSERT_EQUAL(parent->req_cnt, 0); - CU_ASSERT_TRUE(ftl_io_done(parent)); - - ftl_io_complete(parent); - CU_ASSERT_EQUAL(status[0], 0); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - - /* Verify correct behaviour when parent finishes first */ - parent = alloc_io(dev, io_complete_cb, &status[0]); - parent->status = 0; - - ftl_io_inc_req(parent); - status[0] = -1; - - for (i = 0; i < MAX_CHILDREN; ++i) { - status[i + 1] = -1; - - child[i] = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child[i] != NULL); - setup_io(child[i], dev, io_complete_cb, &status[i + 1]); - child[i]->status = 0; - - ftl_io_inc_req(child[i]); - } - - CU_ASSERT_FALSE(ftl_io_done(parent)); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size - MAX_CHILDREN - 1); - - ftl_io_dec_req(parent); - CU_ASSERT_TRUE(ftl_io_done(parent)); - CU_ASSERT_EQUAL(parent->req_cnt, 0); - - ftl_io_complete(parent); - CU_ASSERT_EQUAL(status[0], -1); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size - MAX_CHILDREN - 1); - - for (i = 0; i < MAX_CHILDREN; ++i) { - CU_ASSERT_FALSE(ftl_io_done(child[i])); - ftl_io_dec_req(child[i]); - CU_ASSERT_TRUE(ftl_io_done(child[i])); - - ftl_io_complete(child[i]); - CU_ASSERT_EQUAL(status[i + 1], 0); - } - - CU_ASSERT_EQUAL(status[0], 0); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - free_device(dev); -} - -static void -test_child_status(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_io_channel *ioch; - struct ftl_io *parent, *child[2]; - int parent_status, child_status[2]; - size_t pool_size, i; - - dev = setup_device(1, 16); - ioch = ftl_io_channel_get_ctx(dev->ioch); - pool_size = spdk_mempool_count(ioch->io_pool); - - /* Verify the first error is returned by the parent */ - parent = alloc_io(dev, io_complete_cb, &parent_status); - parent->status = 0; - - for (i = 0; i < 2; ++i) { - child[i] = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child[i] != NULL); - setup_io(child[i], dev, io_complete_cb, &child_status[i]); - } - - child[0]->status = -3; - child[1]->status = -4; - - ftl_io_complete(child[1]); - ftl_io_complete(child[0]); - ftl_io_complete(parent); - - CU_ASSERT_EQUAL(child_status[0], -3); - CU_ASSERT_EQUAL(child_status[1], -4); - CU_ASSERT_EQUAL(parent_status, -4); - - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - /* Verify parent's status is kept if children finish successfully */ - parent = alloc_io(dev, io_complete_cb, &parent_status); - parent->status = -1; - - for (i = 0; i < 2; ++i) { - child[i] = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child[i] != NULL); - setup_io(child[i], dev, io_complete_cb, &child_status[i]); - } - - child[0]->status = 0; - child[1]->status = 0; - - ftl_io_complete(parent); - ftl_io_complete(child[1]); - ftl_io_complete(child[0]); - - CU_ASSERT_EQUAL(child_status[0], 0); - CU_ASSERT_EQUAL(child_status[1], 0); - CU_ASSERT_EQUAL(parent_status, -1); - - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - /* Verify parent's status is kept if children fail too */ - parent = alloc_io(dev, io_complete_cb, &parent_status); - parent->status = -1; - - for (i = 0; i < 2; ++i) { - child[i] = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child[i] != NULL); - setup_io(child[i], dev, io_complete_cb, &child_status[i]); - } - - child[0]->status = -3; - child[1]->status = -4; - - ftl_io_complete(parent); - ftl_io_complete(child[1]); - ftl_io_complete(child[0]); - - CU_ASSERT_EQUAL(child_status[0], -3); - CU_ASSERT_EQUAL(child_status[1], -4); - CU_ASSERT_EQUAL(parent_status, -1); - - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - free_device(dev); -} - -static void -test_multi_generation(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_io_channel *ioch; -#define MAX_GRAND_CHILDREN 32 - struct ftl_io *parent, *child[MAX_CHILDREN], *gchild[MAX_CHILDREN * MAX_GRAND_CHILDREN]; - int parent_status, child_status[MAX_CHILDREN], gchild_status[MAX_CHILDREN * MAX_GRAND_CHILDREN]; - size_t pool_size; - int i, j; - - dev = setup_device(1, 16); - ioch = ftl_io_channel_get_ctx(dev->ioch); - pool_size = spdk_mempool_count(ioch->io_pool); - - /* Verify correct behaviour when children finish first */ - parent = alloc_io(dev, io_complete_cb, &parent_status); - parent->status = 0; - - ftl_io_inc_req(parent); - parent_status = -1; - - for (i = 0; i < MAX_CHILDREN; ++i) { - child_status[i] = -1; - - child[i] = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child[i] != NULL); - setup_io(child[i], dev, io_complete_cb, &child_status[i]); - child[i]->status = 0; - - - for (j = 0; j < MAX_GRAND_CHILDREN; ++j) { - struct ftl_io *io = ftl_io_alloc_child(child[i]); - SPDK_CU_ASSERT_FATAL(io != NULL); - - gchild[i * MAX_GRAND_CHILDREN + j] = io; - gchild_status[i * MAX_GRAND_CHILDREN + j] = -1; - setup_io(io, dev, io_complete_cb, &gchild_status[i * MAX_GRAND_CHILDREN + j]); - io->status = 0; - - ftl_io_inc_req(io); - } - - ftl_io_inc_req(child[i]); - } - - for (i = 0; i < MAX_CHILDREN; ++i) { - CU_ASSERT_FALSE(ftl_io_done(child[i])); - ftl_io_dec_req(child[i]); - CU_ASSERT_TRUE(ftl_io_done(child[i])); - - ftl_io_complete(child[i]); - CU_ASSERT_FALSE(ftl_io_done(parent)); - CU_ASSERT_EQUAL(child_status[i], -1); - - for (j = 0; j < MAX_GRAND_CHILDREN; ++j) { - struct ftl_io *io = gchild[i * MAX_GRAND_CHILDREN + j]; - - CU_ASSERT_FALSE(ftl_io_done(io)); - ftl_io_dec_req(io); - CU_ASSERT_TRUE(ftl_io_done(io)); - ftl_io_complete(io); - CU_ASSERT_EQUAL(gchild_status[i * MAX_GRAND_CHILDREN + j], 0); - } - - CU_ASSERT_EQUAL(child_status[i], 0); - } - - ftl_io_dec_req(parent); - CU_ASSERT_TRUE(ftl_io_done(parent)); - ftl_io_complete(parent); - CU_ASSERT_EQUAL(parent_status, 0); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - /* Verify correct behaviour when parents finish first */ - parent = alloc_io(dev, io_complete_cb, &parent_status); - parent->status = 0; - parent_status = -1; - - for (i = 0; i < MAX_CHILDREN; ++i) { - child_status[i] = -1; - - child[i] = ftl_io_alloc_child(parent); - SPDK_CU_ASSERT_FATAL(child[i] != NULL); - setup_io(child[i], dev, io_complete_cb, &child_status[i]); - child[i]->status = 0; - - for (j = 0; j < MAX_GRAND_CHILDREN; ++j) { - struct ftl_io *io = ftl_io_alloc_child(child[i]); - SPDK_CU_ASSERT_FATAL(io != NULL); - - gchild[i * MAX_GRAND_CHILDREN + j] = io; - gchild_status[i * MAX_GRAND_CHILDREN + j] = -1; - setup_io(io, dev, io_complete_cb, &gchild_status[i * MAX_GRAND_CHILDREN + j]); - io->status = 0; - - ftl_io_inc_req(io); - } - - CU_ASSERT_TRUE(ftl_io_done(child[i])); - ftl_io_complete(child[i]); - CU_ASSERT_EQUAL(child_status[i], -1); - } - - CU_ASSERT_TRUE(ftl_io_done(parent)); - ftl_io_complete(parent); - CU_ASSERT_EQUAL(parent_status, -1); - - for (i = 0; i < MAX_CHILDREN; ++i) { - for (j = 0; j < MAX_GRAND_CHILDREN; ++j) { - struct ftl_io *io = gchild[i * MAX_GRAND_CHILDREN + j]; - - CU_ASSERT_FALSE(ftl_io_done(io)); - ftl_io_dec_req(io); - CU_ASSERT_TRUE(ftl_io_done(io)); - ftl_io_complete(io); - CU_ASSERT_EQUAL(gchild_status[i * MAX_GRAND_CHILDREN + j], 0); - } - - CU_ASSERT_EQUAL(child_status[i], 0); - } - - CU_ASSERT_EQUAL(parent_status, 0); - CU_ASSERT_EQUAL(spdk_mempool_count(ioch->io_pool), pool_size); - - free_device(dev); -} - -static void -test_io_channel_create(void) -{ - struct spdk_ftl_dev *dev; - struct spdk_io_channel *ioch, **ioch_array; - struct ftl_io_channel *ftl_ioch; - uint32_t ioch_idx; - - dev = setup_device(g_default_conf.max_io_channels + 1, 16); - - ioch = spdk_get_io_channel(dev); - CU_ASSERT(ioch != NULL); - CU_ASSERT_EQUAL(dev->num_io_channels, 1); - spdk_put_io_channel(ioch); - poll_threads(); - CU_ASSERT_EQUAL(dev->num_io_channels, 0); - - ioch_array = calloc(dev->conf.max_io_channels, sizeof(*ioch_array)); - SPDK_CU_ASSERT_FATAL(ioch != NULL); - - for (ioch_idx = 0; ioch_idx < dev->conf.max_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - ioch = ioch_array[ioch_idx] = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(ioch != NULL); - poll_threads(); - - ftl_ioch = ftl_io_channel_get_ctx(ioch); - CU_ASSERT_EQUAL(ftl_ioch->index, ioch_idx); - } - - CU_ASSERT_EQUAL(dev->num_io_channels, dev->conf.max_io_channels); - set_thread(dev->conf.max_io_channels); - ioch = spdk_get_io_channel(dev); - CU_ASSERT_EQUAL(dev->num_io_channels, dev->conf.max_io_channels); - CU_ASSERT_EQUAL(ioch, NULL); - - for (ioch_idx = 0; ioch_idx < dev->conf.max_io_channels; ioch_idx += 2) { - set_thread(ioch_idx); - spdk_put_io_channel(ioch_array[ioch_idx]); - ioch_array[ioch_idx] = NULL; - poll_threads(); - } - - poll_threads(); - CU_ASSERT_EQUAL(dev->num_io_channels, dev->conf.max_io_channels / 2); - - for (ioch_idx = 0; ioch_idx < dev->conf.max_io_channels; ioch_idx++) { - set_thread(ioch_idx); - - if (ioch_array[ioch_idx] == NULL) { - ioch = ioch_array[ioch_idx] = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(ioch != NULL); - poll_threads(); - - ftl_ioch = ftl_io_channel_get_ctx(ioch); - CU_ASSERT_EQUAL(ftl_ioch->index, ioch_idx); - } - } - - for (ioch_idx = 0; ioch_idx < dev->conf.max_io_channels; ioch_idx++) { - set_thread(ioch_idx); - spdk_put_io_channel(ioch_array[ioch_idx]); - } - - poll_threads(); - CU_ASSERT_EQUAL(dev->num_io_channels, 0); - - free(ioch_array); - free_device(dev); -} - -static void -test_acquire_entry(void) -{ - struct spdk_ftl_dev *dev; - struct spdk_io_channel *ioch, **ioch_array; - struct ftl_io_channel *ftl_ioch; - struct ftl_wbuf_entry *entry, **entries; - uint32_t num_entries, num_io_channels = 2; - uint32_t ioch_idx, entry_idx, tmp_idx; - - dev = setup_device(num_io_channels, 16); - - num_entries = dev->conf.write_buffer_size / FTL_BLOCK_SIZE; - entries = calloc(num_entries * num_io_channels, sizeof(*entries)); - SPDK_CU_ASSERT_FATAL(entries != NULL); - ioch_array = calloc(num_io_channels, sizeof(*ioch_array)); - SPDK_CU_ASSERT_FATAL(ioch_array != NULL); - - /* Acquire whole buffer of internal entries */ - entry_idx = 0; - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - ioch_array[ioch_idx] = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(ioch_array[ioch_idx] != NULL); - ftl_ioch = ftl_io_channel_get_ctx(ioch_array[ioch_idx]); - poll_threads(); - - for (tmp_idx = 0; tmp_idx < num_entries; ++tmp_idx) { - entries[entry_idx++] = ftl_acquire_wbuf_entry(ftl_ioch, FTL_IO_INTERNAL); - CU_ASSERT(entries[entry_idx - 1] != NULL); - } - - entry = ftl_acquire_wbuf_entry(ftl_ioch, FTL_IO_INTERNAL); - CU_ASSERT(entry == NULL); - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - - for (tmp_idx = 0; tmp_idx < num_entries; ++tmp_idx) { - ftl_release_wbuf_entry(entries[ioch_idx * num_entries + tmp_idx]); - entries[ioch_idx * num_entries + tmp_idx] = NULL; - } - - spdk_put_io_channel(ioch_array[ioch_idx]); - } - poll_threads(); - - /* Do the same for user entries */ - entry_idx = 0; - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - ioch_array[ioch_idx] = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(ioch_array[ioch_idx] != NULL); - ftl_ioch = ftl_io_channel_get_ctx(ioch_array[ioch_idx]); - poll_threads(); - - for (tmp_idx = 0; tmp_idx < num_entries; ++tmp_idx) { - entries[entry_idx++] = ftl_acquire_wbuf_entry(ftl_ioch, 0); - CU_ASSERT(entries[entry_idx - 1] != NULL); - } - - entry = ftl_acquire_wbuf_entry(ftl_ioch, 0); - CU_ASSERT(entry == NULL); - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - - for (tmp_idx = 0; tmp_idx < num_entries; ++tmp_idx) { - ftl_release_wbuf_entry(entries[ioch_idx * num_entries + tmp_idx]); - entries[ioch_idx * num_entries + tmp_idx] = NULL; - } - - spdk_put_io_channel(ioch_array[ioch_idx]); - } - poll_threads(); - - /* Verify limits */ - entry_idx = 0; - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - ioch_array[ioch_idx] = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(ioch_array[ioch_idx] != NULL); - ftl_ioch = ftl_io_channel_get_ctx(ioch_array[ioch_idx]); - poll_threads(); - - ftl_ioch->qdepth_limit = num_entries / 2; - for (tmp_idx = 0; tmp_idx < num_entries / 2; ++tmp_idx) { - entries[entry_idx++] = ftl_acquire_wbuf_entry(ftl_ioch, 0); - CU_ASSERT(entries[entry_idx - 1] != NULL); - } - - entry = ftl_acquire_wbuf_entry(ftl_ioch, 0); - CU_ASSERT(entry == NULL); - - for (; tmp_idx < num_entries; ++tmp_idx) { - entries[entry_idx++] = ftl_acquire_wbuf_entry(ftl_ioch, FTL_IO_INTERNAL); - CU_ASSERT(entries[entry_idx - 1] != NULL); - } - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - - for (tmp_idx = 0; tmp_idx < num_entries; ++tmp_idx) { - ftl_release_wbuf_entry(entries[ioch_idx * num_entries + tmp_idx]); - entries[ioch_idx * num_entries + tmp_idx] = NULL; - } - - spdk_put_io_channel(ioch_array[ioch_idx]); - } - poll_threads(); - - /* Verify acquire/release */ - set_thread(0); - ioch = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(ioch != NULL); - ftl_ioch = ftl_io_channel_get_ctx(ioch); - poll_threads(); - - for (entry_idx = 0; entry_idx < num_entries; ++entry_idx) { - entries[entry_idx] = ftl_acquire_wbuf_entry(ftl_ioch, 0); - CU_ASSERT(entries[entry_idx] != NULL); - } - - entry = ftl_acquire_wbuf_entry(ftl_ioch, 0); - CU_ASSERT(entry == NULL); - - for (entry_idx = 0; entry_idx < num_entries / 2; ++entry_idx) { - ftl_release_wbuf_entry(entries[entry_idx]); - entries[entry_idx] = NULL; - } - - for (; entry_idx < num_entries; ++entry_idx) { - entries[entry_idx - num_entries / 2] = ftl_acquire_wbuf_entry(ftl_ioch, 0); - CU_ASSERT(entries[entry_idx - num_entries / 2] != NULL); - } - - for (entry_idx = 0; entry_idx < num_entries; ++entry_idx) { - ftl_release_wbuf_entry(entries[entry_idx]); - entries[entry_idx] = NULL; - } - - spdk_put_io_channel(ioch); - poll_threads(); - - free(ioch_array); - free(entries); - free_device(dev); -} - -static void -test_submit_batch(void) -{ - struct spdk_ftl_dev *dev; - struct spdk_io_channel **_ioch_array; - struct ftl_io_channel **ioch_array; - struct ftl_wbuf_entry *entry; - struct ftl_batch *batch, *batch2; - uint32_t num_io_channels = 16; - uint32_t ioch_idx, tmp_idx, entry_idx; - uint64_t ioch_bitmap; - size_t num_entries; - - dev = setup_device(num_io_channels, num_io_channels); - - _ioch_array = calloc(num_io_channels, sizeof(*_ioch_array)); - SPDK_CU_ASSERT_FATAL(_ioch_array != NULL); - ioch_array = calloc(num_io_channels, sizeof(*ioch_array)); - SPDK_CU_ASSERT_FATAL(ioch_array != NULL); - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - _ioch_array[ioch_idx] = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(_ioch_array[ioch_idx] != NULL); - ioch_array[ioch_idx] = ftl_io_channel_get_ctx(_ioch_array[ioch_idx]); - poll_threads(); - } - - /* Make sure the IO channels are not starved and entries are popped in RR fashion */ - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - - for (entry_idx = 0; entry_idx < dev->xfer_size; ++entry_idx) { - entry = ftl_acquire_wbuf_entry(ioch_array[ioch_idx], 0); - SPDK_CU_ASSERT_FATAL(entry != NULL); - - num_entries = spdk_ring_enqueue(ioch_array[ioch_idx]->submit_queue, - (void **)&entry, 1, NULL); - CU_ASSERT(num_entries == 1); - } - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - for (tmp_idx = 0; tmp_idx < ioch_idx; ++tmp_idx) { - set_thread(tmp_idx); - - while (spdk_ring_count(ioch_array[tmp_idx]->submit_queue) < dev->xfer_size) { - entry = ftl_acquire_wbuf_entry(ioch_array[tmp_idx], 0); - SPDK_CU_ASSERT_FATAL(entry != NULL); - - num_entries = spdk_ring_enqueue(ioch_array[tmp_idx]->submit_queue, - (void **)&entry, 1, NULL); - CU_ASSERT(num_entries == 1); - } - } - - set_thread(ioch_idx); - - batch = ftl_get_next_batch(dev); - SPDK_CU_ASSERT_FATAL(batch != NULL); - - TAILQ_FOREACH(entry, &batch->entries, tailq) { - CU_ASSERT(entry->ioch == ioch_array[ioch_idx]); - } - - ftl_release_batch(dev, batch); - - CU_ASSERT(spdk_ring_count(ioch_array[ioch_idx]->free_queue) == - ioch_array[ioch_idx]->num_entries); - } - - for (ioch_idx = 0; ioch_idx < num_io_channels - 1; ++ioch_idx) { - batch = ftl_get_next_batch(dev); - SPDK_CU_ASSERT_FATAL(batch != NULL); - ftl_release_batch(dev, batch); - } - - /* Make sure the batch can be built from entries from any IO channel */ - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - entry = ftl_acquire_wbuf_entry(ioch_array[ioch_idx], 0); - SPDK_CU_ASSERT_FATAL(entry != NULL); - - num_entries = spdk_ring_enqueue(ioch_array[ioch_idx]->submit_queue, - (void **)&entry, 1, NULL); - CU_ASSERT(num_entries == 1); - } - - batch = ftl_get_next_batch(dev); - SPDK_CU_ASSERT_FATAL(batch != NULL); - - ioch_bitmap = 0; - TAILQ_FOREACH(entry, &batch->entries, tailq) { - ioch_bitmap |= 1 << entry->ioch->index; - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - CU_ASSERT((ioch_bitmap & (1 << ioch_array[ioch_idx]->index)) != 0); - } - ftl_release_batch(dev, batch); - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - CU_ASSERT(spdk_ring_count(ioch_array[ioch_idx]->free_queue) == - ioch_array[ioch_idx]->num_entries); - } - - /* Make sure pending batches are prioritized */ - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - - while (spdk_ring_count(ioch_array[ioch_idx]->submit_queue) < dev->xfer_size) { - entry = ftl_acquire_wbuf_entry(ioch_array[ioch_idx], 0); - SPDK_CU_ASSERT_FATAL(entry != NULL); - num_entries = spdk_ring_enqueue(ioch_array[ioch_idx]->submit_queue, - (void **)&entry, 1, NULL); - CU_ASSERT(num_entries == 1); - } - } - - batch = ftl_get_next_batch(dev); - SPDK_CU_ASSERT_FATAL(batch != NULL); - - TAILQ_INSERT_TAIL(&dev->pending_batches, batch, tailq); - batch2 = ftl_get_next_batch(dev); - SPDK_CU_ASSERT_FATAL(batch2 != NULL); - - CU_ASSERT(TAILQ_EMPTY(&dev->pending_batches)); - CU_ASSERT(batch == batch2); - - batch = ftl_get_next_batch(dev); - SPDK_CU_ASSERT_FATAL(batch != NULL); - - ftl_release_batch(dev, batch); - ftl_release_batch(dev, batch2); - - for (ioch_idx = 2; ioch_idx < num_io_channels; ++ioch_idx) { - batch = ftl_get_next_batch(dev); - SPDK_CU_ASSERT_FATAL(batch != NULL); - ftl_release_batch(dev, batch); - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - spdk_put_io_channel(_ioch_array[ioch_idx]); - } - poll_threads(); - - free(_ioch_array); - free(ioch_array); - free_device(dev); -} - -static void -test_entry_address(void) -{ - struct spdk_ftl_dev *dev; - struct spdk_io_channel **ioch_array; - struct ftl_io_channel *ftl_ioch; - struct ftl_wbuf_entry **entry_array; - struct ftl_addr addr; - uint32_t num_entries, num_io_channels = 7; - uint32_t ioch_idx, entry_idx; - - dev = setup_device(num_io_channels, num_io_channels); - ioch_array = calloc(num_io_channels, sizeof(*ioch_array)); - SPDK_CU_ASSERT_FATAL(ioch_array != NULL); - - num_entries = dev->conf.write_buffer_size / FTL_BLOCK_SIZE; - entry_array = calloc(num_entries, sizeof(*entry_array)); - SPDK_CU_ASSERT_FATAL(entry_array != NULL); - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - ioch_array[ioch_idx] = spdk_get_io_channel(dev); - SPDK_CU_ASSERT_FATAL(ioch_array[ioch_idx] != NULL); - poll_threads(); - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ++ioch_idx) { - set_thread(ioch_idx); - ftl_ioch = ftl_io_channel_get_ctx(ioch_array[ioch_idx]); - - for (entry_idx = 0; entry_idx < num_entries; ++entry_idx) { - entry_array[entry_idx] = ftl_acquire_wbuf_entry(ftl_ioch, 0); - SPDK_CU_ASSERT_FATAL(entry_array[entry_idx] != NULL); - - addr = ftl_get_addr_from_entry(entry_array[entry_idx]); - CU_ASSERT(addr.cached == 1); - CU_ASSERT((addr.cache_offset >> dev->ioch_shift) == entry_idx); - CU_ASSERT((addr.cache_offset & ((1 << dev->ioch_shift) - 1)) == ioch_idx); - CU_ASSERT(entry_array[entry_idx] == ftl_get_entry_from_addr(dev, addr)); - } - - for (entry_idx = 0; entry_idx < num_entries; ++entry_idx) { - ftl_release_wbuf_entry(entry_array[entry_idx]); - } - } - - for (ioch_idx = 0; ioch_idx < num_io_channels; ioch_idx += 2) { - set_thread(ioch_idx); - spdk_put_io_channel(ioch_array[ioch_idx]); - ioch_array[ioch_idx] = NULL; - } - poll_threads(); - - for (ioch_idx = 1; ioch_idx < num_io_channels; ioch_idx += 2) { - set_thread(ioch_idx); - ftl_ioch = ftl_io_channel_get_ctx(ioch_array[ioch_idx]); - - for (entry_idx = 0; entry_idx < num_entries; ++entry_idx) { - entry_array[entry_idx] = ftl_acquire_wbuf_entry(ftl_ioch, 0); - SPDK_CU_ASSERT_FATAL(entry_array[entry_idx] != NULL); - - addr = ftl_get_addr_from_entry(entry_array[entry_idx]); - CU_ASSERT(addr.cached == 1); - CU_ASSERT(entry_array[entry_idx] == ftl_get_entry_from_addr(dev, addr)); - } - - for (entry_idx = 0; entry_idx < num_entries; ++entry_idx) { - ftl_release_wbuf_entry(entry_array[entry_idx]); - } - } - - for (ioch_idx = 1; ioch_idx < num_io_channels; ioch_idx += 2) { - set_thread(ioch_idx); - spdk_put_io_channel(ioch_array[ioch_idx]); - } - poll_threads(); - - free(entry_array); - free(ioch_array); - free_device(dev); -} - -int -main(int argc, char **argv) -{ - CU_pSuite suite; - unsigned int num_failures; - - CU_set_error_action(CUEA_ABORT); - CU_initialize_registry(); - - suite = CU_add_suite("ftl_io_suite", NULL, NULL); - - - CU_ADD_TEST(suite, test_completion); - CU_ADD_TEST(suite, test_alloc_free); - CU_ADD_TEST(suite, test_child_requests); - CU_ADD_TEST(suite, test_child_status); - CU_ADD_TEST(suite, test_multi_generation); - CU_ADD_TEST(suite, test_io_channel_create); - CU_ADD_TEST(suite, test_acquire_entry); - CU_ADD_TEST(suite, test_submit_batch); - CU_ADD_TEST(suite, test_entry_address); - - CU_basic_set_mode(CU_BRM_VERBOSE); - CU_basic_run_tests(); - num_failures = CU_get_number_of_failures(); - CU_cleanup_registry(); - - return num_failures; -} diff --git a/test/unit/lib/ftl/ftl_md/Makefile b/test/unit/lib/ftl/ftl_md/Makefile deleted file mode 100644 index 91513bd6a..000000000 --- a/test/unit/lib/ftl/ftl_md/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) - -TEST_FILE = ftl_md_ut.c - -include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/ftl/ftl_md/ftl_md_ut.c b/test/unit/lib/ftl/ftl_md/ftl_md_ut.c deleted file mode 100644 index 36f742ee0..000000000 --- a/test/unit/lib/ftl/ftl_md/ftl_md_ut.c +++ /dev/null @@ -1,133 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" - -#include "spdk_cunit.h" -#include "common/lib/test_env.c" - -#include "ftl/ftl_band.c" -#include "../common/utils.c" - -#if defined(DEBUG) -DEFINE_STUB(ftl_band_validate_md, bool, (struct ftl_band *band), true); -#endif -DEFINE_STUB_V(ftl_apply_limits, (struct spdk_ftl_dev *dev)); -DEFINE_STUB(ftl_io_init_internal, struct ftl_io *, - (const struct ftl_io_init_opts *opts), NULL); -DEFINE_STUB_V(ftl_io_read, (struct ftl_io *io)); -DEFINE_STUB_V(ftl_io_write, (struct ftl_io *io)); -DEFINE_STUB_V(ftl_reloc_add, (struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, - size_t num_blocks, int prio, bool defrag)); - -struct base_bdev_geometry g_geo = { - .write_unit_size = 16, - .optimal_open_zones = 12, - .zone_size = 100, - .blockcnt = 1500 * 100 * 12, -}; - -static void -setup_band(struct ftl_band **band, const struct base_bdev_geometry *geo) -{ - int rc; - struct spdk_ftl_dev *dev; - - dev = test_init_ftl_dev(&g_geo); - *band = test_init_ftl_band(dev, 0, geo->zone_size); - rc = ftl_band_alloc_lba_map(*band); - SPDK_CU_ASSERT_FATAL(rc == 0); - (*band)->state = FTL_BAND_STATE_PREP; - ftl_band_clear_lba_map(*band); -} - -static void -cleanup_band(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - test_free_ftl_band(band); - test_free_ftl_dev(dev); -} - -static void -test_md_unpack(void) -{ - struct ftl_band *band; - struct ftl_lba_map *lba_map; - - setup_band(&band, &g_geo); - - lba_map = &band->lba_map; - SPDK_CU_ASSERT_FATAL(lba_map->dma_buf); - - ftl_pack_head_md(band); - CU_ASSERT_EQUAL(ftl_unpack_head_md(band), FTL_MD_SUCCESS); - - ftl_pack_tail_md(band); - CU_ASSERT_EQUAL(ftl_unpack_tail_md(band), FTL_MD_SUCCESS); - - cleanup_band(band); -} - -static void -test_md_unpack_fail(void) -{ - struct ftl_band *band; - struct ftl_lba_map *lba_map; - struct ftl_md_hdr *hdr; - - setup_band(&band, &g_geo); - - lba_map = &band->lba_map; - SPDK_CU_ASSERT_FATAL(lba_map->dma_buf); - - /* check crc */ - ftl_pack_tail_md(band); - /* flip last bit of lba_map */ - *((char *)lba_map->dma_buf + ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE - 1) ^= 0x1; - CU_ASSERT_EQUAL(ftl_unpack_tail_md(band), FTL_MD_INVALID_CRC); - - /* check invalid version */ - hdr = lba_map->dma_buf; - ftl_pack_tail_md(band); - hdr->ver++; - CU_ASSERT_EQUAL(ftl_unpack_tail_md(band), FTL_MD_INVALID_VER); - - /* check wrong UUID */ - ftl_pack_head_md(band); - hdr->uuid.u.raw[0] ^= 0x1; - CU_ASSERT_EQUAL(ftl_unpack_head_md(band), FTL_MD_NO_MD); - - /* check invalid size */ - ftl_pack_tail_md(band); - g_geo.zone_size--; - CU_ASSERT_EQUAL(ftl_unpack_tail_md(band), FTL_MD_INVALID_SIZE); - - cleanup_band(band); -} - -int -main(int argc, char **argv) -{ - CU_pSuite suite = NULL; - unsigned int num_failures; - - CU_set_error_action(CUEA_ABORT); - CU_initialize_registry(); - - suite = CU_add_suite("ftl_meta_suite", NULL, NULL); - - - CU_ADD_TEST(suite, test_md_unpack); - CU_ADD_TEST(suite, test_md_unpack_fail); - - CU_basic_set_mode(CU_BRM_VERBOSE); - CU_basic_run_tests(); - num_failures = CU_get_number_of_failures(); - CU_cleanup_registry(); - - return num_failures; -} diff --git a/test/unit/lib/ftl/ftl_ppa/Makefile b/test/unit/lib/ftl/ftl_ppa/Makefile deleted file mode 100644 index db5db76db..000000000 --- a/test/unit/lib/ftl/ftl_ppa/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) - -TEST_FILE = ftl_ppa_ut.c - -include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/ftl/ftl_ppa/ftl_ppa_ut.c b/test/unit/lib/ftl/ftl_ppa/ftl_ppa_ut.c deleted file mode 100644 index d7bdd6c9f..000000000 --- a/test/unit/lib/ftl/ftl_ppa/ftl_ppa_ut.c +++ /dev/null @@ -1,198 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" - -#include "spdk_cunit.h" -#include "common/lib/test_env.c" - -#include "ftl/ftl_core.h" - -#define L2P_TABLE_SIZE 1024 - -static struct spdk_ftl_dev *g_dev; - -DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *desc), NULL); - -uint64_t -spdk_bdev_get_zone_size(const struct spdk_bdev *bdev) -{ - if (g_dev->addr_len > 32) { - return 1ULL << 32; - } - - return 1024; -} - -uint32_t -spdk_bdev_get_optimal_open_zones(const struct spdk_bdev *bdev) -{ - return 100; -} - -static struct spdk_ftl_dev * -test_alloc_dev(size_t size) -{ - struct spdk_ftl_dev *dev; - - dev = calloc(1, sizeof(*dev)); - - dev->num_lbas = L2P_TABLE_SIZE; - dev->l2p = calloc(L2P_TABLE_SIZE, size); - - return dev; -} - -static int -setup_l2p_32bit(void) -{ - g_dev = test_alloc_dev(sizeof(uint32_t)); - g_dev->addr_len = 24; - return 0; -} - -static int -setup_l2p_64bit(void) -{ - g_dev = test_alloc_dev(sizeof(uint64_t)); - g_dev->addr_len = 63; - return 0; -} - -static void -clean_l2p(void) -{ - size_t l2p_elem_size; - - if (ftl_addr_packed(g_dev)) { - l2p_elem_size = sizeof(uint32_t); - } else { - l2p_elem_size = sizeof(uint64_t); - } - memset(g_dev->l2p, 0, g_dev->num_lbas * l2p_elem_size); -} - -static int -cleanup(void) -{ - free(g_dev->l2p); - free(g_dev); - g_dev = NULL; - return 0; -} - -static void -test_addr_pack32(void) -{ - struct ftl_addr orig = {}, addr; - - /* Check valid address transformation */ - orig.offset = 4; - addr = ftl_addr_to_packed(g_dev, orig); - CU_ASSERT_TRUE(addr.offset <= UINT32_MAX); - CU_ASSERT_FALSE(addr.pack.cached); - addr = ftl_addr_from_packed(g_dev, addr); - CU_ASSERT_FALSE(ftl_addr_invalid(addr)); - CU_ASSERT_EQUAL(addr.offset, orig.offset); - - /* Check invalid address transformation */ - orig = ftl_to_addr(FTL_ADDR_INVALID); - addr = ftl_addr_to_packed(g_dev, orig); - CU_ASSERT_TRUE(addr.offset <= UINT32_MAX); - addr = ftl_addr_from_packed(g_dev, addr); - CU_ASSERT_TRUE(ftl_addr_invalid(addr)); - - /* Check cached entry offset transformation */ - orig.cached = 1; - orig.cache_offset = 1024; - addr = ftl_addr_to_packed(g_dev, orig); - CU_ASSERT_TRUE(addr.offset <= UINT32_MAX); - CU_ASSERT_TRUE(addr.pack.cached); - addr = ftl_addr_from_packed(g_dev, addr); - CU_ASSERT_FALSE(ftl_addr_invalid(addr)); - CU_ASSERT_TRUE(ftl_addr_cached(addr)); - CU_ASSERT_EQUAL(addr.offset, orig.offset); - clean_l2p(); -} - -static void -test_addr_invalid(void) -{ - struct ftl_addr addr; - size_t i; - - /* Set every other LBA as invalid */ - for (i = 0; i < L2P_TABLE_SIZE; i += 2) { - ftl_l2p_set(g_dev, i, ftl_to_addr(FTL_ADDR_INVALID)); - } - - /* Check every even LBA is invalid while others are fine */ - for (i = 0; i < L2P_TABLE_SIZE; ++i) { - addr = ftl_l2p_get(g_dev, i); - - if (i % 2 == 0) { - CU_ASSERT_TRUE(ftl_addr_invalid(addr)); - } else { - CU_ASSERT_FALSE(ftl_addr_invalid(addr)); - } - } - clean_l2p(); -} - -static void -test_addr_cached(void) -{ - struct ftl_addr addr; - size_t i; - - /* Set every other LBA is cached */ - for (i = 0; i < L2P_TABLE_SIZE; i += 2) { - addr.cached = 1; - addr.cache_offset = i; - ftl_l2p_set(g_dev, i, addr); - } - - /* Check every even LBA is cached while others are not */ - for (i = 0; i < L2P_TABLE_SIZE; ++i) { - addr = ftl_l2p_get(g_dev, i); - - if (i % 2 == 0) { - CU_ASSERT_TRUE(ftl_addr_cached(addr)); - CU_ASSERT_EQUAL(addr.cache_offset, i); - } else { - CU_ASSERT_FALSE(ftl_addr_cached(addr)); - } - } - clean_l2p(); -} - -int -main(int argc, char **argv) -{ - CU_pSuite suite32 = NULL, suite64 = NULL; - unsigned int num_failures; - - CU_set_error_action(CUEA_ABORT); - CU_initialize_registry(); - - suite32 = CU_add_suite("ftl_addr32_suite", setup_l2p_32bit, cleanup); - - - suite64 = CU_add_suite("ftl_addr64_suite", setup_l2p_64bit, cleanup); - - - CU_ADD_TEST(suite32, test_addr_pack32); - CU_ADD_TEST(suite32, test_addr_invalid); - CU_ADD_TEST(suite32, test_addr_cached); - CU_ADD_TEST(suite64, test_addr_invalid); - CU_ADD_TEST(suite64, test_addr_cached); - - CU_basic_set_mode(CU_BRM_VERBOSE); - CU_basic_run_tests(); - num_failures = CU_get_number_of_failures(); - CU_cleanup_registry(); - - return num_failures; -} diff --git a/test/unit/lib/ftl/ftl_reloc.c/Makefile b/test/unit/lib/ftl/ftl_reloc.c/Makefile deleted file mode 100644 index e2d3991eb..000000000 --- a/test/unit/lib/ftl/ftl_reloc.c/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) - -TEST_FILE = ftl_reloc_ut.c - -include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c b/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c deleted file mode 100644 index 61eb656d5..000000000 --- a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c +++ /dev/null @@ -1,480 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" - -#include "spdk_cunit.h" -#include "common/lib/test_env.c" - -#include "ftl/ftl_reloc.c" -#include "../common/utils.c" - -#define MAX_ACTIVE_RELOCS 5 -#define MAX_RELOC_QDEPTH 31 - -struct base_bdev_geometry g_geo = { - .write_unit_size = 16, - .optimal_open_zones = 12, - .zone_size = 100, - .blockcnt = 1500 * 100 * 12, -}; - -DEFINE_STUB(ftl_dev_tail_md_disk_size, size_t, (const struct spdk_ftl_dev *dev), 1); -DEFINE_STUB(ftl_addr_is_written, bool, (struct ftl_band *band, struct ftl_addr addr), true); -DEFINE_STUB_V(ftl_band_set_state, (struct ftl_band *band, enum ftl_band_state state)); -DEFINE_STUB_V(ftl_free_io, (struct ftl_io *io)); -#if defined(DEBUG) -DEFINE_STUB_V(ftl_trace_lba_io_init, (struct spdk_ftl_dev *dev, const struct ftl_io *io)); -#endif - -int -ftl_band_alloc_lba_map(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - ftl_band_acquire_lba_map(band); - band->lba_map.map = spdk_mempool_get(dev->lba_pool); - - return 0; -} - -void -ftl_band_release_lba_map(struct ftl_band *band) -{ - struct spdk_ftl_dev *dev = band->dev; - - band->lba_map.ref_cnt--; - spdk_mempool_put(dev->lba_pool, band->lba_map.map); - band->lba_map.map = NULL; -} - -void -ftl_band_acquire_lba_map(struct ftl_band *band) -{ - band->lba_map.ref_cnt++; -} - -size_t -ftl_lba_map_num_blocks(const struct spdk_ftl_dev *dev) -{ - return spdk_divide_round_up(ftl_get_num_blocks_in_band(dev) * sizeof(uint64_t), FTL_BLOCK_SIZE); -} - -int -ftl_band_read_lba_map(struct ftl_band *band, size_t offset, - size_t num_blocks, ftl_io_fn fn, void *ctx) -{ - fn(ctx, ctx, 0); - return 0; -} - -uint64_t -ftl_band_block_offset_from_addr(struct ftl_band *band, struct ftl_addr addr) -{ - return test_offset_from_addr(addr, band); -} - -struct ftl_addr -ftl_band_addr_from_block_offset(struct ftl_band *band, uint64_t block_off) -{ - struct ftl_addr addr = {}; - - addr.offset = block_off + band->id * ftl_get_num_blocks_in_band(band->dev); - return addr; -} - -void -ftl_io_read(struct ftl_io *io) -{ - io->cb_fn(io, io->cb_ctx, 0); - free(io); -} - -void -ftl_io_write(struct ftl_io *io) -{ - io->cb_fn(io, io->cb_ctx, 0); - free(io->lba.vector); - free(io); -} - -struct ftl_io * -ftl_io_init_internal(const struct ftl_io_init_opts *opts) -{ - struct ftl_io *io = opts->io; - - if (!io) { - io = calloc(1, opts->size); - } - - SPDK_CU_ASSERT_FATAL(io != NULL); - - io->dev = opts->dev; - io->band = opts->band; - io->flags = opts->flags; - io->cb_fn = opts->cb_fn; - io->cb_ctx = io; - io->num_blocks = opts->num_blocks; - memcpy(&io->iov, &opts->iovs, sizeof(io->iov)); - io->iov_cnt = opts->iovcnt; - - if (opts->flags & FTL_IO_VECTOR_LBA) { - io->lba.vector = calloc(io->num_blocks, sizeof(uint64_t)); - SPDK_CU_ASSERT_FATAL(io->lba.vector != NULL); - } - - return io; -} - -struct ftl_io * -ftl_io_alloc(struct spdk_io_channel *ch) -{ - size_t io_size = sizeof(struct ftl_md_io); - - return malloc(io_size); -} - -void -ftl_io_reinit(struct ftl_io *io, ftl_io_fn fn, void *ctx, int flags, int type) -{ - io->cb_fn = fn; - io->cb_ctx = ctx; - io->type = type; -} - -static void -single_reloc_move(struct ftl_band_reloc *breloc) -{ - /* Process read */ - ftl_process_reloc(breloc); - /* Process lba map read */ - ftl_process_reloc(breloc); - /* Process write */ - ftl_process_reloc(breloc); -} - -static void -add_to_active_queue(struct ftl_reloc *reloc, struct ftl_band_reloc *breloc) -{ - TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); - breloc->state = FTL_BAND_RELOC_STATE_ACTIVE; - TAILQ_INSERT_HEAD(&reloc->active_queue, breloc, entry); -} - -static void -setup_reloc(struct spdk_ftl_dev **_dev, struct ftl_reloc **_reloc, - const struct base_bdev_geometry *geo) -{ - size_t i; - struct spdk_ftl_dev *dev; - struct ftl_reloc *reloc; - - dev = test_init_ftl_dev(geo); - - dev->conf.max_active_relocs = MAX_ACTIVE_RELOCS; - dev->conf.max_reloc_qdepth = MAX_RELOC_QDEPTH; - - SPDK_CU_ASSERT_FATAL(ftl_get_num_bands(dev) > 0); - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - test_init_ftl_band(dev, i, geo->zone_size); - } - - reloc = ftl_reloc_init(dev); - dev->reloc = reloc; - CU_ASSERT_PTR_NOT_NULL_FATAL(reloc); - ftl_reloc_resume(reloc); - - *_dev = dev; - *_reloc = reloc; -} - -static void -cleanup_reloc(struct spdk_ftl_dev *dev, struct ftl_reloc *reloc) -{ - size_t i; - - for (i = 0; i < ftl_get_num_bands(reloc->dev); ++i) { - SPDK_CU_ASSERT_FATAL(reloc->brelocs[i].state == FTL_BAND_RELOC_STATE_INACTIVE); - } - - ftl_reloc_free(reloc); - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - test_free_ftl_band(&dev->bands[i]); - } - test_free_ftl_dev(dev); -} - -static void -set_band_valid_map(struct ftl_band *band, size_t offset, size_t num_blocks) -{ - struct ftl_lba_map *lba_map = &band->lba_map; - size_t i; - - SPDK_CU_ASSERT_FATAL(lba_map != NULL); - for (i = offset; i < offset + num_blocks; ++i) { - spdk_bit_array_set(lba_map->vld, i); - lba_map->num_vld++; - } -} - -static void -test_reloc_iter_full(void) -{ - size_t num_blocks, num_iters, reminder, i; - struct spdk_ftl_dev *dev; - struct ftl_reloc *reloc; - struct ftl_band_reloc *breloc; - struct ftl_band *band; - struct ftl_addr addr; - - setup_reloc(&dev, &reloc, &g_geo); - - g_geo.zone_size = 100; - breloc = &reloc->brelocs[0]; - band = breloc->band; - - set_band_valid_map(band, 0, ftl_get_num_blocks_in_band(dev)); - - ftl_reloc_add(reloc, band, 0, ftl_get_num_blocks_in_band(dev), 0, true); - - CU_ASSERT_EQUAL(breloc->num_blocks, ftl_get_num_blocks_in_band(dev)); - - num_iters = ftl_get_num_punits(dev) * - (ftl_get_num_blocks_in_zone(dev) / reloc->xfer_size); - - for (i = 0; i < num_iters; i++) { - num_blocks = ftl_reloc_next_blocks(breloc, &addr); - CU_ASSERT_EQUAL(num_blocks, reloc->xfer_size); - } - - num_iters = ftl_get_num_punits(dev); - - /* ftl_reloc_next_blocks is searching for maximum xfer_size */ - /* contiguous valid logic blocks in zone, so we can end up */ - /* with some reminder if number of logical blocks in zone */ - /* is not divisible by xfer_size */ - reminder = ftl_get_num_blocks_in_zone(dev) % reloc->xfer_size; - for (i = 0; i < num_iters; i++) { - num_blocks = ftl_reloc_next_blocks(breloc, &addr); - CU_ASSERT_EQUAL(reminder, num_blocks); - } - - /* num_blocks should remain intact since all the blocks are valid */ - CU_ASSERT_EQUAL(breloc->num_blocks, ftl_get_num_blocks_in_band(dev)); - breloc->state = FTL_BAND_RELOC_STATE_INACTIVE; - - cleanup_reloc(dev, reloc); -} - -static void -test_reloc_empty_band(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_reloc *reloc; - struct ftl_band_reloc *breloc; - struct ftl_band *band; - - setup_reloc(&dev, &reloc, &g_geo); - - breloc = &reloc->brelocs[0]; - band = breloc->band; - - ftl_reloc_add(reloc, band, 0, ftl_get_num_blocks_in_band(dev), 0, true); - - CU_ASSERT_EQUAL(breloc->num_blocks, 0); - - cleanup_reloc(dev, reloc); -} - -static void -test_reloc_full_band(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_reloc *reloc; - struct ftl_band_reloc *breloc; - struct ftl_band *band; - size_t num_moves, num_iters, num_block, i; - - setup_reloc(&dev, &reloc, &g_geo); - - breloc = &reloc->brelocs[0]; - band = breloc->band; - num_moves = MAX_RELOC_QDEPTH * reloc->xfer_size; - num_iters = ftl_get_num_blocks_in_band(dev) / num_moves; - - set_band_valid_map(band, 0, ftl_get_num_blocks_in_band(dev)); - - ftl_reloc_add(reloc, band, 0, ftl_get_num_blocks_in_band(dev), 0, true); - - CU_ASSERT_EQUAL(breloc->num_blocks, ftl_get_num_blocks_in_band(dev)); - - ftl_reloc_prep(breloc); - add_to_active_queue(reloc, breloc); - - for (i = 1; i <= num_iters; ++i) { - single_reloc_move(breloc); - num_block = ftl_get_num_blocks_in_band(dev) - (i * num_moves); - CU_ASSERT_EQUAL(breloc->num_blocks, num_block); - - } - - /* Process reminder blocks */ - single_reloc_move(breloc); - /* Drain move queue */ - ftl_reloc_process_moves(breloc); - - CU_ASSERT_EQUAL(breloc->num_blocks, 0); - CU_ASSERT_TRUE(ftl_reloc_done(breloc)); - ftl_reloc_release(breloc); - - cleanup_reloc(dev, reloc); -} - -static void -test_reloc_scatter_band(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_reloc *reloc; - struct ftl_band_reloc *breloc; - struct ftl_band *band; - size_t num_iters, i; - - setup_reloc(&dev, &reloc, &g_geo); - - breloc = &reloc->brelocs[0]; - band = breloc->band; - num_iters = spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), MAX_RELOC_QDEPTH * 2); - - for (i = 0; i < ftl_get_num_blocks_in_band(dev); ++i) { - if (i % 2) { - set_band_valid_map(band, i, 1); - } - } - - ftl_reloc_add(reloc, band, 0, ftl_get_num_blocks_in_band(dev), 0, true); - ftl_reloc_prep(breloc); - add_to_active_queue(reloc, breloc); - - CU_ASSERT_EQUAL(breloc->num_blocks, ftl_get_num_blocks_in_band(dev)); - - for (i = 0; i < num_iters ; ++i) { - single_reloc_move(breloc); - } - - ftl_process_reloc(breloc); - CU_ASSERT_EQUAL(breloc->num_blocks, 0); - CU_ASSERT_TRUE(ftl_reloc_done(breloc)); - - cleanup_reloc(dev, reloc); -} - -static void -test_reloc_zone(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_reloc *reloc; - struct ftl_band_reloc *breloc; - struct ftl_band *band; - size_t num_io, num_iters, num_block, i; - - setup_reloc(&dev, &reloc, &g_geo); - - breloc = &reloc->brelocs[0]; - band = breloc->band; - /* High priority band have allocated lba map */ - band->high_prio = 1; - ftl_band_alloc_lba_map(band); - num_io = MAX_RELOC_QDEPTH * reloc->xfer_size; - num_iters = ftl_get_num_blocks_in_zone(dev) / num_io; - - set_band_valid_map(band, 0, ftl_get_num_blocks_in_band(dev)); - - ftl_reloc_add(reloc, band, ftl_get_num_blocks_in_zone(dev) * 3, - ftl_get_num_blocks_in_zone(dev), 1, false); - add_to_active_queue(reloc, breloc); - - CU_ASSERT_EQUAL(breloc->num_blocks, ftl_get_num_blocks_in_zone(dev)); - - for (i = 1; i <= num_iters ; ++i) { - single_reloc_move(breloc); - num_block = ftl_get_num_blocks_in_zone(dev) - (i * num_io); - - CU_ASSERT_EQUAL(breloc->num_blocks, num_block); - } - - /* In case num_blocks_in_zone % num_io != 0 one extra iteration is needed */ - single_reloc_move(breloc); - /* Drain move queue */ - ftl_reloc_process_moves(breloc); - - CU_ASSERT_EQUAL(breloc->num_blocks, 0); - CU_ASSERT_TRUE(ftl_reloc_done(breloc)); - ftl_reloc_release(breloc); - - cleanup_reloc(dev, reloc); -} - -static void -test_reloc_single_block(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_reloc *reloc; - struct ftl_band_reloc *breloc; - struct ftl_band *band; -#define TEST_RELOC_OFFSET 6 - - setup_reloc(&dev, &reloc, &g_geo); - - breloc = &reloc->brelocs[0]; - band = breloc->band; - - set_band_valid_map(band, TEST_RELOC_OFFSET, 1); - - ftl_reloc_add(reloc, band, TEST_RELOC_OFFSET, 1, 0, false); - SPDK_CU_ASSERT_FATAL(breloc == TAILQ_FIRST(&reloc->pending_queue)); - ftl_reloc_prep(breloc); - add_to_active_queue(reloc, breloc); - - CU_ASSERT_EQUAL(breloc->num_blocks, 1); - - single_reloc_move(breloc); - /* Drain move queue */ - ftl_reloc_process_moves(breloc); - - CU_ASSERT_EQUAL(breloc->num_blocks, 0); - CU_ASSERT_TRUE(ftl_reloc_done(breloc)); - ftl_reloc_release(breloc); - - cleanup_reloc(dev, reloc); -} - -int -main(int argc, char **argv) -{ - CU_pSuite suite = NULL; - unsigned int num_failures; - - CU_set_error_action(CUEA_ABORT); - CU_initialize_registry(); - - suite = CU_add_suite("ftl_band_suite", NULL, NULL); - - - CU_ADD_TEST(suite, test_reloc_iter_full); - CU_ADD_TEST(suite, test_reloc_empty_band); - CU_ADD_TEST(suite, test_reloc_full_band); - CU_ADD_TEST(suite, test_reloc_scatter_band); - CU_ADD_TEST(suite, test_reloc_zone); - CU_ADD_TEST(suite, test_reloc_single_block); - - CU_basic_set_mode(CU_BRM_VERBOSE); - CU_basic_run_tests(); - num_failures = CU_get_number_of_failures(); - CU_cleanup_registry(); - - return num_failures; -} diff --git a/test/unit/lib/ftl/ftl_wptr/Makefile b/test/unit/lib/ftl/ftl_wptr/Makefile deleted file mode 100644 index 2891a8010..000000000 --- a/test/unit/lib/ftl/ftl_wptr/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) Intel Corporation. -# All rights reserved. -# - -SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../..) - -TEST_FILE = ftl_wptr_ut.c - -include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk diff --git a/test/unit/lib/ftl/ftl_wptr/ftl_wptr_ut.c b/test/unit/lib/ftl/ftl_wptr/ftl_wptr_ut.c deleted file mode 100644 index 85cc532c8..000000000 --- a/test/unit/lib/ftl/ftl_wptr/ftl_wptr_ut.c +++ /dev/null @@ -1,300 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) Intel Corporation. - * All rights reserved. - */ - -#include "spdk/stdinc.h" - -#include "spdk_cunit.h" -#include "common/lib/test_env.c" - -#include "ftl/ftl_core.c" -#include "ftl/ftl_band.c" -#include "ftl/ftl_init.c" -#include "../common/utils.c" - -struct base_bdev_geometry g_geo = { - .write_unit_size = 16, - .optimal_open_zones = 12, - .zone_size = 128, - .blockcnt = 20 * 128 * 12, -}; - -#if defined(DEBUG) -DEFINE_STUB(ftl_band_validate_md, bool, (struct ftl_band *band), true); -DEFINE_STUB_V(ftl_trace_limits, (struct spdk_ftl_dev *dev, int limit, size_t num_free)); - -DEFINE_STUB_V(ftl_trace_completion, (struct spdk_ftl_dev *dev, const struct ftl_io *io, - enum ftl_trace_completion completion)); -DEFINE_STUB_V(ftl_trace_defrag_band, (struct spdk_ftl_dev *dev, const struct ftl_band *band)); -DEFINE_STUB_V(ftl_trace_wbuf_fill, (struct spdk_ftl_dev *dev, const struct ftl_io *io)); -DEFINE_STUB_V(ftl_trace_wbuf_pop, (struct spdk_ftl_dev *dev, const struct ftl_wbuf_entry *entry)); -DEFINE_STUB_V(ftl_trace_write_band, (struct spdk_ftl_dev *dev, const struct ftl_band *band)); -DEFINE_STUB_V(ftl_trace_submission, (struct spdk_ftl_dev *dev, const struct ftl_io *io, - struct ftl_addr addr, size_t addr_cnt)); -#endif -#if defined(FTL_META_DEBUG) -DEFINE_STUB_V(ftl_dev_dump_bands, (struct spdk_ftl_dev *dev)); -#endif -#if defined(FTL_DUMP_STATS) -DEFINE_STUB_V(ftl_dev_dump_stats, (const struct spdk_ftl_dev *dev)); -#endif -DEFINE_STUB_V(ftl_io_call_foreach_child, - (struct ftl_io *io, int (*callback)(struct ftl_io *))); -DEFINE_STUB(ftl_io_current_lba, uint64_t, (const struct ftl_io *io), 0); -DEFINE_STUB_V(ftl_io_dec_req, (struct ftl_io *io)); -DEFINE_STUB_V(ftl_io_free, (struct ftl_io *io)); -DEFINE_STUB(ftl_io_get_lba, uint64_t, - (const struct ftl_io *io, size_t offset), 0); -DEFINE_STUB_V(ftl_io_inc_req, (struct ftl_io *io)); -DEFINE_STUB(ftl_io_iovec_addr, void *, (struct ftl_io *io), NULL); -DEFINE_STUB(ftl_io_iovec_len_left, size_t, (struct ftl_io *io), 0); -DEFINE_STUB_V(ftl_io_fail, (struct ftl_io *io, int status)); -DEFINE_STUB(ftl_io_init_internal, struct ftl_io *, - (const struct ftl_io_init_opts *opts), NULL); -DEFINE_STUB_V(ftl_io_reset, (struct ftl_io *io)); -DEFINE_STUB(ftl_iovec_num_blocks, size_t, - (struct iovec *iov, size_t iov_cnt), 0); -DEFINE_STUB_V(ftl_io_process_error, (struct ftl_io *io, const struct spdk_nvme_cpl *status)); -DEFINE_STUB_V(ftl_io_shrink_iovec, (struct ftl_io *io, size_t num_blocks)); -DEFINE_STUB(ftl_io_wbuf_init, struct ftl_io *, - (struct spdk_ftl_dev *dev, struct ftl_addr addr, - struct ftl_band *band, struct ftl_batch *batch, ftl_io_fn cb), NULL); -DEFINE_STUB(ftl_io_user_init, struct ftl_io *, - (struct spdk_io_channel *ioch, uint64_t lba, size_t num_blocks, - struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, - void *cb_arg, int type), NULL); -DEFINE_STUB(ftl_reloc, bool, (struct ftl_reloc *reloc), false); -DEFINE_STUB_V(ftl_reloc_add, (struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, - size_t num_blocks, int prio, bool defrag)); -DEFINE_STUB_V(ftl_reloc_free, (struct ftl_reloc *reloc)); -DEFINE_STUB_V(ftl_reloc_halt, (struct ftl_reloc *reloc)); -DEFINE_STUB(ftl_reloc_init, struct ftl_reloc *, (struct spdk_ftl_dev *dev), NULL); -DEFINE_STUB(ftl_reloc_is_defrag_active, bool, (const struct ftl_reloc *reloc), false); -DEFINE_STUB(ftl_reloc_is_halted, bool, (const struct ftl_reloc *reloc), false); -DEFINE_STUB_V(ftl_reloc_resume, (struct ftl_reloc *reloc)); -DEFINE_STUB(ftl_restore_device, int, - (struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg), 0); -DEFINE_STUB(ftl_restore_md, int, - (struct spdk_ftl_dev *dev, ftl_restore_fn cb, void *cb_arg), 0); -DEFINE_STUB_V(ftl_restore_nv_cache, - (struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg)); - -DEFINE_STUB_V(spdk_bdev_close, (struct spdk_bdev_desc *desc)); -DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io)); -DEFINE_STUB(spdk_bdev_get_block_size, uint32_t, (const struct spdk_bdev *bdev), 512); -DEFINE_STUB(spdk_bdev_get_buf_align, size_t, (const struct spdk_bdev *bdev), 64); -DEFINE_STUB(spdk_bdev_get_by_name, struct spdk_bdev *, (const char *bdev_name), NULL); -DEFINE_STUB(spdk_bdev_get_dif_type, enum spdk_dif_type, - (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_get_md_size, uint32_t, (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_get_media_events, size_t, - (struct spdk_bdev_desc *bdev_desc, struct spdk_bdev_media_event *events, - size_t max_events), 0); -DEFINE_STUB(spdk_bdev_get_name, const char *, (const struct spdk_bdev *bdev), "test"); -DEFINE_STUB(spdk_bdev_get_num_blocks, uint64_t, (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_get_write_unit_size, uint32_t, - (const struct spdk_bdev *bdev), 0); -DEFINE_STUB(spdk_bdev_get_zone_info, int, - (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - uint64_t zone_id, size_t num_zones, struct spdk_bdev_zone_info *info, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_io_get_append_location, uint64_t, (struct spdk_bdev_io *bdev_io), 0); -DEFINE_STUB(spdk_bdev_io_type_supported, bool, (struct spdk_bdev *bdev, - enum spdk_bdev_io_type io_type), true); -DEFINE_STUB(spdk_bdev_is_md_separate, bool, (const struct spdk_bdev *bdev), false); -DEFINE_STUB(spdk_bdev_is_zoned, bool, (const struct spdk_bdev *bdev), false); -DEFINE_STUB(spdk_bdev_module_claim_bdev, int, - (struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, - struct spdk_bdev_module *module), 0); -DEFINE_STUB_V(spdk_bdev_module_release_bdev, (struct spdk_bdev *bdev)); -DEFINE_STUB(spdk_bdev_open_ext, int, - (const char *bdev_name, bool write, spdk_bdev_event_cb_t event_cb, - void *event_ctx, struct spdk_bdev_desc **desc), 0); -DEFINE_STUB(spdk_bdev_read_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_write_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - void *buf, uint64_t offset_blocks, uint64_t num_blocks, spdk_bdev_io_completion_cb cb, - void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_write_blocks_with_md, int, (struct spdk_bdev_desc *desc, - struct spdk_io_channel *ch, void *buf, void *md, uint64_t offset_blocks, - uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_write_zeroes_blocks, int, - (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_writev_blocks, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - struct iovec *iov, int iovcnt, uint64_t offset_blocks, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_zone_appendv, int, (struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, - struct iovec *iov, int iovcnt, uint64_t zone_id, uint64_t num_blocks, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_zone_management, int, (struct spdk_bdev_desc *desc, - struct spdk_io_channel *ch, - uint64_t zone_id, enum spdk_bdev_zone_action action, - spdk_bdev_io_completion_cb cb, void *cb_arg), 0); -DEFINE_STUB(spdk_mempool_create_ctor, struct spdk_mempool *, - (const char *name, size_t count, size_t ele_size, size_t cache_size, - int socket_id, spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg), NULL); -DEFINE_STUB(spdk_mempool_obj_iter, uint32_t, - (struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, void *obj_cb_arg), 0); - -#ifdef SPDK_CONFIG_PMDK -DEFINE_STUB_V(pmem_persist, (const void *addr, size_t len)); -DEFINE_STUB(pmem_map_file, void *, - (const char *path, size_t len, int flags, mode_t mode, - size_t *mapped_lenp, int *is_pmemp), NULL); -DEFINE_STUB(pmem_unmap, int, (void *addr, size_t len), 0); -DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL); -#endif - -struct spdk_io_channel * -spdk_bdev_get_io_channel(struct spdk_bdev_desc *bdev_desc) -{ - return spdk_get_io_channel(bdev_desc); -} - -struct ftl_io * -ftl_io_erase_init(struct ftl_band *band, size_t num_blocks, ftl_io_fn cb) -{ - struct ftl_io *io; - - io = calloc(1, sizeof(struct ftl_io)); - SPDK_CU_ASSERT_FATAL(io != NULL); - - io->dev = band->dev; - io->band = band; - io->cb_fn = cb; - io->num_blocks = 1; - - return io; -} - -void -ftl_io_advance(struct ftl_io *io, size_t num_blocks) -{ - io->pos += num_blocks; -} - -void -ftl_io_complete(struct ftl_io *io) -{ - io->cb_fn(io, NULL, 0); - free(io); -} - -static void -setup_wptr_test(struct spdk_ftl_dev **dev, const struct base_bdev_geometry *geo) -{ - struct spdk_ftl_dev *t_dev; - struct _ftl_io_channel *_ioch; - size_t i; - - t_dev = test_init_ftl_dev(geo); - for (i = 0; i < ftl_get_num_bands(t_dev); ++i) { - test_init_ftl_band(t_dev, i, geo->zone_size); - t_dev->bands[i].state = FTL_BAND_STATE_CLOSED; - ftl_band_set_state(&t_dev->bands[i], FTL_BAND_STATE_FREE); - } - - _ioch = (struct _ftl_io_channel *)(t_dev->ioch + 1); - _ioch->ioch = calloc(1, sizeof(*_ioch->ioch)); - SPDK_CU_ASSERT_FATAL(_ioch->ioch != NULL); - - *dev = t_dev; -} - -static void -cleanup_wptr_test(struct spdk_ftl_dev *dev) -{ - struct _ftl_io_channel *_ioch; - size_t i; - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - dev->bands[i].lba_map.segments = NULL; - test_free_ftl_band(&dev->bands[i]); - } - - _ioch = (struct _ftl_io_channel *)(dev->ioch + 1); - free(_ioch->ioch); - - test_free_ftl_dev(dev); -} - -static void -test_wptr(void) -{ - struct spdk_ftl_dev *dev; - struct ftl_wptr *wptr; - struct ftl_band *band; - struct ftl_io io = { 0 }; - size_t xfer_size; - size_t zone, block, offset, i; - int rc; - - setup_wptr_test(&dev, &g_geo); - - xfer_size = dev->xfer_size; - ftl_add_wptr(dev); - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - wptr = LIST_FIRST(&dev->wptr_list); - band = wptr->band; - ftl_band_set_state(band, FTL_BAND_STATE_OPENING); - ftl_band_set_state(band, FTL_BAND_STATE_OPEN); - io.band = band; - io.dev = dev; - - for (block = 0, offset = 0; block < ftl_get_num_blocks_in_zone(dev) / xfer_size; ++block) { - for (zone = 0; zone < band->num_zones; ++zone) { - CU_ASSERT_EQUAL(wptr->offset, offset); - ftl_wptr_advance(wptr, xfer_size); - offset += xfer_size; - } - } - - CU_ASSERT_EQUAL(band->state, FTL_BAND_STATE_FULL); - - ftl_band_set_state(band, FTL_BAND_STATE_CLOSING); - - /* Call the metadata completion cb to force band state change */ - /* and removal of the actual wptr */ - ftl_md_write_cb(&io, NULL, 0); - CU_ASSERT_EQUAL(band->state, FTL_BAND_STATE_CLOSED); - CU_ASSERT_TRUE(LIST_EMPTY(&dev->wptr_list)); - - rc = ftl_add_wptr(dev); - - /* There are no free bands during the last iteration, so */ - /* there'll be no new wptr allocation */ - if (i == (ftl_get_num_bands(dev) - 1)) { - CU_ASSERT_EQUAL(rc, -1); - } else { - CU_ASSERT_EQUAL(rc, 0); - } - } - - cleanup_wptr_test(dev); -} - -int -main(int argc, char **argv) -{ - CU_pSuite suite = NULL; - unsigned int num_failures; - - CU_set_error_action(CUEA_ABORT); - CU_initialize_registry(); - - suite = CU_add_suite("ftl_wptr_suite", NULL, NULL); - - - CU_ADD_TEST(suite, test_wptr); - - CU_basic_set_mode(CU_BRM_VERBOSE); - CU_basic_run_tests(); - num_failures = CU_get_number_of_failures(); - CU_cleanup_registry(); - - return num_failures; -} diff --git a/test/unit/unittest.sh b/test/unit/unittest.sh index d131fbba2..8007085df 100755 --- a/test/unit/unittest.sh +++ b/test/unit/unittest.sh @@ -44,15 +44,6 @@ function unittest_event() { $valgrind $testdir/lib/event/reactor.c/reactor_ut } -function unittest_ftl() { - $valgrind $testdir/lib/ftl/ftl_ppa/ftl_ppa_ut - $valgrind $testdir/lib/ftl/ftl_band.c/ftl_band_ut - $valgrind $testdir/lib/ftl/ftl_reloc.c/ftl_reloc_ut - $valgrind $testdir/lib/ftl/ftl_wptr/ftl_wptr_ut - $valgrind $testdir/lib/ftl/ftl_md/ftl_md_ut - $valgrind $testdir/lib/ftl/ftl_io.c/ftl_io_ut -} - function unittest_iscsi() { $valgrind $testdir/lib/iscsi/conn.c/conn_ut $valgrind $testdir/lib/iscsi/param.c/param_ut @@ -215,9 +206,6 @@ fi run_test "unittest_blob_blobfs" unittest_blob run_test "unittest_event" unittest_event -if [ $(uname -s) = Linux ]; then - run_test "unittest_ftl" unittest_ftl -fi run_test "unittest_accel" $valgrind $testdir/lib/accel/accel.c/accel_engine_ut run_test "unittest_ioat" $valgrind $testdir/lib/ioat/ioat.c/ioat_ut