diff --git a/include/spdk/ftl.h b/include/spdk/ftl.h index 4e0d1fcd9..2638d1cec 100644 --- a/include/spdk/ftl.h +++ b/include/spdk/ftl.h @@ -35,8 +35,6 @@ #define SPDK_FTL_H #include "spdk/stdinc.h" -#include "spdk/nvme.h" -#include "spdk/nvme_ocssd.h" #include "spdk/uuid.h" #include "spdk/thread.h" #include "spdk/bdev.h" @@ -109,10 +107,8 @@ enum spdk_ftl_mode { }; struct spdk_ftl_dev_init_opts { - /* NVMe controller */ - struct spdk_nvme_ctrlr *ctrlr; - /* Controller's transport ID */ - struct spdk_nvme_transport_id trid; + /* Underlying device */ + struct spdk_bdev_desc *base_bdev_desc; /* Write buffer cache */ struct spdk_bdev_desc *cache_bdev_desc; @@ -155,8 +151,7 @@ typedef void (*spdk_ftl_init_fn)(struct spdk_ftl_dev *, void *, int); * Initialize the FTL on given NVMe device and parallel unit range. * * Covers the following: - * - initialize and register NVMe ctrlr, - * - retrieve geometry and check if the device has proper configuration, + * - retrieve zone device information, * - allocate buffers and resources, * - initialize internal structures, * - initialize internal thread(s), diff --git a/lib/ftl/ftl_addr.h b/lib/ftl/ftl_addr.h index 588faaa22..9f74fe4e2 100644 --- a/lib/ftl/ftl_addr.h +++ b/lib/ftl/ftl_addr.h @@ -80,22 +80,4 @@ struct ftl_addr { }; }; -struct ftl_ppa_fmt { - /* Logical block */ - unsigned int lbk_offset; - unsigned int lbk_mask; - - /* Chunk */ - unsigned int chk_offset; - unsigned int chk_mask; - - /* Parallel unit (NAND die) */ - unsigned int pu_offset; - unsigned int pu_mask; - - /* Group */ - unsigned int grp_offset; - unsigned int grp_mask; -}; - #endif /* FTL_ADDR_H */ diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index abefe7624..4544bbba1 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -161,12 +161,12 @@ ftl_remove_wptr(struct ftl_wptr *wptr) } static void -ftl_io_cmpl_cb(void *arg, const struct spdk_nvme_cpl *status) +ftl_io_cmpl_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { - struct ftl_io *io = arg; + struct ftl_io *io = cb_arg; - if (spdk_nvme_cpl_is_error(status)) { - ftl_io_process_error(io, status); + if (spdk_unlikely(!success)) { + io->status = -EIO; } ftl_trace_completion(io->dev, io, FTL_TRACE_COMPLETION_DISK); @@ -175,6 +175,8 @@ ftl_io_cmpl_cb(void *arg, const struct spdk_nvme_cpl *status) if (ftl_io_done(io)) { ftl_io_complete(io); } + + spdk_bdev_free_io(bdev_io); } static void @@ -340,11 +342,13 @@ ftl_submit_erase(struct ftl_io *io) struct spdk_ftl_dev *dev = io->dev; struct ftl_band *band = io->band; struct ftl_addr addr = io->addr; + struct ftl_io_channel *ioch; struct ftl_zone *zone; - uint64_t addr_packed; int rc = 0; size_t i; + ioch = spdk_io_channel_get_ctx(ftl_get_io_channel(dev)); + for (i = 0; i < io->lbk_cnt; ++i) { if (i != 0) { zone = ftl_band_next_zone(band, ftl_band_zone_from_addr(band, addr)); @@ -353,11 +357,11 @@ ftl_submit_erase(struct ftl_io *io) } assert(addr.offset == 0); - addr_packed = ftl_addr_addr_pack(dev, addr); ftl_trace_submission(dev, io, addr, 1); - rc = spdk_nvme_ocssd_ns_cmd_vector_reset(dev->ns, ftl_get_write_qpair(dev), - &addr_packed, 1, NULL, ftl_io_cmpl_cb, io); + rc = spdk_bdev_zone_management(dev->base_bdev_desc, ioch->base_ioch, + ftl_block_offset_from_addr(dev, addr), + SPDK_BDEV_ZONE_RESET, ftl_io_cmpl_cb, io); if (spdk_unlikely(rc)) { ftl_io_fail(io, rc); SPDK_ERRLOG("Vector reset failed with status: %d\n", rc); @@ -957,7 +961,8 @@ ftl_read_next_logical_addr(struct ftl_io *io, struct ftl_addr *addr) break; } - if (ftl_addr_addr_pack(dev, *addr) + i != ftl_addr_addr_pack(dev, next_addr)) { + if (ftl_block_offset_from_addr(dev, *addr) + i != + ftl_block_offset_from_addr(dev, next_addr)) { break; } } @@ -969,9 +974,12 @@ static int ftl_submit_read(struct ftl_io *io) { struct spdk_ftl_dev *dev = io->dev; + struct ftl_io_channel *ioch; struct ftl_addr addr; int rc = 0, lbk_cnt; + ioch = spdk_io_channel_get_ctx(io->ioch); + assert(LIST_EMPTY(&io->children)); while (io->pos < io->lbk_cnt) { @@ -1000,10 +1008,10 @@ ftl_submit_read(struct ftl_io *io) assert(lbk_cnt > 0); ftl_trace_submission(dev, io, addr, lbk_cnt); - rc = spdk_nvme_ns_cmd_read(dev->ns, ftl_get_read_qpair(dev), + rc = spdk_bdev_read_blocks(dev->base_bdev_desc, ioch->base_ioch, ftl_io_iovec_addr(io), - ftl_addr_addr_pack(io->dev, addr), lbk_cnt, - ftl_io_cmpl_cb, io, 0); + ftl_block_offset_from_addr(dev, addr), + lbk_cnt, ftl_io_cmpl_cb, io); if (spdk_unlikely(rc)) { if (rc == -ENOMEM) { ftl_add_to_retry_queue(io); @@ -1496,9 +1504,12 @@ static int ftl_submit_child_write(struct ftl_wptr *wptr, struct ftl_io *io, int lbk_cnt) { struct spdk_ftl_dev *dev = io->dev; + struct ftl_io_channel *ioch; struct ftl_io *child; - int rc; struct ftl_addr addr; + int rc; + + ioch = spdk_io_channel_get_ctx(io->ioch); if (spdk_likely(!wptr->direct_mode)) { addr = wptr->addr; @@ -1516,15 +1527,16 @@ ftl_submit_child_write(struct ftl_wptr *wptr, struct ftl_io *io, int lbk_cnt) } wptr->num_outstanding++; - rc = spdk_nvme_ns_cmd_write_with_md(dev->ns, ftl_get_write_qpair(dev), - ftl_io_iovec_addr(child), child->md, - ftl_addr_addr_pack(dev, addr), - lbk_cnt, ftl_io_cmpl_cb, child, 0, 0, 0); + + rc = spdk_bdev_write_blocks(dev->base_bdev_desc, ioch->base_ioch, + ftl_io_iovec_addr(child), + ftl_block_offset_from_addr(dev, addr), + lbk_cnt, ftl_io_cmpl_cb, child); if (rc) { wptr->num_outstanding--; ftl_io_fail(child, rc); ftl_io_complete(child); - SPDK_ERRLOG("spdk_nvme_ns_cmd_write_with_md failed with status:%d, addr:%lu\n", + SPDK_ERRLOG("spdk_bdev_write_blocks_with_md failed with status:%d, addr:%lu\n", rc, addr.addr); return -EIO; } @@ -1676,7 +1688,7 @@ ftl_wptr_process_writes(struct ftl_wptr *wptr) } SPDK_DEBUGLOG(SPDK_LOG_FTL_CORE, "Write addr:%lx, %lx\n", wptr->addr.addr, - ftl_addr_addr_pack(dev, wptr->addr)); + ftl_block_offset_from_addr(dev, wptr->addr)); if (ftl_submit_write(wptr, io)) { /* TODO: we need some recovery here */ @@ -1887,8 +1899,8 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at attrs->lbk_cnt = dev->num_lbas; attrs->lbk_size = FTL_BLOCK_SIZE; attrs->cache_bdev_desc = dev->nv_cache.bdev_desc; - attrs->num_zones = dev->geo.num_chk; - attrs->zone_size = dev->geo.clba; + attrs->num_zones = ftl_get_num_zones(dev); + attrs->zone_size = ftl_get_num_blocks_in_zone(dev); attrs->conf = dev->conf; } @@ -2154,8 +2166,6 @@ ftl_task_read(void *ctx) { struct ftl_thread *thread = ctx; struct spdk_ftl_dev *dev = thread->dev; - struct spdk_nvme_qpair *qpair = ftl_get_read_qpair(dev); - size_t num_completed; if (dev->halt) { if (ftl_shutdown_complete(dev)) { @@ -2164,13 +2174,12 @@ ftl_task_read(void *ctx) } } - num_completed = spdk_nvme_qpair_process_completions(qpair, 0); - - if (num_completed && !TAILQ_EMPTY(&dev->retry_queue)) { + if (!TAILQ_EMPTY(&dev->retry_queue)) { ftl_process_retry_queue(dev); + return 1; } - return num_completed; + return 0; } int @@ -2178,7 +2187,6 @@ ftl_task_core(void *ctx) { struct ftl_thread *thread = ctx; struct spdk_ftl_dev *dev = thread->dev; - struct spdk_nvme_qpair *qpair = ftl_get_write_qpair(dev); if (dev->halt) { if (ftl_shutdown_complete(dev)) { @@ -2188,7 +2196,6 @@ ftl_task_core(void *ctx) } ftl_process_writes(dev); - spdk_nvme_qpair_process_completions(qpair, 0); ftl_process_relocs(dev); return 0; diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 4263c27f8..c60dafc06 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -35,8 +35,6 @@ #define FTL_CORE_H #include "spdk/stdinc.h" -#include "spdk/nvme.h" -#include "spdk/nvme_ocssd.h" #include "spdk/uuid.h" #include "spdk/thread.h" #include "spdk/util.h" @@ -45,6 +43,7 @@ #include "spdk/queue.h" #include "spdk/ftl.h" #include "spdk/bdev.h" +#include "spdk/bdev_zone.h" #include "ftl_addr.h" #include "ftl_io.h" @@ -78,8 +77,6 @@ struct ftl_stats { struct ftl_thread { /* Owner */ struct spdk_ftl_dev *dev; - /* I/O queue pair */ - struct spdk_nvme_qpair *qpair; /* Thread on which the poller is running */ struct spdk_thread *thread; @@ -158,12 +155,8 @@ struct spdk_ftl_dev { /* Destruction context */ struct ftl_init_context fini_ctx; - /* NVMe controller */ - struct spdk_nvme_ctrlr *ctrlr; - /* NVMe namespace */ - struct spdk_nvme_ns *ns; - /* NVMe transport ID */ - struct spdk_nvme_transport_id trid; + /* Underlying device */ + struct spdk_bdev_desc *base_bdev_desc; /* Non-volatile write buffer cache */ struct ftl_nv_cache nv_cache; @@ -201,12 +194,8 @@ struct spdk_ftl_dev { /* Size of the l2p table */ uint64_t num_lbas; - /* PPA format */ - struct ftl_ppa_fmt ppaf; /* Address size */ size_t addr_len; - /* Device's geometry */ - struct spdk_ocssd_geometry_data geo; /* Flush list */ LIST_HEAD(, ftl_flush) flush_list; @@ -279,9 +268,6 @@ int ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb); int ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb); void ftl_restore_nv_cache(struct ftl_restore *restore, ftl_restore_fn cb); int ftl_band_set_direct_access(struct ftl_band *band, bool access); -int ftl_retrieve_chunk_info(struct spdk_ftl_dev *dev, struct ftl_addr addr, - struct spdk_ocssd_chunk_information_entry *info, - unsigned int num_entries); bool ftl_addr_is_written(struct ftl_band *band, struct ftl_addr addr); int ftl_flush_active_bands(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); int ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, @@ -304,22 +290,46 @@ ftl_get_core_thread(const struct spdk_ftl_dev *dev) return dev->core_thread.thread; } -static inline struct spdk_nvme_qpair * -ftl_get_write_qpair(const struct spdk_ftl_dev *dev) -{ - return dev->core_thread.qpair; -} - static inline struct spdk_thread * ftl_get_read_thread(const struct spdk_ftl_dev *dev) { return dev->read_thread.thread; } -static inline struct spdk_nvme_qpair * -ftl_get_read_qpair(const struct spdk_ftl_dev *dev) +static inline size_t +ftl_get_num_bands(const struct spdk_ftl_dev *dev) { - return dev->read_thread.qpair; + return dev->num_bands; +} + +static inline size_t +ftl_get_num_punits(const struct spdk_ftl_dev *dev) +{ + return spdk_bdev_get_optimal_open_zones(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); +} + +static inline size_t +ftl_get_num_zones(const struct spdk_ftl_dev *dev) +{ + return ftl_get_num_bands(dev) * ftl_get_num_punits(dev); +} + +static inline size_t +ftl_get_num_blocks_in_zone(const struct spdk_ftl_dev *dev) +{ + return spdk_bdev_get_zone_size(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); +} + +static inline uint64_t +ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev) +{ + return ftl_get_num_punits(dev) * ftl_get_num_blocks_in_zone(dev); +} + +static inline size_t +ftl_vld_map_size(const struct spdk_ftl_dev *dev) +{ + return (size_t)spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), CHAR_BIT); } static inline int @@ -340,35 +350,25 @@ ftl_addr_cached(struct ftl_addr addr) return !ftl_addr_invalid(addr) && addr.cached; } -static inline uint64_t -ftl_addr_addr_pack(const struct spdk_ftl_dev *dev, struct ftl_addr addr) +static inline struct ftl_addr +ftl_addr_from_block_offset(const struct spdk_ftl_dev *dev, uint64_t offset) { - uint64_t lbk, chk, pu, grp; + struct ftl_addr addr = {}; + uint64_t zone_num; - lbk = addr.offset; - chk = addr.zone_id; - pu = addr.pu / dev->geo.num_grp; - grp = addr.pu % dev->geo.num_grp; + addr.offset = offset % ftl_get_num_blocks_in_zone(dev); + zone_num = offset / ftl_get_num_blocks_in_zone(dev); + addr.pu = zone_num % ftl_get_num_punits(dev); + addr.zone_id = zone_num / ftl_get_num_punits(dev); - return (lbk << dev->ppaf.lbk_offset) | - (chk << dev->ppaf.chk_offset) | - (pu << dev->ppaf.pu_offset) | - (grp << dev->ppaf.grp_offset); + return addr; } -static inline struct ftl_addr -ftl_addr_addr_unpack(const struct spdk_ftl_dev *dev, uint64_t addr) +static inline uint64_t +ftl_block_offset_from_addr(const struct spdk_ftl_dev *dev, struct ftl_addr addr) { - struct ftl_addr res = {}; - unsigned int pu, grp; - - res.offset = (addr >> dev->ppaf.lbk_offset) & dev->ppaf.lbk_mask; - res.zone_id = (addr >> dev->ppaf.chk_offset) & dev->ppaf.chk_mask; - pu = (addr >> dev->ppaf.pu_offset) & dev->ppaf.pu_mask; - grp = (addr >> dev->ppaf.grp_offset) & dev->ppaf.grp_mask; - res.pu = grp * dev->geo.num_pu + pu; - - return res; + return (addr.zone_id * ftl_get_num_punits(dev) + addr.pu) * + ftl_get_num_blocks_in_zone(dev) + addr.offset; } static inline struct ftl_addr @@ -382,7 +382,7 @@ ftl_addr_to_packed(const struct spdk_ftl_dev *dev, struct ftl_addr addr) p.pack.cached = 1; p.pack.cache_offset = (uint32_t) addr.cache_offset; } else { - p.pack.addr = (uint32_t) ftl_addr_addr_pack(dev, addr); + p.pack.addr = (uint32_t) ftl_block_offset_from_addr(dev, addr); } return p; @@ -399,7 +399,7 @@ ftl_addr_from_packed(const struct spdk_ftl_dev *dev, struct ftl_addr p) addr.cached = 1; addr.cache_offset = p.pack.cache_offset; } else { - addr = ftl_addr_addr_unpack(dev, p.pack.addr); + addr = ftl_addr_from_block_offset(dev, p.pack.addr); } return addr; @@ -450,36 +450,6 @@ ftl_l2p_get(struct spdk_ftl_dev *dev, uint64_t lba) return ftl_to_addr(_ftl_l2p_get64(dev->l2p, lba)); } } -static inline size_t -ftl_get_num_bands(const struct spdk_ftl_dev *dev) -{ - return dev->geo.num_chk; -} - -static inline size_t -ftl_get_num_blocks_in_zone(const struct spdk_ftl_dev *dev) -{ - return dev->geo.clba; -} - -static inline size_t -ftl_get_num_punits(const struct spdk_ftl_dev *dev) -{ - return dev->geo.num_pu * dev->geo.num_grp; -} - -static inline uint64_t -ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev) -{ - return ftl_get_num_punits(dev) * ftl_get_num_blocks_in_zone(dev); -} - -static inline size_t -ftl_vld_map_size(const struct spdk_ftl_dev *dev) -{ - return (size_t)spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), CHAR_BIT); -} - static inline bool ftl_dev_has_nv_cache(const struct spdk_ftl_dev *dev) { diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index 22ee6ab5b..4ff430667 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -52,11 +52,14 @@ #define FTL_CORE_RING_SIZE 4096 #define FTL_INIT_TIMEOUT 30 #define FTL_NSID 1 +#define FTL_ZONE_INFO_COUNT 64 -struct ftl_admin_cmpl { - struct spdk_nvme_cpl status; - - int complete; +struct ftl_dev_init_ctx { + struct spdk_ftl_dev *dev; + struct spdk_ftl_dev_init_opts opts; + struct spdk_io_channel *ioch; + struct spdk_bdev_zone_info info[FTL_ZONE_INFO_COUNT]; + size_t zone_id; }; static STAILQ_HEAD(, spdk_ftl_dev) g_ftl_queue = STAILQ_HEAD_INITIALIZER(g_ftl_queue); @@ -103,17 +106,6 @@ static const struct spdk_ftl_conf g_default_conf = { } }; -static void ftl_dev_free_sync(struct spdk_ftl_dev *dev); - -static void -ftl_admin_cb(void *ctx, const struct spdk_nvme_cpl *cpl) -{ - struct ftl_admin_cmpl *cmpl = ctx; - - cmpl->complete = 1; - cmpl->status = *cpl; -} - static int ftl_band_init_md(struct ftl_band *band) { @@ -130,8 +122,7 @@ ftl_band_init_md(struct ftl_band *band) } static int -ftl_check_conf(const struct spdk_ftl_conf *conf, - const struct spdk_ocssd_geometry_data *geo) +ftl_check_conf(const struct spdk_ftl_dev *dev, const struct spdk_ftl_conf *conf) { size_t i; @@ -150,7 +141,7 @@ ftl_check_conf(const struct spdk_ftl_conf *conf, if (conf->rwb_size % FTL_BLOCK_SIZE != 0) { return -1; } - if (geo->ws_opt % conf->num_interleave_units != 0) { + if (dev->xfer_size % conf->num_interleave_units != 0) { return -1; } @@ -163,127 +154,22 @@ ftl_check_conf(const struct spdk_ftl_conf *conf, return 0; } -int -ftl_retrieve_chunk_info(struct spdk_ftl_dev *dev, struct ftl_addr addr, - struct spdk_ocssd_chunk_information_entry *info, - unsigned int num_entries) -{ - volatile struct ftl_admin_cmpl cmpl = {}; - uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); - unsigned int grp = addr.pu % dev->geo.num_grp; - unsigned int punit = addr.pu / dev->geo.num_grp; - uint64_t offset = (grp * dev->geo.num_pu + punit) * - dev->geo.num_chk + addr.zone_id; - int rc; - - rc = spdk_nvme_ctrlr_cmd_get_log_page(dev->ctrlr, SPDK_OCSSD_LOG_CHUNK_INFO, nsid, - info, num_entries * sizeof(*info), - offset * sizeof(*info), - ftl_admin_cb, (void *)&cmpl); - if (spdk_unlikely(rc != 0)) { - SPDK_ERRLOG("spdk_nvme_ctrlr_cmd_get_log_page: %s\n", spdk_strerror(-rc)); - return -1; - } - - while (!cmpl.complete) { - spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); - } - - if (spdk_nvme_cpl_is_error(&cmpl.status)) { - SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", - cmpl.status.status.sc, cmpl.status.status.sct); - return -1; - } - - return 0; -} - -static int -ftl_retrieve_punit_chunk_info(struct spdk_ftl_dev *dev, unsigned int punit, - struct spdk_ocssd_chunk_information_entry *info) -{ - uint32_t i = 0; - unsigned int num_entries = FTL_BLOCK_SIZE / sizeof(*info); - struct ftl_addr chunk_addr = { .pu = punit }; - char addr_buf[128]; - - for (i = 0; i < dev->geo.num_chk; i += num_entries, chunk_addr.zone_id += num_entries) { - if (num_entries > dev->geo.num_chk - i) { - num_entries = dev->geo.num_chk - i; - } - - if (ftl_retrieve_chunk_info(dev, chunk_addr, &info[i], num_entries)) { - SPDK_ERRLOG("Failed to retrieve chunk information @addr: %s\n", - ftl_addr2str(chunk_addr, addr_buf, sizeof(addr_buf))); - return -1; - } - } - - return 0; -} - -static unsigned char -ftl_get_zone_state(const struct spdk_ocssd_chunk_information_entry *info) -{ - if (info->cs.free) { - return SPDK_BDEV_ZONE_STATE_EMPTY; - } - - if (info->cs.open) { - return SPDK_BDEV_ZONE_STATE_OPEN; - } - - if (info->cs.closed) { - return SPDK_BDEV_ZONE_STATE_CLOSED; - } - - if (info->cs.offline) { - return SPDK_BDEV_ZONE_STATE_OFFLINE; - } - - assert(0 && "Invalid block state"); - return SPDK_BDEV_ZONE_STATE_OFFLINE; -} - -static void -ftl_remove_empty_bands(struct spdk_ftl_dev *dev) -{ - struct ftl_band *band, *temp_band; - - /* Remove band from shut_bands list to prevent further processing */ - /* if all blocks on this band are bad */ - LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { - if (!band->num_zones) { - dev->num_bands--; - LIST_REMOVE(band, list_entry); - } - } -} - static int ftl_dev_init_bands(struct spdk_ftl_dev *dev) { - struct spdk_ocssd_chunk_information_entry *info; - struct ftl_band *band, *pband; - struct ftl_zone *zone; - unsigned int i, j; - int rc = 0; + struct ftl_band *band, *pband; + unsigned int i; + int rc = 0; LIST_INIT(&dev->free_bands); LIST_INIT(&dev->shut_bands); dev->num_free = 0; - dev->num_bands = ftl_get_num_bands(dev); dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); if (!dev->bands) { return -1; } - info = calloc(dev->geo.num_chk, sizeof(*info)); - if (!info) { - return -1; - } - for (i = 0; i < ftl_get_num_bands(dev); ++i) { band = &dev->bands[i]; band->id = i; @@ -302,133 +188,25 @@ ftl_dev_init_bands(struct spdk_ftl_dev *dev) if (!band->zone_buf) { SPDK_ERRLOG("Failed to allocate block state table for band: [%u]\n", i); rc = -1; - goto out; + break; } rc = ftl_band_init_md(band); if (rc) { SPDK_ERRLOG("Failed to initialize metadata structures for band [%u]\n", i); - goto out; + break; } band->reloc_bitmap = spdk_bit_array_create(ftl_get_num_bands(dev)); if (!band->reloc_bitmap) { SPDK_ERRLOG("Failed to allocate band relocation bitmap\n"); - goto out; + break; } } - for (i = 0; i < ftl_get_num_punits(dev); ++i) { - rc = ftl_retrieve_punit_chunk_info(dev, i, info); - if (rc) { - goto out; - } - - for (j = 0; j < ftl_get_num_bands(dev); ++j) { - band = &dev->bands[j]; - zone = &band->zone_buf[i]; - zone->state = ftl_get_zone_state(&info[j]); - zone->start_addr.pu = i; - zone->start_addr.zone_id = band->id; - zone->write_offset = ftl_get_num_blocks_in_zone(dev); - - if (zone->state != SPDK_BDEV_ZONE_STATE_OFFLINE) { - band->num_zones++; - CIRCLEQ_INSERT_TAIL(&band->zones, zone, circleq); - } - } - } - - for (i = 0; i < ftl_get_num_bands(dev); ++i) { - band = &dev->bands[i]; - band->tail_md_addr = ftl_band_tail_md_addr(band); - } - - ftl_remove_empty_bands(dev); -out: - free(info); return rc; } -static int -ftl_dev_retrieve_geo(struct spdk_ftl_dev *dev) -{ - volatile struct ftl_admin_cmpl cmpl = {}; - uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); - - if (spdk_nvme_ocssd_ctrlr_cmd_geometry(dev->ctrlr, nsid, &dev->geo, sizeof(dev->geo), - ftl_admin_cb, (void *)&cmpl)) { - SPDK_ERRLOG("Unable to retrieve geometry\n"); - return -1; - } - - /* TODO: add a timeout */ - while (!cmpl.complete) { - spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); - } - - if (spdk_nvme_cpl_is_error(&cmpl.status)) { - SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", - cmpl.status.status.sc, cmpl.status.status.sct); - return -1; - } - - /* TODO: add sanity checks for the geo */ - dev->addr_len = dev->geo.lbaf.grp_len + - dev->geo.lbaf.pu_len + - dev->geo.lbaf.chk_len + - dev->geo.lbaf.lbk_len; - - dev->ppaf.lbk_offset = 0; - dev->ppaf.lbk_mask = (1 << dev->geo.lbaf.lbk_len) - 1; - dev->ppaf.chk_offset = dev->ppaf.lbk_offset + dev->geo.lbaf.lbk_len; - dev->ppaf.chk_mask = (1 << dev->geo.lbaf.chk_len) - 1; - dev->ppaf.pu_offset = dev->ppaf.chk_offset + dev->geo.lbaf.chk_len; - dev->ppaf.pu_mask = (1 << dev->geo.lbaf.pu_len) - 1; - dev->ppaf.grp_offset = dev->ppaf.pu_offset + dev->geo.lbaf.pu_len; - dev->ppaf.grp_mask = (1 << dev->geo.lbaf.grp_len) - 1; - - /* We're using optimal write size as our xfer size */ - dev->xfer_size = dev->geo.ws_opt; - - return 0; -} - -static int -ftl_dev_nvme_init(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) -{ - uint32_t block_size; - - dev->ctrlr = opts->ctrlr; - - if (spdk_nvme_ctrlr_get_num_ns(dev->ctrlr) != 1) { - SPDK_ERRLOG("Unsupported number of namespaces\n"); - return -1; - } - - dev->ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, FTL_NSID); - if (dev->ns == NULL) { - SPDK_ERRLOG("Invalid NS (%"PRIu32")\n", FTL_NSID); - return -1; - } - dev->trid = opts->trid; - dev->md_size = spdk_nvme_ns_get_md_size(dev->ns); - - block_size = spdk_nvme_ns_get_extended_sector_size(dev->ns); - if (block_size != FTL_BLOCK_SIZE) { - SPDK_ERRLOG("Unsupported block size (%"PRIu32")\n", block_size); - return -1; - } - - if (dev->md_size % sizeof(uint32_t) != 0) { - /* Metadata pointer must be dword aligned */ - SPDK_ERRLOG("Unsupported metadata size (%zu)\n", dev->md_size); - return -1; - } - - return 0; -} - static int ftl_dev_init_nv_cache(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *bdev_desc) { @@ -673,12 +451,6 @@ ftl_dev_init_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread, thread->thread = spdk_thread; thread->period_us = period_us; - thread->qpair = spdk_nvme_ctrlr_alloc_io_qpair(dev->ctrlr, NULL, 0); - if (!thread->qpair) { - SPDK_ERRLOG("Unable to initialize qpair\n"); - return -1; - } - spdk_thread_send_msg(spdk_thread, _ftl_dev_init_thread, thread); return 0; } @@ -709,10 +481,8 @@ ftl_dev_free_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread) assert(thread->poller == NULL); spdk_put_io_channel(thread->ioch); - spdk_nvme_ctrlr_free_io_qpair(thread->qpair); thread->thread = NULL; thread->ioch = NULL; - thread->qpair = NULL; } static int @@ -946,9 +716,9 @@ error: } static int -ftl_restore_state(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) +ftl_restore_state(struct spdk_ftl_dev *dev, struct spdk_uuid uuid) { - dev->uuid = opts->uuid; + dev->uuid = uuid; if (ftl_restore_md(dev, ftl_restore_md_cb)) { SPDK_ERRLOG("Failed to start metadata restoration from the SSD\n"); @@ -958,6 +728,166 @@ ftl_restore_state(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts return 0; } +static void +ftl_dev_update_bands(struct spdk_ftl_dev *dev) +{ + struct ftl_band *band, *temp_band; + size_t i; + + for (i = 0; i < ftl_get_num_bands(dev); ++i) { + band = &dev->bands[i]; + band->tail_md_addr = ftl_band_tail_md_addr(band); + } + + /* Remove band from shut_bands list to prevent further processing */ + /* if all blocks on this band are bad */ + LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { + if (!band->num_zones) { + dev->num_bands--; + LIST_REMOVE(band, list_entry); + } + } +} + +static void +ftl_dev_free_init_ctx(struct ftl_dev_init_ctx *init_ctx) +{ + if (!init_ctx) { + return; + } + + if (init_ctx->ioch) { + spdk_put_io_channel(init_ctx->ioch); + } + + free(init_ctx); +} + +static void +ftl_dev_init_state(struct ftl_dev_init_ctx *init_ctx) +{ + struct spdk_ftl_dev *dev = init_ctx->dev; + + ftl_dev_update_bands(dev); + + if (ftl_dev_init_threads(dev, &init_ctx->opts)) { + SPDK_ERRLOG("Unable to initialize device threads\n"); + goto fail; + } + + if (init_ctx->opts.mode & SPDK_FTL_MODE_CREATE) { + if (ftl_setup_initial_state(dev)) { + SPDK_ERRLOG("Failed to setup initial state of the device\n"); + goto fail; + } + } else { + if (ftl_restore_state(dev, init_ctx->opts.uuid)) { + SPDK_ERRLOG("Unable to restore device's state from the SSD\n"); + goto fail; + } + } + + ftl_dev_free_init_ctx(init_ctx); + return; +fail: + ftl_dev_free_init_ctx(init_ctx); + ftl_init_fail(dev); +} + +static void ftl_dev_get_zone_info(struct ftl_dev_init_ctx *init_ctx); + +static void +ftl_dev_get_zone_info_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct ftl_dev_init_ctx *init_ctx = cb_arg; + struct spdk_ftl_dev *dev = init_ctx->dev; + struct ftl_band *band; + struct ftl_zone *zone; + struct ftl_addr addr; + size_t i, zones_left, num_zones; + + spdk_bdev_free_io(bdev_io); + + if (spdk_unlikely(!success)) { + SPDK_ERRLOG("Unable to read zone info for zone id: %"PRIu64"\n", init_ctx->zone_id); + ftl_dev_free_init_ctx(init_ctx); + ftl_init_fail(dev); + return; + } + + zones_left = ftl_get_num_zones(dev) - (init_ctx->zone_id / ftl_get_num_blocks_in_zone(dev)); + num_zones = spdk_min(zones_left, FTL_ZONE_INFO_COUNT); + + for (i = 0; i < num_zones; ++i) { + addr = ftl_addr_from_block_offset(dev, init_ctx->info[i].zone_id); + band = &dev->bands[addr.zone_id]; + zone = &band->zone_buf[addr.pu]; + zone->state = init_ctx->info[i].state; + zone->start_addr = addr; + addr = ftl_addr_from_block_offset(dev, init_ctx->info[i].write_pointer); + zone->write_offset = addr.offset; + + /* TODO: add support for zone capacity less than zone size */ + if (init_ctx->info[i].capacity != ftl_get_num_blocks_in_zone(dev)) { + zone->state = SPDK_BDEV_ZONE_STATE_OFFLINE; + SPDK_ERRLOG("Zone capacity is not equal zone size for " + "zone id: %"PRIu64"\n", init_ctx->zone_id); + } + + if (zone->state != SPDK_BDEV_ZONE_STATE_OFFLINE) { + band->num_zones++; + CIRCLEQ_INSERT_TAIL(&band->zones, zone, circleq); + } + } + + init_ctx->zone_id = init_ctx->zone_id + num_zones * ftl_get_num_blocks_in_zone(dev); + + ftl_dev_get_zone_info(init_ctx); +} + +static void +ftl_dev_get_zone_info(struct ftl_dev_init_ctx *init_ctx) +{ + struct spdk_ftl_dev *dev = init_ctx->dev; + size_t zones_left, num_zones; + int rc; + + zones_left = ftl_get_num_zones(dev) - (init_ctx->zone_id / ftl_get_num_blocks_in_zone(dev)); + if (zones_left == 0) { + ftl_dev_init_state(init_ctx); + return; + } + + num_zones = spdk_min(zones_left, FTL_ZONE_INFO_COUNT); + + rc = spdk_bdev_get_zone_info(dev->base_bdev_desc, init_ctx->ioch, + init_ctx->zone_id, num_zones, init_ctx->info, + ftl_dev_get_zone_info_cb, init_ctx); + + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("Unable to read zone info for zone id: %"PRIu64"\n", init_ctx->zone_id); + ftl_dev_free_init_ctx(init_ctx); + ftl_init_fail(dev); + } +} + +static int +ftl_dev_init_zones(struct ftl_dev_init_ctx *init_ctx) +{ + struct spdk_ftl_dev *dev = init_ctx->dev; + + init_ctx->zone_id = 0; + init_ctx->ioch = spdk_bdev_get_io_channel(dev->base_bdev_desc); + if (!init_ctx->ioch) { + SPDK_ERRLOG("Failed to get base bdev IO channel\n"); + return -1; + } + + ftl_dev_get_zone_info(init_ctx); + + return 0; +} + static int ftl_io_channel_create_cb(void *io_device, void *ctx) { @@ -979,11 +909,19 @@ ftl_io_channel_create_cb(void *io_device, void *ctx) return -1; } + ioch->base_ioch = spdk_bdev_get_io_channel(dev->base_bdev_desc); + if (!ioch->base_ioch) { + SPDK_ERRLOG("Failed to create base bdev IO channel\n"); + spdk_mempool_free(ioch->io_pool); + return -1; + } + if (ftl_dev_has_nv_cache(dev)) { ioch->cache_ioch = spdk_bdev_get_io_channel(dev->nv_cache.bdev_desc); if (!ioch->cache_ioch) { SPDK_ERRLOG("Failed to create cache IO channel\n"); spdk_mempool_free(ioch->io_pool); + spdk_put_io_channel(ioch->base_ioch); return -1; } } @@ -998,6 +936,8 @@ ftl_io_channel_destroy_cb(void *io_device, void *ctx) spdk_mempool_free(ioch->io_pool); + spdk_put_io_channel(ioch->base_ioch); + if (ioch->cache_ioch) { spdk_put_io_channel(ioch->cache_ioch); } @@ -1013,115 +953,39 @@ ftl_dev_init_io_channel(struct spdk_ftl_dev *dev) return 0; } -int -spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *_opts, spdk_ftl_init_fn cb_fn, void *cb_arg) +static int +ftl_dev_init_base_bdev(struct spdk_ftl_dev *dev) { - struct spdk_ftl_dev *dev; - struct spdk_ftl_dev_init_opts opts = *_opts; + uint32_t block_size; + uint64_t block_cnt; + struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); - dev = calloc(1, sizeof(*dev)); - if (!dev) { - return -ENOMEM; + if (!spdk_bdev_is_zoned(bdev)) { + SPDK_ERRLOG("Bdev dosen't support zone capabilities: %s\n", + spdk_bdev_get_name(bdev)); + return -1; } - if (!opts.conf) { - opts.conf = &g_default_conf; + dev->xfer_size = spdk_bdev_get_write_unit_size(bdev); + dev->md_size = spdk_bdev_get_md_size(bdev); + + block_size = spdk_bdev_get_block_size(bdev); + if (block_size != FTL_BLOCK_SIZE) { + SPDK_ERRLOG("Unsupported block size (%"PRIu32")\n", block_size); + return -1; } - TAILQ_INIT(&dev->retry_queue); - dev->conf = *opts.conf; - dev->init_ctx.cb_fn = cb_fn; - dev->init_ctx.cb_arg = cb_arg; - dev->init_ctx.thread = spdk_get_thread(); - dev->limit = SPDK_FTL_LIMIT_MAX; - - dev->name = strdup(opts.name); - if (!dev->name) { - SPDK_ERRLOG("Unable to set device name\n"); - goto fail_sync; + block_cnt = spdk_bdev_get_num_blocks(bdev); + if (block_cnt % ftl_get_num_punits(dev)) { + SPDK_ERRLOG("Unsupported geometry. Base bdev block count must be multiple " + "of optimal number of zones.\n"); + return -1; } - if (ftl_dev_nvme_init(dev, &opts)) { - SPDK_ERRLOG("Unable to initialize NVMe structures\n"); - goto fail_sync; - } - - /* In case of errors, we free all of the memory in ftl_dev_free_sync(), */ - /* so we don't have to clean up in each of the init functions. */ - if (ftl_dev_retrieve_geo(dev)) { - SPDK_ERRLOG("Unable to retrieve geometry\n"); - goto fail_sync; - } - - if (ftl_check_conf(opts.conf, &dev->geo)) { - SPDK_ERRLOG("Invalid device configuration\n"); - goto fail_sync; - } - - if (ftl_init_lba_map_pools(dev)) { - SPDK_ERRLOG("Unable to init LBA map pools\n"); - goto fail_sync; - } - - ftl_init_wptr_list(dev); - - if (ftl_dev_init_bands(dev)) { - SPDK_ERRLOG("Unable to initialize band array\n"); - goto fail_sync; - } - - if (ftl_dev_init_nv_cache(dev, opts.cache_bdev_desc)) { - SPDK_ERRLOG("Unable to initialize persistent cache\n"); - goto fail_sync; - } - - dev->rwb = ftl_rwb_init(&dev->conf, dev->geo.ws_opt, dev->md_size, ftl_get_num_punits(dev)); - if (!dev->rwb) { - SPDK_ERRLOG("Unable to initialize rwb structures\n"); - goto fail_sync; - } - - dev->reloc = ftl_reloc_init(dev); - if (!dev->reloc) { - SPDK_ERRLOG("Unable to initialize reloc structures\n"); - goto fail_sync; - } - - if (ftl_dev_init_io_channel(dev)) { - SPDK_ERRLOG("Unable to initialize IO channels\n"); - goto fail_sync; - } - - if (ftl_dev_init_threads(dev, &opts)) { - SPDK_ERRLOG("Unable to initialize device threads\n"); - goto fail_sync; - } - - if (opts.mode & SPDK_FTL_MODE_CREATE) { - if (ftl_setup_initial_state(dev)) { - SPDK_ERRLOG("Failed to setup initial state of the device\n"); - goto fail_async; - } - } else { - if (ftl_restore_state(dev, &opts)) { - SPDK_ERRLOG("Unable to restore device's state from the SSD\n"); - goto fail_async; - } - } + dev->num_bands = block_cnt / (ftl_get_num_punits(dev) * ftl_get_num_blocks_in_zone(dev)); + dev->addr_len = spdk_u64log2(block_cnt) + 1; return 0; -fail_sync: - ftl_dev_free_sync(dev); - return -ENOMEM; -fail_async: - ftl_init_fail(dev); - return 0; -} - -static void -_ftl_halt_defrag(void *arg) -{ - ftl_reloc_halt(((struct spdk_ftl_dev *)arg)->reloc); } static void @@ -1193,6 +1057,119 @@ ftl_dev_free_sync(struct spdk_ftl_dev *dev) free(dev); } +int +spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *_opts, spdk_ftl_init_fn cb_fn, void *cb_arg) +{ + struct spdk_ftl_dev *dev; + struct spdk_ftl_dev_init_opts opts = *_opts; + struct ftl_dev_init_ctx *init_ctx = NULL; + int rc = -ENOMEM; + + dev = calloc(1, sizeof(*dev)); + if (!dev) { + return -ENOMEM; + } + + init_ctx = calloc(1, sizeof(*init_ctx)); + if (!init_ctx) { + goto fail_sync; + } + + init_ctx->dev = dev; + init_ctx->opts = *_opts; + + if (!opts.conf) { + opts.conf = &g_default_conf; + } + + if (!opts.base_bdev_desc) { + SPDK_ERRLOG("Lack of underlying device in configuration\n"); + rc = -EINVAL; + goto fail_sync; + } + + TAILQ_INIT(&dev->retry_queue); + dev->conf = *opts.conf; + dev->init_ctx.cb_fn = cb_fn; + dev->init_ctx.cb_arg = cb_arg; + dev->init_ctx.thread = spdk_get_thread(); + dev->base_bdev_desc = opts.base_bdev_desc; + dev->limit = SPDK_FTL_LIMIT_MAX; + + dev->name = strdup(opts.name); + if (!dev->name) { + SPDK_ERRLOG("Unable to set device name\n"); + goto fail_sync; + } + + if (ftl_dev_init_base_bdev(dev)) { + SPDK_ERRLOG("Unsupported underlying device\n"); + goto fail_sync; + } + + /* In case of errors, we free all of the memory in ftl_dev_free_sync(), */ + /* so we don't have to clean up in each of the init functions. */ + if (ftl_check_conf(dev, opts.conf)) { + SPDK_ERRLOG("Invalid device configuration\n"); + goto fail_sync; + } + + if (ftl_init_lba_map_pools(dev)) { + SPDK_ERRLOG("Unable to init LBA map pools\n"); + goto fail_sync; + } + + ftl_init_wptr_list(dev); + + if (ftl_dev_init_bands(dev)) { + SPDK_ERRLOG("Unable to initialize band array\n"); + goto fail_sync; + } + + if (ftl_dev_init_nv_cache(dev, opts.cache_bdev_desc)) { + SPDK_ERRLOG("Unable to initialize persistent cache\n"); + goto fail_sync; + } + + dev->rwb = ftl_rwb_init(&dev->conf, dev->xfer_size, dev->md_size, ftl_get_num_punits(dev)); + if (!dev->rwb) { + SPDK_ERRLOG("Unable to initialize rwb structures\n"); + goto fail_sync; + } + + dev->reloc = ftl_reloc_init(dev); + if (!dev->reloc) { + SPDK_ERRLOG("Unable to initialize reloc structures\n"); + goto fail_sync; + } + + if (ftl_dev_init_io_channel(dev)) { + SPDK_ERRLOG("Unable to initialize IO channels\n"); + goto fail_sync; + } + + if (ftl_dev_init_zones(init_ctx)) { + SPDK_ERRLOG("Failed to initialize zones\n"); + goto fail_async; + } + + return 0; +fail_sync: + ftl_dev_free_sync(dev); + ftl_dev_free_init_ctx(init_ctx); + return rc; +fail_async: + ftl_init_fail(dev); + ftl_dev_free_init_ctx(init_ctx); + return 0; +} + +static void +_ftl_halt_defrag(void *arg) +{ + ftl_reloc_halt(((struct spdk_ftl_dev *)arg)->reloc); +} + static void ftl_call_fini_complete(struct spdk_ftl_dev *dev, int status) { diff --git a/lib/ftl/ftl_io.c b/lib/ftl/ftl_io.c index 90bef7113..125ea09a1 100644 --- a/lib/ftl/ftl_io.c +++ b/lib/ftl/ftl_io.c @@ -486,24 +486,6 @@ ftl_io_alloc_child(struct ftl_io *parent) return io; } -void -ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status) -{ - char addr_buf[128]; - - /* TODO: add error handling for specifc cases */ - if (status->status.sct == SPDK_NVME_SCT_MEDIA_ERROR && - status->status.sc == SPDK_OCSSD_SC_READ_HIGH_ECC) { - return; - } - - SPDK_ERRLOG("Status code type 0x%x, status code 0x%x for IO type %u @addr: %s, lba 0x%lx, cnt %lu\n", - status->status.sct, status->status.sc, io->type, ftl_addr2str(io->addr, addr_buf, sizeof(addr_buf)), - ftl_io_get_lba(io, 0), io->lbk_cnt); - - io->status = -EIO; -} - void ftl_io_fail(struct ftl_io *io, int status) { io->status = status; diff --git a/lib/ftl/ftl_io.h b/lib/ftl/ftl_io.h index e58e52ae8..ba217087b 100644 --- a/lib/ftl/ftl_io.h +++ b/lib/ftl/ftl_io.h @@ -130,6 +130,8 @@ struct ftl_io_channel { size_t elem_size; /* IO pool */ struct spdk_mempool *io_pool; + /* Underlying device IO channel */ + struct spdk_io_channel *base_ioch; /* Persistent cache IO channel */ struct spdk_io_channel *cache_ioch; }; diff --git a/lib/ftl/ftl_restore.c b/lib/ftl/ftl_restore.c index 23f6d433e..5cdd8a1ce 100644 --- a/lib/ftl/ftl_restore.c +++ b/lib/ftl/ftl_restore.c @@ -1164,7 +1164,7 @@ ftl_pad_zone_cb(struct ftl_io *io, void *arg, int status) goto end; } - if (io->addr.offset + io->lbk_cnt == band->dev->geo.clba) { + if (io->addr.offset + io->lbk_cnt == ftl_get_num_blocks_in_zone(restore->dev)) { zone = ftl_band_zone_from_addr(band, io->addr); zone->state = SPDK_BDEV_ZONE_STATE_CLOSED; } else { @@ -1188,7 +1188,6 @@ end: static void ftl_restore_pad_band(struct ftl_restore_band *rband) { - struct spdk_ocssd_chunk_information_entry info; struct ftl_restore *restore = rband->parent; struct ftl_band *band = rband->band; struct spdk_ftl_dev *dev = band->dev; @@ -1219,12 +1218,8 @@ ftl_restore_pad_band(struct ftl_restore_band *rband) continue; } - rc = ftl_retrieve_chunk_info(dev, band->zone_buf[i].start_addr, &info, 1); - if (spdk_unlikely(rc)) { - goto error; - } addr = band->zone_buf[i].start_addr; - addr.offset = info.wp; + addr.offset = band->zone_buf[i].write_offset; buffer = spdk_dma_zmalloc(FTL_BLOCK_SIZE * dev->xfer_size, 0, NULL); if (spdk_unlikely(!buffer)) { diff --git a/mk/spdk.lib_deps.mk b/mk/spdk.lib_deps.mk index 34e694d1a..8c4ff5482 100644 --- a/mk/spdk.lib_deps.mk +++ b/mk/spdk.lib_deps.mk @@ -72,7 +72,7 @@ DEPDIRS-bdev := log util conf thread $(JSON_LIBS) notify trace DEPDIRS-blobfs := log conf thread blob trace DEPDIRS-event := log util conf thread $(JSON_LIBS) trace -DEPDIRS-ftl := log util nvme thread trace bdev +DEPDIRS-ftl := log util thread trace bdev DEPDIRS-nbd := log util thread $(JSON_LIBS) bdev DEPDIRS-nvmf := log sock util nvme thread $(JSON_LIBS) trace bdev DEPDIRS-scsi := log util thread $(JSON_LIBS) trace bdev diff --git a/module/bdev/nvme/bdev_ftl.c b/module/bdev/nvme/bdev_ftl.c index 87a819ab9..606f105b4 100644 --- a/module/bdev/nvme/bdev_ftl.c +++ b/module/bdev/nvme/bdev_ftl.c @@ -43,6 +43,7 @@ #include "spdk/string.h" #include "spdk/ftl.h" #include "spdk_internal/log.h" +#include "spdk/nvme_ocssd.h" #include "bdev_ftl.h" #include "common.h" @@ -763,8 +764,6 @@ bdev_ftl_create(struct spdk_nvme_ctrlr *ctrlr, const struct ftl_bdev_init_opts * ftl_bdev->init_cb = cb; ftl_bdev->init_arg = cb_arg; - opts.ctrlr = ctrlr; - opts.trid = bdev_opts->trid; opts.mode = bdev_opts->mode; opts.uuid = bdev_opts->uuid; opts.name = ftl_bdev->bdev.name; diff --git a/test/unit/lib/ftl/common/utils.c b/test/unit/lib/ftl/common/utils.c index dbf5dc423..e079aaeb5 100644 --- a/test/unit/lib/ftl/common/utils.c +++ b/test/unit/lib/ftl/common/utils.c @@ -36,26 +36,50 @@ #include "spdk/ftl.h" #include "ftl/ftl_core.h" -struct spdk_ftl_dev *test_init_ftl_dev(const struct spdk_ocssd_geometry_data *geo); +struct base_bdev_geometry { + size_t write_unit_size; + size_t zone_size; + size_t optimal_open_zones; + size_t blockcnt; +}; + +extern struct base_bdev_geometry g_geo; + +struct spdk_ftl_dev *test_init_ftl_dev(const struct base_bdev_geometry *geo); struct ftl_band *test_init_ftl_band(struct spdk_ftl_dev *dev, size_t id); void test_free_ftl_dev(struct spdk_ftl_dev *dev); void test_free_ftl_band(struct ftl_band *band); uint64_t test_offset_from_addr(struct ftl_addr addr, struct ftl_band *band); +DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *desc), NULL); + +uint64_t +spdk_bdev_get_zone_size(const struct spdk_bdev *bdev) +{ + return g_geo.zone_size; +} + +uint32_t +spdk_bdev_get_optimal_open_zones(const struct spdk_bdev *bdev) +{ + return g_geo.optimal_open_zones; +} + struct spdk_ftl_dev * -test_init_ftl_dev(const struct spdk_ocssd_geometry_data *geo) +test_init_ftl_dev(const struct base_bdev_geometry *geo) { struct spdk_ftl_dev *dev; dev = calloc(1, sizeof(*dev)); SPDK_CU_ASSERT_FATAL(dev != NULL); - dev->xfer_size = geo->ws_opt; - dev->geo = *geo; + dev->xfer_size = geo->write_unit_size; dev->core_thread.thread = spdk_thread_create("unit_test_thread", NULL); spdk_set_thread(dev->core_thread.thread); - - dev->bands = calloc(geo->num_chk, sizeof(*dev->bands)); + dev->core_thread.ioch = calloc(1, sizeof(*dev->core_thread.ioch) + + sizeof(struct ftl_io_channel)); + dev->num_bands = geo->blockcnt / (geo->zone_size * geo->optimal_open_zones); + dev->bands = calloc(dev->num_bands, sizeof(*dev->bands)); SPDK_CU_ASSERT_FATAL(dev->bands != NULL); dev->lba_pool = spdk_mempool_create("ftl_ut", 2, 0x18000, @@ -76,7 +100,7 @@ test_init_ftl_band(struct spdk_ftl_dev *dev, size_t id) struct ftl_zone *zone; SPDK_CU_ASSERT_FATAL(dev != NULL); - SPDK_CU_ASSERT_FATAL(id < dev->geo.num_chk); + SPDK_CU_ASSERT_FATAL(id < dev->num_bands); band = &dev->bands[id]; band->dev = dev; @@ -112,6 +136,7 @@ void test_free_ftl_dev(struct spdk_ftl_dev *dev) { SPDK_CU_ASSERT_FATAL(dev != NULL); + free(dev->core_thread.ioch); spdk_set_thread(dev->core_thread.thread); spdk_thread_exit(dev->core_thread.thread); spdk_thread_destroy(dev->core_thread.thread); diff --git a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c b/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c index 64707b94d..96ed9e5c8 100644 --- a/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c +++ b/test/unit/lib/ftl/ftl_band.c/ftl_band_ut.c @@ -43,16 +43,14 @@ #define TEST_BAND_IDX 68 #define TEST_LBA 0x68676564 -static struct spdk_ocssd_geometry_data g_geo = { - .num_grp = 3, - .num_pu = 3, - .num_chk = 1500, - .clba = 100, - .ws_opt = 16, - .ws_min = 4, +struct base_bdev_geometry g_geo = { + .write_unit_size = 16, + .optimal_open_zones = 9, + .zone_size = 100, + .blockcnt = 1500 * 100 * 8, }; -static struct spdk_ftl_dev *g_dev; +static struct spdk_ftl_dev *g_dev; static struct ftl_band *g_band; static void @@ -108,7 +106,7 @@ test_band_lbkoff_from_addr_offset(void) setup_band(); for (i = 0; i < ftl_get_num_punits(g_dev); ++i) { - for (j = 0; j < g_geo.clba; ++j) { + for (j = 0; j < g_geo.zone_size; ++j) { addr = addr_from_punit(i); addr.zone_id = TEST_BAND_IDX; addr.offset = j; @@ -130,7 +128,7 @@ test_band_addr_from_lbkoff(void) setup_band(); for (i = 0; i < ftl_get_num_punits(g_dev); ++i) { - for (j = 0; j < g_geo.clba; ++j) { + for (j = 0; j < g_geo.zone_size; ++j) { expect = addr_from_punit(i); expect.zone_id = TEST_BAND_IDX; expect.offset = j; diff --git a/test/unit/lib/ftl/ftl_md/ftl_md_ut.c b/test/unit/lib/ftl/ftl_md/ftl_md_ut.c index 358907a33..668b1481c 100644 --- a/test/unit/lib/ftl/ftl_md/ftl_md_ut.c +++ b/test/unit/lib/ftl/ftl_md/ftl_md_ut.c @@ -39,22 +39,20 @@ #include "ftl/ftl_band.c" #include "../common/utils.c" -static struct spdk_ocssd_geometry_data g_geo = { - .num_grp = 4, - .num_pu = 3, - .num_chk = 1500, - .clba = 100, - .ws_opt = 16, - .ws_min = 4, +struct base_bdev_geometry g_geo = { + .write_unit_size = 16, + .optimal_open_zones = 12, + .zone_size = 100, + .blockcnt = 1500 * 100 * 12, }; static void -setup_band(struct ftl_band **band, const struct spdk_ocssd_geometry_data *geo) +setup_band(struct ftl_band **band, const struct base_bdev_geometry *geo) { int rc; struct spdk_ftl_dev *dev; - dev = test_init_ftl_dev(geo); + dev = test_init_ftl_dev(&g_geo); *band = test_init_ftl_band(dev, 0); rc = ftl_band_alloc_lba_map(*band); SPDK_CU_ASSERT_FATAL(rc == 0); @@ -122,7 +120,7 @@ test_md_unpack_fail(void) /* check invalid size */ ftl_pack_tail_md(band); - band->dev->geo.clba--; + g_geo.zone_size--; CU_ASSERT_EQUAL(ftl_unpack_tail_md(band), FTL_MD_INVALID_SIZE); cleanup_band(band); diff --git a/test/unit/lib/ftl/ftl_ppa/ftl_ppa_ut.c b/test/unit/lib/ftl/ftl_ppa/ftl_ppa_ut.c index 83d8e2e5f..95d9632c0 100644 --- a/test/unit/lib/ftl/ftl_ppa/ftl_ppa_ut.c +++ b/test/unit/lib/ftl/ftl_ppa/ftl_ppa_ut.c @@ -42,6 +42,24 @@ static struct spdk_ftl_dev *g_dev; +DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *desc), NULL); + +uint64_t +spdk_bdev_get_zone_size(const struct spdk_bdev *bdev) +{ + if (g_dev->addr_len > 32) { + return 1ULL << 32; + } + + return 1024; +} + +uint32_t +spdk_bdev_get_optimal_open_zones(const struct spdk_bdev *bdev) +{ + return 100; +} + static struct spdk_ftl_dev * test_alloc_dev(size_t size) { @@ -51,11 +69,26 @@ test_alloc_dev(size_t size) dev->num_lbas = L2P_TABLE_SIZE; dev->l2p = calloc(L2P_TABLE_SIZE, size); - dev->geo.num_grp = 1; return dev; } +static int +setup_l2p_32bit(void) +{ + g_dev = test_alloc_dev(sizeof(uint32_t)); + g_dev->addr_len = 24; + return 0; +} + +static int +setup_l2p_64bit(void) +{ + g_dev = test_alloc_dev(sizeof(uint64_t)); + g_dev->addr_len = 63; + return 0; +} + static void clean_l2p(void) { @@ -69,40 +102,6 @@ clean_l2p(void) memset(g_dev->l2p, 0, g_dev->num_lbas * l2p_elem_size); } -static int -setup_l2p_32bit(void) -{ - g_dev = test_alloc_dev(sizeof(uint32_t)); - g_dev->ppaf.lbk_offset = 0; - g_dev->ppaf.lbk_mask = (1 << 8) - 1; - g_dev->ppaf.chk_offset = 8; - g_dev->ppaf.chk_mask = (1 << 4) - 1; - g_dev->ppaf.pu_offset = g_dev->ppaf.chk_offset + 4; - g_dev->ppaf.pu_mask = (1 << 3) - 1; - g_dev->ppaf.grp_offset = g_dev->ppaf.pu_offset + 3; - g_dev->ppaf.grp_mask = (1 << 2) - 1; - g_dev->addr_len = g_dev->ppaf.grp_offset + 2; - - return 0; -} - -static int -setup_l2p_64bit(void) -{ - g_dev = test_alloc_dev(sizeof(uint64_t)); - g_dev->ppaf.lbk_offset = 0; - g_dev->ppaf.lbk_mask = (1UL << 31) - 1; - g_dev->ppaf.chk_offset = 31; - g_dev->ppaf.chk_mask = (1 << 4) - 1; - g_dev->ppaf.pu_offset = g_dev->ppaf.chk_offset + 4; - g_dev->ppaf.pu_mask = (1 << 3) - 1; - g_dev->ppaf.grp_offset = g_dev->ppaf.pu_offset + 3; - g_dev->ppaf.grp_mask = (1 << 2) - 1; - g_dev->addr_len = g_dev->ppaf.grp_offset + 2; - - return 0; -} - static int cleanup(void) { @@ -158,8 +157,8 @@ test_addr_pack64(void) orig.pu = 2; /* Check valid address transformation */ - addr.addr = ftl_addr_addr_pack(g_dev, orig); - addr = ftl_addr_addr_unpack(g_dev, addr.addr); + addr.addr = ftl_block_offset_from_addr(g_dev, orig); + addr = ftl_addr_from_block_offset(g_dev, addr.addr); CU_ASSERT_FALSE(ftl_addr_invalid(addr)); CU_ASSERT_EQUAL(addr.addr, orig.addr); @@ -167,46 +166,23 @@ test_addr_pack64(void) orig.zone_id = 0x6; orig.pu = 0x4; - addr.addr = ftl_addr_addr_pack(g_dev, orig); - addr = ftl_addr_addr_unpack(g_dev, addr.addr); + addr.addr = ftl_block_offset_from_addr(g_dev, orig); + addr = ftl_addr_from_block_offset(g_dev, addr.addr); CU_ASSERT_FALSE(ftl_addr_invalid(addr)); CU_ASSERT_EQUAL(addr.addr, orig.addr); - /* Check maximum valid address for addrf */ - orig.offset = 0x7fffffff; + /* Check maximum valid address */ + orig.offset = 0xffffffff; orig.zone_id = 0xf; orig.pu = 0x7; - addr.addr = ftl_addr_addr_pack(g_dev, orig); - addr = ftl_addr_addr_unpack(g_dev, addr.addr); + addr.addr = ftl_block_offset_from_addr(g_dev, orig); + addr = ftl_addr_from_block_offset(g_dev, addr.addr); CU_ASSERT_FALSE(ftl_addr_invalid(addr)); CU_ASSERT_EQUAL(addr.addr, orig.addr); clean_l2p(); } -static void -test_addr_trans(void) -{ - struct ftl_addr addr = {}, orig = {}; - size_t i; - - for (i = 0; i < L2P_TABLE_SIZE; ++i) { - addr.offset = i % (g_dev->ppaf.lbk_mask + 1); - addr.zone_id = i % (g_dev->ppaf.chk_mask + 1); - addr.pu = i % (g_dev->ppaf.pu_mask + 1); - ftl_l2p_set(g_dev, i, addr); - } - - for (i = 0; i < L2P_TABLE_SIZE; ++i) { - orig.offset = i % (g_dev->ppaf.lbk_mask + 1); - orig.zone_id = i % (g_dev->ppaf.chk_mask + 1); - orig.pu = i % (g_dev->ppaf.pu_mask + 1); - addr = ftl_l2p_get(g_dev, i); - CU_ASSERT_EQUAL(addr.addr, orig.addr); - } - clean_l2p(); -} - static void test_addr_invalid(void) { @@ -285,14 +261,10 @@ main(int argc, char **argv) test_addr_pack32) == NULL || CU_add_test(suite32, "test_addr32_invalid", test_addr_invalid) == NULL - || CU_add_test(suite32, "test_addr32_trans", - test_addr_trans) == NULL || CU_add_test(suite32, "test_addr32_cached", test_addr_cached) == NULL || CU_add_test(suite64, "test_addr64_invalid", test_addr_invalid) == NULL - || CU_add_test(suite64, "test_addr64_trans", - test_addr_trans) == NULL || CU_add_test(suite64, "test_addr64_cached", test_addr_cached) == NULL || CU_add_test(suite64, "test_addr64_pack", diff --git a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c b/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c index 88674f845..1c3e0aa6e 100644 --- a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c +++ b/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c @@ -42,13 +42,11 @@ #define MAX_ACTIVE_RELOCS 5 #define MAX_RELOC_QDEPTH 31 -static struct spdk_ocssd_geometry_data g_geo = { - .num_grp = 4, - .num_pu = 3, - .num_chk = 500, - .clba = 100, - .ws_opt = 16, - .ws_min = 4, +struct base_bdev_geometry g_geo = { + .write_unit_size = 16, + .optimal_open_zones = 12, + .zone_size = 100, + .blockcnt = 1500 * 100 * 12, }; DEFINE_STUB(ftl_dev_tail_md_disk_size, size_t, (const struct spdk_ftl_dev *dev), 1); @@ -199,13 +197,14 @@ add_to_active_queue(struct ftl_reloc *reloc, struct ftl_band_reloc *breloc) static void setup_reloc(struct spdk_ftl_dev **_dev, struct ftl_reloc **_reloc, - const struct spdk_ocssd_geometry_data *geo) + const struct base_bdev_geometry *geo) { size_t i; struct spdk_ftl_dev *dev; struct ftl_reloc *reloc; dev = test_init_ftl_dev(geo); + dev->conf.max_active_relocs = MAX_ACTIVE_RELOCS; dev->conf.max_reloc_qdepth = MAX_RELOC_QDEPTH; @@ -266,7 +265,7 @@ test_reloc_iter_full(void) setup_reloc(&dev, &reloc, &g_geo); - dev->geo.clba = 100; + g_geo.zone_size = 100; breloc = &reloc->brelocs[0]; band = breloc->band; diff --git a/test/unit/lib/ftl/ftl_wptr/ftl_wptr_ut.c b/test/unit/lib/ftl/ftl_wptr/ftl_wptr_ut.c index 064407d23..a7e923d4b 100644 --- a/test/unit/lib/ftl/ftl_wptr/ftl_wptr_ut.c +++ b/test/unit/lib/ftl/ftl_wptr/ftl_wptr_ut.c @@ -40,18 +40,17 @@ #include "ftl/ftl_band.c" #include "../common/utils.c" -static struct spdk_ocssd_geometry_data g_geo = { - .num_grp = 4, - .num_pu = 3, - .num_chk = 20, - .clba = 128, - .ws_opt = 16, - .ws_min = 4, +struct base_bdev_geometry g_geo = { + .write_unit_size = 16, + .optimal_open_zones = 12, + .zone_size = 128, + .blockcnt = 20 * 128 * 12, }; #if defined(DEBUG) DEFINE_STUB(ftl_band_validate_md, bool, (struct ftl_band *band), true); #endif +DEFINE_STUB_V(spdk_bdev_free_io, (struct spdk_bdev_io *bdev_io)); DEFINE_STUB_V(ftl_io_dec_req, (struct ftl_io *io)); DEFINE_STUB_V(ftl_io_inc_req, (struct ftl_io *io)); DEFINE_STUB_V(ftl_io_fail, (struct ftl_io *io, int status)); @@ -67,12 +66,11 @@ DEFINE_STUB_V(ftl_io_process_error, (struct ftl_io *io, const struct spdk_nvme_c DEFINE_STUB_V(ftl_trace_limits, (struct spdk_ftl_dev *dev, const size_t *limits, size_t num_free)); DEFINE_STUB(ftl_rwb_entry_cnt, size_t, (const struct ftl_rwb *rwb), 0); DEFINE_STUB_V(ftl_rwb_set_limits, (struct ftl_rwb *rwb, const size_t limit[FTL_RWB_TYPE_MAX])); -DEFINE_STUB(spdk_nvme_ocssd_ns_cmd_vector_reset, int, (struct spdk_nvme_ns *ns, - struct spdk_nvme_qpair *qpair, uint64_t *lba_list, uint32_t num_lbas, - struct spdk_ocssd_chunk_information_entry *chunk_info, - spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0); -DEFINE_STUB(spdk_bdev_desc_get_bdev, struct spdk_bdev *, (struct spdk_bdev_desc *dsc), NULL); DEFINE_STUB(spdk_bdev_get_num_blocks, uint64_t, (const struct spdk_bdev *bdev), 0); +DEFINE_STUB(spdk_bdev_zone_management, int, (struct spdk_bdev_desc *desc, + struct spdk_io_channel *ch, + uint64_t zone_id, enum spdk_bdev_zone_action action, + spdk_bdev_io_completion_cb cb, void *cb_arg), 0); struct ftl_io * ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, ftl_io_fn cb) @@ -104,13 +102,12 @@ ftl_io_complete(struct ftl_io *io) } static void -setup_wptr_test(struct spdk_ftl_dev **dev, const struct spdk_ocssd_geometry_data *geo) +setup_wptr_test(struct spdk_ftl_dev **dev, const struct base_bdev_geometry *geo) { size_t i; struct spdk_ftl_dev *t_dev; t_dev = test_init_ftl_dev(geo); - for (i = 0; i < ftl_get_num_bands(t_dev); ++i) { test_init_ftl_band(t_dev, i); t_dev->bands[i].state = FTL_BAND_STATE_CLOSED;