From d9a631ad4ca3322a5cac504cc58469103e161293 Mon Sep 17 00:00:00 2001 From: Artur Paszkiewicz Date: Tue, 21 Jun 2022 13:03:31 +0200 Subject: [PATCH] FTL: Add io channel logic Signed-off-by: Kozlowski Mateusz Signed-off-by: Artur Paszkiewicz Change-Id: Ibf6bfbabc03c43e7938531c4fe08fde01ce02a3f Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13307 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- lib/ftl/Makefile | 2 +- lib/ftl/ftl_core.c | 72 +++++++++++ lib/ftl/ftl_core.h | 19 +++ lib/ftl/ftl_init.c | 1 + lib/ftl/ftl_io.c | 14 ++- lib/ftl/ftl_io.h | 15 +++ lib/ftl/mngt/ftl_mngt_ioch.c | 202 +++++++++++++++++++++++++++++++ lib/ftl/mngt/ftl_mngt_shutdown.c | 8 ++ lib/ftl/mngt/ftl_mngt_startup.c | 10 ++ lib/ftl/mngt/ftl_mngt_steps.h | 8 ++ 10 files changed, 347 insertions(+), 4 deletions(-) create mode 100644 lib/ftl/mngt/ftl_mngt_ioch.c diff --git a/lib/ftl/Makefile b/lib/ftl/Makefile index 9df1202fd..1110d89b8 100644 --- a/lib/ftl/Makefile +++ b/lib/ftl/Makefile @@ -19,7 +19,7 @@ FTL_SUBDIRS := mngt utils C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c -C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c +C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ftl.map) diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index a77f692d5..89cd29d34 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -46,6 +46,76 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at attrs->optimum_io_size = dev->xfer_size; } +static void +start_io(struct ftl_io *io) +{ + struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch); + + io->map = ftl_mempool_get(ioch->map_pool); + if (spdk_unlikely(!io->map)) { + io->status = -ENOMEM; + ftl_io_complete(io); + return; + } + + switch (io->type) { + case FTL_IO_READ: + case FTL_IO_WRITE: + case FTL_IO_UNMAP: + default: + io->status = -EOPNOTSUPP; + ftl_io_complete(io); + } +} + +#define FTL_IO_QUEUE_BATCH 16 +int +ftl_io_channel_poll(void *arg) +{ + struct ftl_io_channel *ch = arg; + void *ios[FTL_IO_QUEUE_BATCH]; + uint64_t i, count; + + count = spdk_ring_dequeue(ch->cq, ios, FTL_IO_QUEUE_BATCH); + if (count == 0) { + return SPDK_POLLER_IDLE; + } + + for (i = 0; i < count; i++) { + struct ftl_io *io = ios[i]; + io->user_fn(io->cb_ctx, io->status); + } + + return SPDK_POLLER_BUSY; +} + +static void +ftl_process_io_channel(struct spdk_ftl_dev *dev, struct ftl_io_channel *ioch) +{ + void *ios[FTL_IO_QUEUE_BATCH]; + size_t count, i; + + count = spdk_ring_dequeue(ioch->sq, ios, FTL_IO_QUEUE_BATCH); + if (count == 0) { + return; + } + + for (i = 0; i < count; i++) { + struct ftl_io *io = ios[i]; + start_io(io); + } +} + +static void +ftl_process_io_queue(struct spdk_ftl_dev *dev) +{ + struct ftl_io_channel *ioch; + + TAILQ_FOREACH(ioch, &dev->ioch_queue, entry) { + ftl_process_io_channel(dev, ioch); + } +} + int ftl_core_poller(void *ctx) { @@ -57,6 +127,8 @@ ftl_core_poller(void *ctx) return SPDK_POLLER_IDLE; } + ftl_process_io_queue(dev); + if (io_activity_total_old != dev->io_activity_total) { return SPDK_POLLER_BUSY; } diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index bf1b1e476..4374f27e8 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -28,6 +28,9 @@ struct spdk_ftl_dev { /* FTL device layout */ struct ftl_layout layout; + /* Queue of registered IO channels */ + TAILQ_HEAD(, ftl_io_channel) ioch_queue; + /* Underlying device */ struct spdk_bdev_desc *base_bdev_desc; @@ -49,6 +52,12 @@ struct spdk_ftl_dev { /* Indicates the device is about to be stopped */ bool halt; + /* Indicates if the device is registered as an IO device */ + bool io_device_registered; + + /* Management process to be continued after IO device unregistration completes */ + struct ftl_mngt_process *unregister_process; + /* counters for poller busy, include 1. nv cache read/write 2. metadata read/write @@ -99,6 +108,10 @@ struct spdk_ftl_dev { int ftl_core_poller(void *ctx); +int ftl_io_channel_poll(void *arg); + +struct ftl_io_channel *ftl_io_channel_get_ctx(struct spdk_io_channel *ioch); + static inline uint64_t ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev) { @@ -128,6 +141,12 @@ ftl_get_write_unit_size(struct spdk_bdev *bdev) return 32; } +static inline struct spdk_thread * +ftl_get_core_thread(const struct spdk_ftl_dev *dev) +{ + return dev->core_thread; +} + static inline size_t ftl_get_num_bands(const struct spdk_ftl_dev *dev) { diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index 74755acc4..f87612080 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -115,6 +115,7 @@ allocate_dev(const struct spdk_ftl_conf *conf, int *error) TAILQ_INIT(&dev->rd_sq); TAILQ_INIT(&dev->wr_sq); + TAILQ_INIT(&dev->ioch_queue); return dev; error: diff --git a/lib/ftl/ftl_io.c b/lib/ftl/ftl_io.c index 9290968ed..b7e0fa0a4 100644 --- a/lib/ftl/ftl_io.c +++ b/lib/ftl/ftl_io.c @@ -117,6 +117,9 @@ ftl_io_iovec_len_left(struct ftl_io *io) static void ftl_io_cb(struct ftl_io *io, void *arg, int status) { + struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch); + size_t result; + if (spdk_unlikely(status)) { io->status = status; @@ -145,15 +148,20 @@ ftl_io_cb(struct ftl_io *io, void *arg, int status) } } - /* User completion added in next patch */ + if (io->map) { + ftl_mempool_put(ioch->map_pool, io->map); + } + + result = spdk_ring_enqueue(ioch->cq, (void **)&io, 1, NULL); + assert(result != 0); } int ftl_io_init(struct spdk_io_channel *_ioch, struct ftl_io *io, uint64_t lba, size_t num_blocks, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type) { - /* dev initialized from io channel context in next patch */ - struct spdk_ftl_dev *dev = NULL; + struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(_ioch); + struct spdk_ftl_dev *dev = ioch->dev; memset(io, 0, sizeof(struct ftl_io)); io->ioch = _ioch; diff --git a/lib/ftl/ftl_io.h b/lib/ftl/ftl_io.h index 6aa3dc92e..0af449914 100644 --- a/lib/ftl/ftl_io.h +++ b/lib/ftl/ftl_io.h @@ -34,6 +34,21 @@ enum ftl_io_type { #define FTL_IO_MAX_IOVEC 4 +struct ftl_io_channel { + /* Device */ + struct spdk_ftl_dev *dev; + /* Entry of IO channels queue/list */ + TAILQ_ENTRY(ftl_io_channel) entry; + /* IO map pool */ + struct ftl_mempool *map_pool; + /* Poller used for completing user requests and retrying IO */ + struct spdk_poller *poller; + /* Submission queue */ + struct spdk_ring *sq; + /* Completion queue */ + struct spdk_ring *cq; +}; + /* General IO descriptor for user requests */ struct ftl_io { /* Device */ diff --git a/lib/ftl/mngt/ftl_mngt_ioch.c b/lib/ftl/mngt/ftl_mngt_ioch.c new file mode 100644 index 000000000..013c4312e --- /dev/null +++ b/lib/ftl/mngt/ftl_mngt_ioch.c @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) Intel Corporation. + * All rights reserved. + */ + +#include "spdk/thread.h" + +#include "ftl_core.h" +#include "ftl_mngt.h" +#include "ftl_mngt_steps.h" +#include "utils/ftl_mempool.h" + +struct ftl_io_channel_ctx { + struct ftl_io_channel *ioch; +}; + +struct ftl_io_channel * +ftl_io_channel_get_ctx(struct spdk_io_channel *ioch) +{ + struct ftl_io_channel_ctx *ctx = spdk_io_channel_get_ctx(ioch); + return ctx->ioch; +} + +static void +ftl_dev_register_channel(void *ctx) +{ + struct ftl_io_channel *ioch = ctx; + struct spdk_ftl_dev *dev = ioch->dev; + + /* This only runs on the core thread, so it's safe to do this lockless */ + TAILQ_INSERT_TAIL(&dev->ioch_queue, ioch, entry); +} + +static void +io_channel_unregister(void *ctx) +{ + struct ftl_io_channel *ioch = ctx; + struct spdk_ftl_dev *dev = ioch->dev; + + TAILQ_REMOVE(&dev->ioch_queue, ioch, entry); + + spdk_ring_free(ioch->cq); + spdk_ring_free(ioch->sq); + ftl_mempool_destroy(ioch->map_pool); + free(ioch); +} + +static int +io_channel_create_cb(void *io_device, void *ctx) +{ + struct spdk_ftl_dev *dev = io_device; + struct ftl_io_channel_ctx *_ioch = ctx; + struct ftl_io_channel *ioch; + char mempool_name[32]; + int rc; + + FTL_NOTICELOG(dev, "FTL IO channel created on %s\n", + spdk_thread_get_name(spdk_get_thread())); + + /* This gets unregistered asynchronously with the device - + * we can't just use the ctx buffer passed by the thread library + */ + ioch = calloc(1, sizeof(*ioch)); + if (ioch == NULL) { + FTL_ERRLOG(dev, "Failed to allocate IO channel\n"); + return -1; + } + + rc = snprintf(mempool_name, sizeof(mempool_name), "ftl_io_%p", ioch); + if (rc < 0 || rc >= (int)sizeof(mempool_name)) { + FTL_ERRLOG(dev, "Failed to create IO channel pool name\n"); + free(ioch); + return -1; + } + + ioch->dev = dev; + + ioch->map_pool = ftl_mempool_create( + dev->conf.user_io_pool_size, + sizeof(ftl_addr) * dev->xfer_size, + 64, + SPDK_ENV_SOCKET_ID_ANY); + if (!ioch->map_pool) { + FTL_ERRLOG(dev, "Failed to create IO channel's map IO pool\n"); + goto fail_io_pool; + } + + ioch->cq = spdk_ring_create(SPDK_RING_TYPE_SP_SC, spdk_align64pow2(dev->conf.user_io_pool_size + 1), + SPDK_ENV_SOCKET_ID_ANY); + if (!ioch->cq) { + FTL_ERRLOG(dev, "Failed to create IO channel completion queue\n"); + goto fail_io_pool; + } + + ioch->sq = spdk_ring_create(SPDK_RING_TYPE_SP_SC, spdk_align64pow2(dev->conf.user_io_pool_size + 1), + SPDK_ENV_SOCKET_ID_ANY); + if (!ioch->sq) { + FTL_ERRLOG(dev, "Failed to create IO channel submission queue\n"); + goto fail_cq; + } + + ioch->poller = SPDK_POLLER_REGISTER(ftl_io_channel_poll, ioch, 0); + if (!ioch->poller) { + FTL_ERRLOG(dev, "Failed to register IO channel poller\n"); + goto fail_sq; + } + + if (spdk_thread_send_msg(dev->core_thread, ftl_dev_register_channel, ioch)) { + FTL_ERRLOG(dev, "Failed to register IO channel\n"); + goto fail_poller; + } + + _ioch->ioch = ioch; + return 0; + +fail_poller: + spdk_poller_unregister(&ioch->poller); +fail_cq: + spdk_ring_free(ioch->cq); +fail_sq: + spdk_ring_free(ioch->sq); +fail_io_pool: + ftl_mempool_destroy(ioch->map_pool); + free(ioch); + + return -1; +} + +static void +io_channel_destroy_cb(void *io_device, void *ctx) +{ + struct ftl_io_channel_ctx *_ioch = ctx; + struct ftl_io_channel *ioch = _ioch->ioch; + struct spdk_ftl_dev *dev = ioch->dev; + + FTL_NOTICELOG(dev, "FTL IO channel destroy on %s\n", + spdk_thread_get_name(spdk_get_thread())); + + spdk_poller_unregister(&ioch->poller); + spdk_thread_send_msg(ftl_get_core_thread(dev), + io_channel_unregister, ioch); +} + +void +ftl_mngt_register_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + dev->io_device_registered = true; + + spdk_io_device_register(dev, io_channel_create_cb, + io_channel_destroy_cb, + sizeof(struct ftl_io_channel_ctx), + NULL); + + ftl_mngt_next_step(mngt); +} + +static void +unregister_cb(void *io_device) +{ + struct spdk_ftl_dev *dev = io_device; + struct ftl_mngt_process *mngt = dev->unregister_process; + + dev->io_device_registered = false; + dev->unregister_process = NULL; + + ftl_mngt_next_step(mngt); +} + +void +ftl_mngt_unregister_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + if (dev->io_device_registered) { + dev->unregister_process = mngt; + spdk_io_device_unregister(dev, unregister_cb); + } else { + ftl_mngt_skip_step(mngt); + } +} + +void +ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + dev->ioch = spdk_get_io_channel(dev); + if (!dev->ioch) { + FTL_ERRLOG(dev, "Unable to get IO channel for core thread"); + ftl_mngt_fail_step(mngt); + return; + } + + ftl_mngt_next_step(mngt); +} + +void +ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) +{ + if (dev->ioch) { + spdk_put_io_channel(dev->ioch); + dev->ioch = NULL; + } + + ftl_mngt_next_step(mngt); +} diff --git a/lib/ftl/mngt/ftl_mngt_shutdown.c b/lib/ftl/mngt/ftl_mngt_shutdown.c index 5f1e3bdf5..0e05e63e2 100644 --- a/lib/ftl/mngt/ftl_mngt_shutdown.c +++ b/lib/ftl/mngt/ftl_mngt_shutdown.c @@ -15,6 +15,14 @@ static const struct ftl_mngt_process_desc desc_shutdown = { .name = "FTL shutdown", .error_handler = ftl_mngt_rollback_device, .steps = { + { + .name = "Deinit core IO channel", + .action = ftl_mngt_deinit_io_channel + }, + { + .name = "Unregister IO device", + .action = ftl_mngt_unregister_io_device + }, { .name = "Stop core poller", .action = ftl_mngt_stop_core_poller diff --git a/lib/ftl/mngt/ftl_mngt_startup.c b/lib/ftl/mngt/ftl_mngt_startup.c index 9e483a1fe..640d57490 100644 --- a/lib/ftl/mngt/ftl_mngt_startup.c +++ b/lib/ftl/mngt/ftl_mngt_startup.c @@ -42,6 +42,16 @@ static const struct ftl_mngt_process_desc desc_startup = { .action = ftl_mngt_open_cache_bdev, .cleanup = ftl_mngt_close_cache_bdev }, + { + .name = "Register IO device", + .action = ftl_mngt_register_io_device, + .cleanup = ftl_mngt_unregister_io_device + }, + { + .name = "Initialize core IO channel", + .action = ftl_mngt_init_io_channel, + .cleanup = ftl_mngt_deinit_io_channel + }, { .name = "Initialize layout", .action = ftl_mngt_init_layout diff --git a/lib/ftl/mngt/ftl_mngt_steps.h b/lib/ftl/mngt/ftl_mngt_steps.h index e41879ed5..60c63b406 100644 --- a/lib/ftl/mngt/ftl_mngt_steps.h +++ b/lib/ftl/mngt/ftl_mngt_steps.h @@ -18,6 +18,14 @@ void ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process void ftl_mngt_close_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); +void ftl_mngt_register_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + +void ftl_mngt_unregister_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + +void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + +void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); + void ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); void ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);