ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FTL: Add io channel logic

Browse Source 
Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: Ibf6bfbabc03c43e7938531c4fe08fde01ce02a3f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13307
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Artur Paszkiewicz 2022-06-21 13:03:31 +02:00 committed by Jim Harris
parent e7a03e68e1
commit d9a631ad4c
10 changed files with 347 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/ftl/Makefile
View File

@ -19,7 +19,7 @@ FTL_SUBDIRS := mngt utils
C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c
C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c
C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c
C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c
C_SRCS += utils/ftl_conf.c utils/ftl_md.c utils/ftl_mempool.c
SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_ftl.map)

72
lib/ftl/ftl_core.c
View File

@ -46,6 +46,76 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at
attrs->optimum_io_size = dev->xfer_size;
}
static void
start_io(struct ftl_io *io)
{
struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch);
io->map = ftl_mempool_get(ioch->map_pool);
if (spdk_unlikely(!io->map)) {
io->status = -ENOMEM;
ftl_io_complete(io);
return;
}
switch (io->type) {
case FTL_IO_READ:
case FTL_IO_WRITE:
case FTL_IO_UNMAP:
default:
io->status = -EOPNOTSUPP;
ftl_io_complete(io);
}
}
#define FTL_IO_QUEUE_BATCH 16
int
ftl_io_channel_poll(void *arg)
{
struct ftl_io_channel *ch = arg;
void *ios[FTL_IO_QUEUE_BATCH];
uint64_t i, count;
count = spdk_ring_dequeue(ch->cq, ios, FTL_IO_QUEUE_BATCH);
if (count == 0) {
return SPDK_POLLER_IDLE;
}
for (i = 0; i < count; i++) {
struct ftl_io *io = ios[i];
io->user_fn(io->cb_ctx, io->status);
}
return SPDK_POLLER_BUSY;
}
static void
ftl_process_io_channel(struct spdk_ftl_dev *dev, struct ftl_io_channel *ioch)
{
void *ios[FTL_IO_QUEUE_BATCH];
size_t count, i;
count = spdk_ring_dequeue(ioch->sq, ios, FTL_IO_QUEUE_BATCH);
if (count == 0) {
return;
}
for (i = 0; i < count; i++) {
struct ftl_io *io = ios[i];
start_io(io);
}
}
static void
ftl_process_io_queue(struct spdk_ftl_dev *dev)
{
struct ftl_io_channel *ioch;
TAILQ_FOREACH(ioch, &dev->ioch_queue, entry) {
ftl_process_io_channel(dev, ioch);
}
}
int
ftl_core_poller(void *ctx)
{
@ -57,6 +127,8 @@ ftl_core_poller(void *ctx)
return SPDK_POLLER_IDLE;
}
ftl_process_io_queue(dev);
if (io_activity_total_old != dev->io_activity_total) {
return SPDK_POLLER_BUSY;
}

19
lib/ftl/ftl_core.h
View File

@ -28,6 +28,9 @@ struct spdk_ftl_dev {
/* FTL device layout */
struct ftl_layout layout;
/* Queue of registered IO channels */
TAILQ_HEAD(, ftl_io_channel) ioch_queue;
/* Underlying device */
struct spdk_bdev_desc *base_bdev_desc;
@ -49,6 +52,12 @@ struct spdk_ftl_dev {
/* Indicates the device is about to be stopped */
bool halt;
/* Indicates if the device is registered as an IO device */
bool io_device_registered;
/* Management process to be continued after IO device unregistration completes */
struct ftl_mngt_process *unregister_process;
/* counters for poller busy, include
1. nv cache read/write
2. metadata read/write
@ -99,6 +108,10 @@ struct spdk_ftl_dev {
int ftl_core_poller(void *ctx);
int ftl_io_channel_poll(void *arg);
struct ftl_io_channel *ftl_io_channel_get_ctx(struct spdk_io_channel *ioch);
static inline uint64_t
ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev)
{
@ -128,6 +141,12 @@ ftl_get_write_unit_size(struct spdk_bdev *bdev)
return 32;
}
static inline struct spdk_thread *
ftl_get_core_thread(const struct spdk_ftl_dev *dev)
{
return dev->core_thread;
}
static inline size_t
ftl_get_num_bands(const struct spdk_ftl_dev *dev)
{

1
lib/ftl/ftl_init.c
View File

@ -115,6 +115,7 @@ allocate_dev(const struct spdk_ftl_conf *conf, int *error)
TAILQ_INIT(&dev->rd_sq);
TAILQ_INIT(&dev->wr_sq);
TAILQ_INIT(&dev->ioch_queue);
return dev;
error:

14
lib/ftl/ftl_io.c
View File

@ -117,6 +117,9 @@ ftl_io_iovec_len_left(struct ftl_io *io)
static void
ftl_io_cb(struct ftl_io *io, void *arg, int status)
{
struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch);
size_t result;
if (spdk_unlikely(status)) {
io->status = status;
@ -145,15 +148,20 @@ ftl_io_cb(struct ftl_io *io, void *arg, int status)
}
}
/* User completion added in next patch */
if (io->map) {
ftl_mempool_put(ioch->map_pool, io->map);
}
result = spdk_ring_enqueue(ioch->cq, (void **)&io, 1, NULL);
assert(result != 0);
}
int
ftl_io_init(struct spdk_io_channel *_ioch, struct ftl_io *io, uint64_t lba, size_t num_blocks,
struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type)
{
/* dev initialized from io channel context in next patch */
struct spdk_ftl_dev *dev = NULL;
struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(_ioch);
struct spdk_ftl_dev *dev = ioch->dev;
memset(io, 0, sizeof(struct ftl_io));
io->ioch = _ioch;

15
lib/ftl/ftl_io.h
View File

@ -34,6 +34,21 @@ enum ftl_io_type {
#define FTL_IO_MAX_IOVEC 4
struct ftl_io_channel {
/* Device */
struct spdk_ftl_dev *dev;
/* Entry of IO channels queue/list */
TAILQ_ENTRY(ftl_io_channel) entry;
/* IO map pool */
struct ftl_mempool *map_pool;
/* Poller used for completing user requests and retrying IO */
struct spdk_poller *poller;
/* Submission queue */
struct spdk_ring *sq;
/* Completion queue */
struct spdk_ring *cq;
};
/* General IO descriptor for user requests */
struct ftl_io {
/* Device */

202
lib/ftl/mngt/ftl_mngt_ioch.c Normal file
View File

@ -0,0 +1,202 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) Intel Corporation.
* All rights reserved.
*/
#include "spdk/thread.h"
#include "ftl_core.h"
#include "ftl_mngt.h"
#include "ftl_mngt_steps.h"
#include "utils/ftl_mempool.h"
struct ftl_io_channel_ctx {
struct ftl_io_channel *ioch;
};
struct ftl_io_channel *
ftl_io_channel_get_ctx(struct spdk_io_channel *ioch)
{
struct ftl_io_channel_ctx *ctx = spdk_io_channel_get_ctx(ioch);
return ctx->ioch;
}
static void
ftl_dev_register_channel(void *ctx)
{
struct ftl_io_channel *ioch = ctx;
struct spdk_ftl_dev *dev = ioch->dev;
/* This only runs on the core thread, so it's safe to do this lockless */
TAILQ_INSERT_TAIL(&dev->ioch_queue, ioch, entry);
}
static void
io_channel_unregister(void *ctx)
{
struct ftl_io_channel *ioch = ctx;
struct spdk_ftl_dev *dev = ioch->dev;
TAILQ_REMOVE(&dev->ioch_queue, ioch, entry);
spdk_ring_free(ioch->cq);
spdk_ring_free(ioch->sq);
ftl_mempool_destroy(ioch->map_pool);
free(ioch);
}
static int
io_channel_create_cb(void *io_device, void *ctx)
{
struct spdk_ftl_dev *dev = io_device;
struct ftl_io_channel_ctx *_ioch = ctx;
struct ftl_io_channel *ioch;
char mempool_name[32];
int rc;
FTL_NOTICELOG(dev, "FTL IO channel created on %s\n",
spdk_thread_get_name(spdk_get_thread()));
/* This gets unregistered asynchronously with the device -
* we can't just use the ctx buffer passed by the thread library
*/
ioch = calloc(1, sizeof(*ioch));
if (ioch == NULL) {
FTL_ERRLOG(dev, "Failed to allocate IO channel\n");
return -1;
}
rc = snprintf(mempool_name, sizeof(mempool_name), "ftl_io_%p", ioch);
if (rc < 0 || rc >= (int)sizeof(mempool_name)) {
FTL_ERRLOG(dev, "Failed to create IO channel pool name\n");
free(ioch);
return -1;
}
ioch->dev = dev;
ioch->map_pool = ftl_mempool_create(
dev->conf.user_io_pool_size,
sizeof(ftl_addr) * dev->xfer_size,
64,
SPDK_ENV_SOCKET_ID_ANY);
if (!ioch->map_pool) {
FTL_ERRLOG(dev, "Failed to create IO channel's map IO pool\n");
goto fail_io_pool;
}
ioch->cq = spdk_ring_create(SPDK_RING_TYPE_SP_SC, spdk_align64pow2(dev->conf.user_io_pool_size + 1),
SPDK_ENV_SOCKET_ID_ANY);
if (!ioch->cq) {
FTL_ERRLOG(dev, "Failed to create IO channel completion queue\n");
goto fail_io_pool;
}
ioch->sq = spdk_ring_create(SPDK_RING_TYPE_SP_SC, spdk_align64pow2(dev->conf.user_io_pool_size + 1),
SPDK_ENV_SOCKET_ID_ANY);
if (!ioch->sq) {
FTL_ERRLOG(dev, "Failed to create IO channel submission queue\n");
goto fail_cq;
}
ioch->poller = SPDK_POLLER_REGISTER(ftl_io_channel_poll, ioch, 0);
if (!ioch->poller) {
FTL_ERRLOG(dev, "Failed to register IO channel poller\n");
goto fail_sq;
}
if (spdk_thread_send_msg(dev->core_thread, ftl_dev_register_channel, ioch)) {
FTL_ERRLOG(dev, "Failed to register IO channel\n");
goto fail_poller;
}
_ioch->ioch = ioch;
return 0;
fail_poller:
spdk_poller_unregister(&ioch->poller);
fail_cq:
spdk_ring_free(ioch->cq);
fail_sq:
spdk_ring_free(ioch->sq);
fail_io_pool:
ftl_mempool_destroy(ioch->map_pool);
free(ioch);
return -1;
}
static void
io_channel_destroy_cb(void *io_device, void *ctx)
{
struct ftl_io_channel_ctx *_ioch = ctx;
struct ftl_io_channel *ioch = _ioch->ioch;
struct spdk_ftl_dev *dev = ioch->dev;
FTL_NOTICELOG(dev, "FTL IO channel destroy on %s\n",
spdk_thread_get_name(spdk_get_thread()));
spdk_poller_unregister(&ioch->poller);
spdk_thread_send_msg(ftl_get_core_thread(dev),
io_channel_unregister, ioch);
}
void
ftl_mngt_register_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
{
dev->io_device_registered = true;
spdk_io_device_register(dev, io_channel_create_cb,
io_channel_destroy_cb,
sizeof(struct ftl_io_channel_ctx),
NULL);
ftl_mngt_next_step(mngt);
}
static void
unregister_cb(void *io_device)
{
struct spdk_ftl_dev *dev = io_device;
struct ftl_mngt_process *mngt = dev->unregister_process;
dev->io_device_registered = false;
dev->unregister_process = NULL;
ftl_mngt_next_step(mngt);
}
void
ftl_mngt_unregister_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
{
if (dev->io_device_registered) {
dev->unregister_process = mngt;
spdk_io_device_unregister(dev, unregister_cb);
} else {
ftl_mngt_skip_step(mngt);
}
}
void
ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
{
dev->ioch = spdk_get_io_channel(dev);
if (!dev->ioch) {
FTL_ERRLOG(dev, "Unable to get IO channel for core thread");
ftl_mngt_fail_step(mngt);
return;
}
ftl_mngt_next_step(mngt);
}
void
ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)
{
if (dev->ioch) {
spdk_put_io_channel(dev->ioch);
dev->ioch = NULL;
}
ftl_mngt_next_step(mngt);
}

8
lib/ftl/mngt/ftl_mngt_shutdown.c
View File

@ -15,6 +15,14 @@ static const struct ftl_mngt_process_desc desc_shutdown = {
.name = "FTL shutdown",
.error_handler = ftl_mngt_rollback_device,
.steps = {
{
.name = "Deinit core IO channel",
.action = ftl_mngt_deinit_io_channel
},
{
.name = "Unregister IO device",
.action = ftl_mngt_unregister_io_device
},
{
.name = "Stop core poller",
.action = ftl_mngt_stop_core_poller

10
lib/ftl/mngt/ftl_mngt_startup.c
View File

@ -42,6 +42,16 @@ static const struct ftl_mngt_process_desc desc_startup = {
.action = ftl_mngt_open_cache_bdev,
.cleanup = ftl_mngt_close_cache_bdev
},
{
.name = "Register IO device",
.action = ftl_mngt_register_io_device,
.cleanup = ftl_mngt_unregister_io_device
},
{
.name = "Initialize core IO channel",
.action = ftl_mngt_init_io_channel,
.cleanup = ftl_mngt_deinit_io_channel
},
{
.name = "Initialize layout",
.action = ftl_mngt_init_layout

8
lib/ftl/mngt/ftl_mngt_steps.h
View File

@ -18,6 +18,14 @@ void ftl_mngt_open_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process
void ftl_mngt_close_cache_bdev(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
void ftl_mngt_register_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
void ftl_mngt_unregister_io_device(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
void ftl_mngt_init_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
void ftl_mngt_deinit_io_channel(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
void ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);
void ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt);