From 04814b72a85ab270844f2999219beab0809e9e64 Mon Sep 17 00:00:00 2001 From: Wojciech Malikowski Date: Tue, 19 Mar 2019 11:29:34 -0400 Subject: [PATCH] lib/ftl: Internal IO retry mechanism in case ENOMEM from nvme layer Added internal retry IO queue. In case ENOMEM from nvme layer request is put to retry queue. When some inflight operations are completed try to make progress on IOs in retry queue in first place. Change-Id: Ie9d3d20bd34431ee57f9454f242b0cdca349c804 Signed-off-by: Wojciech Malikowski Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/448461 Tested-by: SPDK CI Jenkins Reviewed-by: Konrad Sztyber Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- lib/ftl/ftl_core.c | 60 +++++++++++++++++--- lib/ftl/ftl_core.h | 4 +- lib/ftl/ftl_init.c | 1 + lib/ftl/ftl_io.h | 6 +- lib/ftl/ftl_reloc.c | 9 ++- test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c | 3 +- 6 files changed, 70 insertions(+), 13 deletions(-) diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 8766a959a..412f72a55 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -702,6 +702,15 @@ ftl_read_canceled(int rc) } static void +ftl_add_to_retry_queue(struct ftl_io *io) +{ + if (!(io->flags & FTL_IO_RETRY)) { + io->flags |= FTL_IO_RETRY; + TAILQ_INSERT_TAIL(&io->dev->retry_queue, io, retry_entry); + } +} + +static int ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa, void *ctx) { @@ -733,11 +742,11 @@ ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa, ftl_io_iovec_addr(io), ftl_ppa_addr_pack(io->dev, ppa), lbk_cnt, ftl_io_cmpl_cb, io, 0); - if (rc) { + if (rc == -ENOMEM) { + ftl_add_to_retry_queue(io); + break; + } else if (rc) { io->status = rc; - if (rc != -ENOMEM) { - SPDK_ERRLOG("spdk_nvme_ns_cmd_read failed with status: %d\n", rc); - } break; } @@ -750,6 +759,8 @@ ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa, if (ftl_io_done(io)) { ftl_io_complete(io); } + + return rc; } static int @@ -1384,7 +1395,7 @@ spdk_ftl_write(struct spdk_ftl_dev *dev, struct spdk_io_channel *ch, uint64_t lb return _spdk_ftl_write(io); } -void +int ftl_io_read(struct ftl_io *io) { struct spdk_ftl_dev *dev = io->dev; @@ -1397,11 +1408,11 @@ ftl_io_read(struct ftl_io *io) next_ppa = ftl_lba_read_next_ppa; } - ftl_submit_read(io, next_ppa, NULL); - return; + return ftl_submit_read(io, next_ppa, NULL); } spdk_thread_send_msg(ftl_get_read_thread(dev), _ftl_read, io); + return 0; } static void @@ -1517,12 +1528,39 @@ ftl_process_anm_event(struct ftl_anm_event *event) ftl_anm_event_complete(event); } +static void +ftl_process_retry_queue(struct spdk_ftl_dev *dev) +{ + struct ftl_io *io; + int rc; + + while (!TAILQ_EMPTY(&dev->retry_queue)) { + io = TAILQ_FIRST(&dev->retry_queue); + + /* Retry only if IO is still healthy */ + if (spdk_likely(io->status == 0)) { + rc = ftl_io_read(io); + if (rc == -ENOMEM) { + break; + } + } + + io->flags &= ~FTL_IO_RETRY; + TAILQ_REMOVE(&dev->retry_queue, io, retry_entry); + + if (ftl_io_done(io)) { + ftl_io_complete(io); + } + } +} + int ftl_task_read(void *ctx) { struct ftl_thread *thread = ctx; struct spdk_ftl_dev *dev = thread->dev; struct spdk_nvme_qpair *qpair = ftl_get_read_qpair(dev); + size_t num_completed; if (dev->halt) { if (ftl_shutdown_complete(dev)) { @@ -1531,7 +1569,13 @@ ftl_task_read(void *ctx) } } - return spdk_nvme_qpair_process_completions(qpair, 0); + num_completed = spdk_nvme_qpair_process_completions(qpair, 0); + + if (num_completed && !TAILQ_EMPTY(&dev->retry_queue)) { + ftl_process_retry_queue(dev); + } + + return num_completed; } int diff --git a/lib/ftl/ftl_core.h b/lib/ftl/ftl_core.h index 2ce5faecd..0c35006a9 100644 --- a/lib/ftl/ftl_core.h +++ b/lib/ftl/ftl_core.h @@ -200,6 +200,8 @@ struct spdk_ftl_dev { /* Inflight IO operations */ uint32_t num_inflight; + /* Queue of IO awaiting retry */ + TAILQ_HEAD(, ftl_io) retry_queue; /* Manages data relocation */ struct ftl_reloc *reloc; @@ -215,7 +217,7 @@ struct spdk_ftl_dev { typedef void (*ftl_restore_fn)(struct spdk_ftl_dev *, struct ftl_restore *, int); void ftl_apply_limits(struct spdk_ftl_dev *dev); -void ftl_io_read(struct ftl_io *io); +int ftl_io_read(struct ftl_io *io); int ftl_io_write(struct ftl_io *io); int ftl_io_erase(struct ftl_io *io); int ftl_io_flush(struct ftl_io *io); diff --git a/lib/ftl/ftl_init.c b/lib/ftl/ftl_init.c index 4e1c4f944..891ec9e7c 100644 --- a/lib/ftl/ftl_init.c +++ b/lib/ftl/ftl_init.c @@ -838,6 +838,7 @@ spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *opts, spdk_ftl_init_fn cb sizeof(struct ftl_io_channel), NULL); + TAILQ_INIT(&dev->retry_queue); dev->ioch = spdk_get_io_channel(dev); dev->init_cb = cb; dev->init_arg = cb_arg; diff --git a/lib/ftl/ftl_io.h b/lib/ftl/ftl_io.h index b2325fb55..5f5519e87 100644 --- a/lib/ftl/ftl_io.h +++ b/lib/ftl/ftl_io.h @@ -68,6 +68,8 @@ enum ftl_io_flags { FTL_IO_PPA_MODE = (1 << 6), /* Indicates that IO contains noncontiguous LBAs */ FTL_IO_VECTOR_LBA = (1 << 7), + /* Indicates that IO is being retried */ + FTL_IO_RETRY = (1 << 8), }; enum ftl_io_type { @@ -197,6 +199,8 @@ struct ftl_io { /* Trace group id */ uint64_t trace; + + TAILQ_ENTRY(ftl_io) retry_entry; }; /* Metadata IO */ @@ -232,7 +236,7 @@ ftl_io_mode_lba(const struct ftl_io *io) static inline bool ftl_io_done(const struct ftl_io *io) { - return io->req_cnt == 0; + return io->req_cnt == 0 && !(io->flags & FTL_IO_RETRY); } struct ftl_io *ftl_io_alloc(struct spdk_io_channel *ch); diff --git a/lib/ftl/ftl_reloc.c b/lib/ftl/ftl_reloc.c index cdc4a57b4..1d5648aed 100644 --- a/lib/ftl/ftl_reloc.c +++ b/lib/ftl/ftl_reloc.c @@ -473,6 +473,7 @@ ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io) { struct ftl_ppa ppa; size_t num_lbks; + int rc; num_lbks = ftl_reloc_next_lbks(breloc, &ppa); @@ -486,8 +487,12 @@ ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io) return -1; } - ftl_io_read(io); - return 0; + rc = ftl_io_read(io); + if (rc == -ENOMEM) { + rc = 0; + } + + return rc; } static void diff --git a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c b/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c index fd800b0e7..20d2e8eb0 100644 --- a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c +++ b/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c @@ -125,10 +125,11 @@ ftl_band_ppa_from_lbkoff(struct ftl_band *band, uint64_t lbkoff) return ppa; } -void +int ftl_io_read(struct ftl_io *io) { io->cb.fn(io->cb.ctx, 0); + return 0; } int