lib/ftl: Internal IO retry mechanism in case ENOMEM from nvme layer

Added internal retry IO queue. In case ENOMEM from
nvme layer request is put to retry queue. When some
inflight operations are completed try to make progress
on IOs in retry queue in first place.

Change-Id: Ie9d3d20bd34431ee57f9454f242b0cdca349c804
Signed-off-by: Wojciech Malikowski <wojciech.malikowski@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/448461
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>

lib/ftl/ftl_core.c

@@ -702,6 +702,15 @@ ftl_read_canceled(int rc)
 }
 
 static void
+ftl_add_to_retry_queue(struct ftl_io *io)
+{
+	if (!(io->flags & FTL_IO_RETRY)) {
+		io->flags |= FTL_IO_RETRY;
+		TAILQ_INSERT_TAIL(&io->dev->retry_queue, io, retry_entry);
+	}
+}
+
+static int
 ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa,
 		void *ctx)
 {
@@ -733,11 +742,11 @@ ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa,
 					   ftl_io_iovec_addr(io),
 					   ftl_ppa_addr_pack(io->dev, ppa), lbk_cnt,
 					   ftl_io_cmpl_cb, io, 0);
-		if (rc) {
+		if (rc == -ENOMEM) {
+			ftl_add_to_retry_queue(io);
+			break;
+		} else if (rc) {
 			io->status = rc;
-			if (rc != -ENOMEM) {
-				SPDK_ERRLOG("spdk_nvme_ns_cmd_read failed with status: %d\n", rc);
-			}
 			break;
 		}
 
@@ -750,6 +759,8 @@ ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa,
 	if (ftl_io_done(io)) {
 		ftl_io_complete(io);
 	}
+
+	return rc;
 }
 
 static int
@@ -1384,7 +1395,7 @@ spdk_ftl_write(struct spdk_ftl_dev *dev, struct spdk_io_channel *ch, uint64_t lb
 	return _spdk_ftl_write(io);
 }
 
-void
+int
 ftl_io_read(struct ftl_io *io)
 {
 	struct spdk_ftl_dev *dev = io->dev;
@@ -1397,11 +1408,11 @@ ftl_io_read(struct ftl_io *io)
 			next_ppa = ftl_lba_read_next_ppa;
 		}
 
-		ftl_submit_read(io, next_ppa, NULL);
-		return;
+		return ftl_submit_read(io, next_ppa, NULL);
 	}
 
 	spdk_thread_send_msg(ftl_get_read_thread(dev), _ftl_read, io);
+	return 0;
 }
 
 static void
@@ -1517,12 +1528,39 @@ ftl_process_anm_event(struct ftl_anm_event *event)
 	ftl_anm_event_complete(event);
 }
 
+static void
+ftl_process_retry_queue(struct spdk_ftl_dev *dev)
+{
+	struct ftl_io *io;
+	int rc;
+
+	while (!TAILQ_EMPTY(&dev->retry_queue)) {
+		io = TAILQ_FIRST(&dev->retry_queue);
+
+		/* Retry only if IO is still healthy */
+		if (spdk_likely(io->status == 0)) {
+			rc = ftl_io_read(io);
+			if (rc == -ENOMEM) {
+				break;
+			}
+		}
+
+		io->flags &= ~FTL_IO_RETRY;
+		TAILQ_REMOVE(&dev->retry_queue, io, retry_entry);
+
+		if (ftl_io_done(io)) {
+			ftl_io_complete(io);
+		}
+	}
+}
+
 int
 ftl_task_read(void *ctx)
 {
 	struct ftl_thread *thread = ctx;
 	struct spdk_ftl_dev *dev = thread->dev;
 	struct spdk_nvme_qpair *qpair = ftl_get_read_qpair(dev);
+	size_t num_completed;
 
 	if (dev->halt) {
 		if (ftl_shutdown_complete(dev)) {
@@ -1531,7 +1569,13 @@ ftl_task_read(void *ctx)
 		}
 	}
 
-	return spdk_nvme_qpair_process_completions(qpair, 0);
+	num_completed = spdk_nvme_qpair_process_completions(qpair, 0);
+
+	if (num_completed && !TAILQ_EMPTY(&dev->retry_queue)) {
+		ftl_process_retry_queue(dev);
+	}
+
+	return num_completed;
 }
 
 int

lib/ftl/ftl_core.h

@@ -200,6 +200,8 @@ struct spdk_ftl_dev {
 
 	/* Inflight IO operations */
 	uint32_t				num_inflight;
+	/* Queue of IO awaiting retry */
+	TAILQ_HEAD(, ftl_io)			retry_queue;
 
 	/* Manages data relocation */
 	struct ftl_reloc			*reloc;
@@ -215,7 +217,7 @@ struct spdk_ftl_dev {
 typedef void (*ftl_restore_fn)(struct spdk_ftl_dev *, struct ftl_restore *, int);
 
 void	ftl_apply_limits(struct spdk_ftl_dev *dev);
-void	ftl_io_read(struct ftl_io *io);
+int	ftl_io_read(struct ftl_io *io);
 int	ftl_io_write(struct ftl_io *io);
 int	ftl_io_erase(struct ftl_io *io);
 int	ftl_io_flush(struct ftl_io *io);

lib/ftl/ftl_init.c

@@ -838,6 +838,7 @@ spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *opts, spdk_ftl_init_fn cb
 				sizeof(struct ftl_io_channel),
 				NULL);
 
+	TAILQ_INIT(&dev->retry_queue);
 	dev->ioch = spdk_get_io_channel(dev);
 	dev->init_cb = cb;
 	dev->init_arg = cb_arg;

lib/ftl/ftl_io.h

@@ -68,6 +68,8 @@ enum ftl_io_flags {
 	FTL_IO_PPA_MODE		= (1 << 6),
 	/* Indicates that IO contains noncontiguous LBAs */
 	FTL_IO_VECTOR_LBA	= (1 << 7),
+	/* Indicates that IO is being retried */
+	FTL_IO_RETRY		= (1 << 8),
 };
 
 enum ftl_io_type {
@@ -197,6 +199,8 @@ struct ftl_io {
 
 	/* Trace group id */
 	uint64_t				trace;
+
+	TAILQ_ENTRY(ftl_io)			retry_entry;
 };
 
 /* Metadata IO */
@@ -232,7 +236,7 @@ ftl_io_mode_lba(const struct ftl_io *io)
 static inline bool
 ftl_io_done(const struct ftl_io *io)
 {
-	return io->req_cnt == 0;
+	return io->req_cnt == 0 && !(io->flags & FTL_IO_RETRY);
 }
 
 struct ftl_io *ftl_io_alloc(struct spdk_io_channel *ch);

lib/ftl/ftl_reloc.c

@@ -473,6 +473,7 @@ ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io)
 {
 	struct ftl_ppa ppa;
 	size_t num_lbks;
+	int rc;
 
 	num_lbks = ftl_reloc_next_lbks(breloc, &ppa);
 
@@ -486,8 +487,12 @@ ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io)
 		return -1;
 	}
 
-	ftl_io_read(io);
-	return 0;
+	rc = ftl_io_read(io);
+	if (rc == -ENOMEM) {
+		rc = 0;
+	}
+
+	return rc;
 }
 
 static void

test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c

@@ -125,10 +125,11 @@ ftl_band_ppa_from_lbkoff(struct ftl_band *band, uint64_t lbkoff)
 	return ppa;
 }
 
-void
+int
 ftl_io_read(struct ftl_io *io)
 {
 	io->cb.fn(io->cb.ctx, 0);
+	return 0;
 }
 
 int