lib/ftl: Add direct wptr and IO

Direct wptr allows for writing to band with externally
provided PPA, rather than one assigned by the wptr. This
can be useful when padding open bands during recovery or
bdev creation, as ftl can find partially written end meta
with no way to close the band with valid data, that would
conform to an already written CRC value.

This does mean that I/O sender is responsible for calculating
and obeying per-chunk write pointer arithmetics.

Signed-off-by: Mateusz Kozlowski <mateusz.kozlowski@intel.com>
Change-Id: Ifcbc4e8ffb69c829469a571bed8e27b148d7a651
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/455511
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

lib/ftl/ftl_core.c

@@ -69,6 +69,12 @@ struct ftl_wptr {
 
 	/* List link */
 	LIST_ENTRY(ftl_wptr)		list_entry;
+
+	/*
+	 * If setup in direct mode, there will be no offset or band state update after IO.
+	 * The PPA is not assigned by wptr, and is instead taken directly from the request.
+	 */
+	bool				direct_mode;
 };
 
 struct ftl_flush {
@@ -409,6 +415,57 @@ ftl_wptr_init(struct ftl_band *band)
 	return wptr;
 }
 
+static int
+ftl_add_direct_wptr(struct ftl_band *band)
+{
+	struct spdk_ftl_dev *dev = band->dev;
+	struct ftl_wptr *wptr;
+
+	assert(band->state == FTL_BAND_STATE_OPEN);
+
+	wptr = ftl_wptr_init(band);
+	if (!wptr) {
+		return -1;
+	}
+
+	wptr->direct_mode = true;
+
+	if (ftl_band_alloc_lba_map(band)) {
+		ftl_wptr_free(wptr);
+		return -1;
+	}
+
+	LIST_INSERT_HEAD(&dev->wptr_list, wptr, list_entry);
+
+	SPDK_DEBUGLOG(SPDK_LOG_FTL_CORE, "wptr: direct band %u\n", band->id);
+	ftl_trace_write_band(dev, band);
+	return 0;
+}
+
+static void
+ftl_close_direct_wptr(struct ftl_band *band)
+{
+	struct ftl_wptr *wptr = ftl_wptr_from_band(band);
+
+	assert(wptr->direct_mode);
+	assert(band->state == FTL_BAND_STATE_CLOSED);
+
+	ftl_band_release_lba_map(band);
+
+	ftl_remove_wptr(wptr);
+}
+
+int
+ftl_band_set_direct_access(struct ftl_band *band, bool access)
+{
+	if (access) {
+		return ftl_add_direct_wptr(band);
+	} else {
+		ftl_close_direct_wptr(band);
+		return 0;
+	}
+}
+
 static int
 ftl_add_wptr(struct spdk_ftl_dev *dev)
 {
@@ -445,6 +502,10 @@ ftl_wptr_advance(struct ftl_wptr *wptr, size_t xfer_size)
 	struct spdk_ftl_conf *conf = &dev->conf;
 	size_t next_thld;
 
+	if (spdk_unlikely(wptr->direct_mode)) {
+		return;
+	}
+
 	wptr->offset += xfer_size;
 	next_thld = (ftl_band_num_usable_lbks(band) * conf->band_thld) / 100;
 
@@ -1202,9 +1263,18 @@ ftl_submit_child_write(struct ftl_wptr *wptr, struct ftl_io *io, int lbk_cnt)
 	struct spdk_ftl_dev	*dev = io->dev;
 	struct ftl_io		*child;
 	int			rc;
+	struct ftl_ppa		ppa;
+
+	if (spdk_likely(!wptr->direct_mode)) {
+		ppa = wptr->ppa;
+	} else {
+		assert(io->flags & FTL_IO_DIRECT_ACCESS);
+		assert(io->ppa.chk == wptr->band->id);
+		ppa = io->ppa;
+	}
 
 	/* Split IO to child requests and release chunk immediately after child is completed */
-	child = ftl_io_init_child_write(io, wptr->ppa, ftl_io_iovec_addr(io),
+	child = ftl_io_init_child_write(io, ppa, ftl_io_iovec_addr(io),
 					ftl_io_get_md(io), ftl_io_child_write_cb);
 	if (!child) {
 		return -EAGAIN;
@@ -1212,13 +1282,13 @@ ftl_submit_child_write(struct ftl_wptr *wptr, struct ftl_io *io, int lbk_cnt)
 
 	rc = spdk_nvme_ns_cmd_write_with_md(dev->ns, ftl_get_write_qpair(dev),
 					    ftl_io_iovec_addr(child), child->md,
-					    ftl_ppa_addr_pack(dev, wptr->ppa),
+					    ftl_ppa_addr_pack(dev, ppa),
 					    lbk_cnt, ftl_io_cmpl_cb, child, 0, 0, 0);
 	if (rc) {
 		ftl_io_fail(child, rc);
 		ftl_io_complete(child);
 		SPDK_ERRLOG("spdk_nvme_ns_cmd_write failed with status:%d, ppa:%lu\n",
-			    rc, wptr->ppa.ppa);
+			    rc, ppa.ppa);
 
 		return -EIO;
 	}

lib/ftl/ftl_core.h

@@ -248,6 +248,7 @@ size_t	ftl_lba_map_num_lbks(const struct spdk_ftl_dev *dev);
 size_t	ftl_head_md_num_lbks(const struct spdk_ftl_dev *dev);
 int	ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb);
 int	ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb);
+int	ftl_band_set_direct_access(struct ftl_band *band, bool access);
 
 #define ftl_to_ppa(addr) \
 	(struct ftl_ppa) { .ppa = (uint64_t)(addr) }

lib/ftl/ftl_io.h

@@ -72,6 +72,9 @@ enum ftl_io_flags {
 	FTL_IO_RETRY		= (1 << 8),
 	/* The IO is directed to non-volatile cache */
 	FTL_IO_CACHE		= (1 << 9),
+	/* Indicates that PPA should be taken from IO struct, */
+	/* not assigned by wptr, only works if wptr is also in direct mode */
+	FTL_IO_DIRECT_ACCESS	= (1 << 10),
 };
 
 enum ftl_io_type {