FTL: Add read path

Signed-off-by: Kozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: Ib5bac109b59d5a21a7dad1f8e79b5da7633ffa9d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13334
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

autotest.sh

@@ -305,6 +305,10 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 		run_test "ocf" ./test/ocf/ocf.sh
 	fi
 
+	if [ $SPDK_TEST_FTL -eq 1 ]; then
+		run_test "ftl" ./test/ftl/ftl.sh
+	fi
+
 	if [ $SPDK_TEST_VMD -eq 1 ]; then
 		run_test "vmd" ./test/vmd/vmd.sh
 	fi

include/spdk/ftl.h

@@ -155,6 +155,25 @@ void spdk_ftl_conf_deinit(struct spdk_ftl_conf *conf);
  */
 void spdk_ftl_get_default_conf(struct spdk_ftl_conf *conf);
 
+/**
+ * Submits a read to the specified device.
+ *
+ * \param dev Device
+ * \param io Allocated ftl_io
+ * \param ch I/O channel
+ * \param lba Starting LBA to read the data
+ * \param lba_cnt Number of sectors to read
+ * \param iov Single IO vector or pointer to IO vector table
+ * \param iov_cnt Number of IO vectors
+ * \param cb_fn Callback function to invoke when the I/O is completed
+ * \param cb_arg Argument to pass to the callback function
+ *
+ * \return 0 if successfully submitted, negative errno otherwise.
+ */
+int spdk_ftl_readv(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch,
+		   uint64_t lba, uint64_t lba_cnt,
+		   struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg);
+
 /**
  * Submits a write to the specified device.
  *

lib/ftl/ftl_core.c

@@ -26,6 +26,23 @@ spdk_ftl_io_size(void)
 	return sizeof(struct ftl_io);
 }
 
+static void
+ftl_io_cmpl_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
+{
+	struct ftl_io *io = cb_arg;
+
+	if (spdk_unlikely(!success)) {
+		io->status = -EIO;
+	}
+
+	ftl_io_dec_req(io);
+	if (ftl_io_done(io)) {
+		ftl_io_complete(io);
+	}
+
+	spdk_bdev_free_io(bdev_io);
+}
+
 static void
 ftl_band_erase(struct ftl_band *band)
 {
@@ -118,6 +135,126 @@ ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr)
 	}
 }
 
+static int
+ftl_read_canceled(int rc)
+{
+	return rc == -EFAULT;
+}
+
+static int
+ftl_get_next_read_addr(struct ftl_io *io, ftl_addr *addr)
+{
+	struct spdk_ftl_dev *dev = io->dev;
+	ftl_addr next_addr;
+	size_t i;
+	bool addr_cached = false;
+
+	*addr = ftl_l2p_get(dev, ftl_io_current_lba(io));
+	io->map[io->pos] = *addr;
+
+	/* If the address is invalid, skip it */
+	if (*addr == FTL_ADDR_INVALID) {
+		return -EFAULT;
+	}
+
+	addr_cached = ftl_addr_in_nvc(dev, *addr);
+
+	for (i = 1; i < ftl_io_iovec_len_left(io); ++i) {
+		next_addr = ftl_l2p_get(dev, ftl_io_get_lba(io, io->pos + i));
+
+		if (next_addr == FTL_ADDR_INVALID) {
+			break;
+		}
+
+		/* It's not enough to check for contiguity, if user data is on the last block
+		 * of base device and first nvc, then they're 'contiguous', but can't be handled
+		 * with one read request.
+		 */
+		if (addr_cached != ftl_addr_in_nvc(dev, next_addr)) {
+			break;
+		}
+
+		if (*addr + i != next_addr) {
+			break;
+		}
+
+		io->map[io->pos + i] = next_addr;
+	}
+
+	return i;
+}
+
+static void ftl_submit_read(struct ftl_io *io);
+
+static void
+_ftl_submit_read(void *_io)
+{
+	struct ftl_io *io = _io;
+
+	ftl_submit_read(io);
+}
+
+static void
+ftl_submit_read(struct ftl_io *io)
+{
+	struct spdk_ftl_dev *dev = io->dev;
+	ftl_addr addr;
+	int rc = 0, num_blocks;
+
+	while (io->pos < io->num_blocks) {
+		num_blocks = ftl_get_next_read_addr(io, &addr);
+		rc = num_blocks;
+
+		/* User LBA doesn't hold valid data (trimmed or never written to), fill with 0 and skip this block */
+		if (ftl_read_canceled(rc)) {
+			memset(ftl_io_iovec_addr(io), 0, FTL_BLOCK_SIZE);
+			ftl_io_advance(io, 1);
+			continue;
+		}
+
+		assert(num_blocks > 0);
+
+		if (ftl_addr_in_nvc(dev, addr)) {
+			rc = ftl_nv_cache_read(io, addr, num_blocks, ftl_io_cmpl_cb, io);
+		} else {
+			rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch,
+						   ftl_io_iovec_addr(io),
+						   addr, num_blocks, ftl_io_cmpl_cb, io);
+		}
+
+		if (spdk_unlikely(rc)) {
+			if (rc == -ENOMEM) {
+				struct spdk_bdev *bdev;
+				struct spdk_io_channel *ch;
+
+				if (ftl_addr_in_nvc(dev, addr)) {
+					bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc);
+					ch = dev->nv_cache.cache_ioch;
+				} else {
+					bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc);
+					ch = dev->base_ioch;
+				}
+				io->bdev_io_wait.bdev = bdev;
+				io->bdev_io_wait.cb_fn = _ftl_submit_read;
+				io->bdev_io_wait.cb_arg = io;
+				spdk_bdev_queue_io_wait(bdev, ch, &io->bdev_io_wait);
+				return;
+			} else {
+				ftl_abort();
+			}
+		}
+
+		ftl_io_inc_req(io);
+		ftl_io_advance(io, num_blocks);
+	}
+
+	/* If we didn't have to read anything from the device, */
+	/* complete the request right away */
+	if (ftl_io_done(io)) {
+		ftl_io_complete(io);
+	}
+}
+
 void
 spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attrs)
 {
@@ -126,6 +263,38 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at
 	attrs->optimum_io_size = dev->xfer_size;
 }
 
+static void
+ftl_io_pin_cb(struct spdk_ftl_dev *dev, int status, struct ftl_l2p_pin_ctx *pin_ctx)
+{
+	struct ftl_io *io = pin_ctx->cb_ctx;
+
+	if (spdk_unlikely(status != 0)) {
+		/* Retry on the internal L2P fault */
+		io->status = -EAGAIN;
+		ftl_io_complete(io);
+		return;
+	}
+
+	io->flags |= FTL_IO_PINNED;
+	ftl_submit_read(io);
+}
+
+static void
+ftl_io_pin(struct ftl_io *io)
+{
+	if (spdk_unlikely(io->flags & FTL_IO_PINNED)) {
+		/*
+		 * The IO is in a retry path and it had been pinned already.
+		 * Continue with further processing.
+		 */
+		ftl_l2p_pin_skip(io->dev, ftl_io_pin_cb, io, &io->l2p_pin_ctx);
+	} else {
+		/* First time when pinning the IO */
+		ftl_l2p_pin(io->dev, io->lba, io->num_blocks,
+			    ftl_io_pin_cb, io, &io->l2p_pin_ctx);
+	}
+}
+
 static void
 start_io(struct ftl_io *io)
 {
@@ -141,8 +310,7 @@ start_io(struct ftl_io *io)
 
 	switch (io->type) {
 	case FTL_IO_READ:
-		io->status = -EOPNOTSUPP;
-		ftl_io_complete(io);
+		TAILQ_INSERT_TAIL(&dev->rd_sq, io, queue_entry);
 		break;
 	case FTL_IO_WRITE:
 		TAILQ_INSERT_TAIL(&dev->wr_sq, io, queue_entry);
@@ -201,6 +369,38 @@ spdk_ftl_writev(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_chan
 	return queue_io(dev, io);
 }
 
+int
+spdk_ftl_readv(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch,
+	       uint64_t lba, uint64_t lba_cnt, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg)
+{
+	int rc;
+
+	if (iov_cnt == 0) {
+		return -EINVAL;
+	}
+
+	if (lba_cnt == 0) {
+		return -EINVAL;
+	}
+
+	if (lba_cnt != ftl_iovec_num_blocks(iov, iov_cnt)) {
+		FTL_ERRLOG(dev, "Invalid IO vector to handle, device %s, LBA %"PRIu64"\n",
+			   dev->conf.name, lba);
+		return -EINVAL;
+	}
+
+	if (!dev->initialized) {
+		return -EBUSY;
+	}
+
+	rc = ftl_io_init(ch, io, lba, lba_cnt, iov, iov_cnt, cb_fn, cb_arg, FTL_IO_READ);
+	if (rc) {
+		return rc;
+	}
+
+	return queue_io(dev, io);
+}
+
 #define FTL_IO_QUEUE_BATCH 16
 int
 ftl_io_channel_poll(void *arg)
@@ -245,6 +445,16 @@ ftl_process_io_queue(struct spdk_ftl_dev *dev)
 	struct ftl_io_channel *ioch;
 	struct ftl_io *io;
 
+	/* TODO: Try to figure out a mechanism to batch more requests at the same time,
+	 * with keeping enough resources (pinned pages), between reads, writes and gc/compaction
+	 */
+	if (!TAILQ_EMPTY(&dev->rd_sq)) {
+		io = TAILQ_FIRST(&dev->rd_sq);
+		TAILQ_REMOVE(&dev->rd_sq, io, queue_entry);
+		assert(io->type == FTL_IO_READ);
+		ftl_io_pin(io);
+	}
+
 	if (!ftl_nv_cache_full(&dev->nv_cache) && !TAILQ_EMPTY(&dev->wr_sq)) {
 		io = TAILQ_FIRST(&dev->wr_sq);
 		TAILQ_REMOVE(&dev->wr_sq, io, queue_entry);

lib/ftl/spdk_ftl.map

@@ -13,6 +13,7 @@
 	spdk_ftl_conf_deinit;
 	spdk_ftl_get_io_channel;
 	spdk_ftl_io_size;
+	spdk_ftl_readv;
 	spdk_ftl_writev;
 
 	local: *;

module/bdev/ftl/bdev_ftl.c

@@ -105,6 +105,31 @@ bdev_ftl_cb(void *arg, int rc)
 	spdk_bdev_io_complete(bdev_io, status);
 }
 
+static void
+bdev_ftl_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
+		    bool success)
+{
+	struct ftl_bdev *ftl_bdev;
+	int rc;
+
+	ftl_bdev = bdev_io->bdev->ctxt;
+
+	if (!success) {
+		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
+		return;
+	}
+
+	rc = spdk_ftl_readv(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx,
+			    ch,
+			    bdev_io->u.bdev.offset_blocks,
+			    bdev_io->u.bdev.num_blocks,
+			    bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io);
+
+	if (spdk_unlikely(rc != 0)) {
+		spdk_bdev_io_complete(bdev_io, rc);
+	}
+}
+
 static int
 _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
 {
@@ -112,7 +137,8 @@ _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_i
 
 	switch (bdev_io->type) {
 	case SPDK_BDEV_IO_TYPE_READ:
-		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS);
+		spdk_bdev_io_get_buf(bdev_io, bdev_ftl_get_buf_cb,
+				     bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
 		return 0;
 
 	case SPDK_BDEV_IO_TYPE_WRITE:

test/ftl/config/fio/randr.fio

@@ -0,0 +1,19 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=${FTL_JSON_CONF}
+filename=${FTL_BDEV_NAME}
+direct=1
+thread=1
+buffered=0
+size=100%
+randrepeat=0
+time_based
+norandommap
+
+[test]
+stonewall
+bs=4k
+numjobs=4
+rw=randread
+iodepth=128
+runtime=1200

test/ftl/config/fio/randw-verify-depth128.fio

@@ -7,7 +7,7 @@ direct=1
 iodepth=128
 rw=randwrite
 verify=crc32c
-do_verify=0
+do_verify=1
 verify_dump=0
 verify_state_save=0
 verify_fatal=1

test/ftl/config/fio/randw-verify-j2.fio

@@ -7,7 +7,7 @@ direct=1
 iodepth=128
 rw=randwrite
 verify=crc32c
-do_verify=0
+do_verify=1
 verify_dump=0
 verify_state_save=0
 verify_backlog=5000

test/ftl/config/fio/randw-verify-qd2048-ext.fio

@@ -11,7 +11,7 @@ size=100%
 numjobs=1
 rw=randwrite
 verify=crc32c
-do_verify=0
+do_verify=1
 verify_dump=0
 verify_state_save=0
 verify_fatal=1

test/ftl/config/fio/randw-verify.fio

@@ -8,7 +8,7 @@ iodepth=1
 rw=randwrite
 size=256M
 verify=crc32c
-do_verify=0
+do_verify=1
 verify_dump=0
 verify_state_save=0
 verify_backlog=16

test/ftl/fio.sh

@@ -7,7 +7,7 @@ source $testdir/common.sh
 
 declare -A suite
 suite['basic']='randw-verify randw-verify-j2 randw-verify-depth128'
-suite['extended']='randw-verify-qd2048-ext'
+suite['extended']='randw-verify-qd2048-ext randr'
 
 rpc_py=$rootdir/scripts/rpc.py