From 0291b2845af1dfb3e432f391de3beb7a9f9fb6b1 Mon Sep 17 00:00:00 2001 From: Artur Paszkiewicz Date: Tue, 28 Jun 2022 15:47:14 +0200 Subject: [PATCH] FTL: Add read path Signed-off-by: Kozlowski Mateusz Signed-off-by: Artur Paszkiewicz Change-Id: Ib5bac109b59d5a21a7dad1f8e79b5da7633ffa9d Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13334 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Jim Harris --- autotest.sh | 4 + include/spdk/ftl.h | 19 ++ lib/ftl/ftl_core.c | 214 +++++++++++++++++- lib/ftl/spdk_ftl.map | 1 + module/bdev/ftl/bdev_ftl.c | 28 ++- test/ftl/config/fio/randr.fio | 19 ++ test/ftl/config/fio/randw-verify-depth128.fio | 2 +- test/ftl/config/fio/randw-verify-j2.fio | 2 +- .../config/fio/randw-verify-qd2048-ext.fio | 2 +- test/ftl/config/fio/randw-verify.fio | 2 +- test/ftl/fio.sh | 2 +- 11 files changed, 287 insertions(+), 8 deletions(-) create mode 100644 test/ftl/config/fio/randr.fio diff --git a/autotest.sh b/autotest.sh index 6d446f00f..85b90efc7 100755 --- a/autotest.sh +++ b/autotest.sh @@ -305,6 +305,10 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then run_test "ocf" ./test/ocf/ocf.sh fi + if [ $SPDK_TEST_FTL -eq 1 ]; then + run_test "ftl" ./test/ftl/ftl.sh + fi + if [ $SPDK_TEST_VMD -eq 1 ]; then run_test "vmd" ./test/vmd/vmd.sh fi diff --git a/include/spdk/ftl.h b/include/spdk/ftl.h index 512f6191b..4f29fb92a 100644 --- a/include/spdk/ftl.h +++ b/include/spdk/ftl.h @@ -155,6 +155,25 @@ void spdk_ftl_conf_deinit(struct spdk_ftl_conf *conf); */ void spdk_ftl_get_default_conf(struct spdk_ftl_conf *conf); +/** + * Submits a read to the specified device. + * + * \param dev Device + * \param io Allocated ftl_io + * \param ch I/O channel + * \param lba Starting LBA to read the data + * \param lba_cnt Number of sectors to read + * \param iov Single IO vector or pointer to IO vector table + * \param iov_cnt Number of IO vectors + * \param cb_fn Callback function to invoke when the I/O is completed + * \param cb_arg Argument to pass to the callback function + * + * \return 0 if successfully submitted, negative errno otherwise. + */ +int spdk_ftl_readv(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch, + uint64_t lba, uint64_t lba_cnt, + struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg); + /** * Submits a write to the specified device. * diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index dee909ede..1e52c6e6a 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -26,6 +26,23 @@ spdk_ftl_io_size(void) return sizeof(struct ftl_io); } +static void +ftl_io_cmpl_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct ftl_io *io = cb_arg; + + if (spdk_unlikely(!success)) { + io->status = -EIO; + } + + ftl_io_dec_req(io); + if (ftl_io_done(io)) { + ftl_io_complete(io); + } + + spdk_bdev_free_io(bdev_io); +} + static void ftl_band_erase(struct ftl_band *band) { @@ -118,6 +135,126 @@ ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr) } } +static int +ftl_read_canceled(int rc) +{ + return rc == -EFAULT; +} + +static int +ftl_get_next_read_addr(struct ftl_io *io, ftl_addr *addr) +{ + struct spdk_ftl_dev *dev = io->dev; + ftl_addr next_addr; + size_t i; + bool addr_cached = false; + + *addr = ftl_l2p_get(dev, ftl_io_current_lba(io)); + io->map[io->pos] = *addr; + + /* If the address is invalid, skip it */ + if (*addr == FTL_ADDR_INVALID) { + return -EFAULT; + } + + addr_cached = ftl_addr_in_nvc(dev, *addr); + + for (i = 1; i < ftl_io_iovec_len_left(io); ++i) { + next_addr = ftl_l2p_get(dev, ftl_io_get_lba(io, io->pos + i)); + + if (next_addr == FTL_ADDR_INVALID) { + break; + } + + /* It's not enough to check for contiguity, if user data is on the last block + * of base device and first nvc, then they're 'contiguous', but can't be handled + * with one read request. + */ + if (addr_cached != ftl_addr_in_nvc(dev, next_addr)) { + break; + } + + if (*addr + i != next_addr) { + break; + } + + io->map[io->pos + i] = next_addr; + } + + return i; +} + +static void ftl_submit_read(struct ftl_io *io); + +static void +_ftl_submit_read(void *_io) +{ + struct ftl_io *io = _io; + + ftl_submit_read(io); +} + +static void +ftl_submit_read(struct ftl_io *io) +{ + struct spdk_ftl_dev *dev = io->dev; + ftl_addr addr; + int rc = 0, num_blocks; + + while (io->pos < io->num_blocks) { + num_blocks = ftl_get_next_read_addr(io, &addr); + rc = num_blocks; + + /* User LBA doesn't hold valid data (trimmed or never written to), fill with 0 and skip this block */ + if (ftl_read_canceled(rc)) { + memset(ftl_io_iovec_addr(io), 0, FTL_BLOCK_SIZE); + ftl_io_advance(io, 1); + continue; + } + + assert(num_blocks > 0); + + if (ftl_addr_in_nvc(dev, addr)) { + rc = ftl_nv_cache_read(io, addr, num_blocks, ftl_io_cmpl_cb, io); + } else { + rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch, + ftl_io_iovec_addr(io), + addr, num_blocks, ftl_io_cmpl_cb, io); + } + + if (spdk_unlikely(rc)) { + if (rc == -ENOMEM) { + struct spdk_bdev *bdev; + struct spdk_io_channel *ch; + + if (ftl_addr_in_nvc(dev, addr)) { + bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); + ch = dev->nv_cache.cache_ioch; + } else { + bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); + ch = dev->base_ioch; + } + io->bdev_io_wait.bdev = bdev; + io->bdev_io_wait.cb_fn = _ftl_submit_read; + io->bdev_io_wait.cb_arg = io; + spdk_bdev_queue_io_wait(bdev, ch, &io->bdev_io_wait); + return; + } else { + ftl_abort(); + } + } + + ftl_io_inc_req(io); + ftl_io_advance(io, num_blocks); + } + + /* If we didn't have to read anything from the device, */ + /* complete the request right away */ + if (ftl_io_done(io)) { + ftl_io_complete(io); + } +} + void spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attrs) { @@ -126,6 +263,38 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at attrs->optimum_io_size = dev->xfer_size; } +static void +ftl_io_pin_cb(struct spdk_ftl_dev *dev, int status, struct ftl_l2p_pin_ctx *pin_ctx) +{ + struct ftl_io *io = pin_ctx->cb_ctx; + + if (spdk_unlikely(status != 0)) { + /* Retry on the internal L2P fault */ + io->status = -EAGAIN; + ftl_io_complete(io); + return; + } + + io->flags |= FTL_IO_PINNED; + ftl_submit_read(io); +} + +static void +ftl_io_pin(struct ftl_io *io) +{ + if (spdk_unlikely(io->flags & FTL_IO_PINNED)) { + /* + * The IO is in a retry path and it had been pinned already. + * Continue with further processing. + */ + ftl_l2p_pin_skip(io->dev, ftl_io_pin_cb, io, &io->l2p_pin_ctx); + } else { + /* First time when pinning the IO */ + ftl_l2p_pin(io->dev, io->lba, io->num_blocks, + ftl_io_pin_cb, io, &io->l2p_pin_ctx); + } +} + static void start_io(struct ftl_io *io) { @@ -141,8 +310,7 @@ start_io(struct ftl_io *io) switch (io->type) { case FTL_IO_READ: - io->status = -EOPNOTSUPP; - ftl_io_complete(io); + TAILQ_INSERT_TAIL(&dev->rd_sq, io, queue_entry); break; case FTL_IO_WRITE: TAILQ_INSERT_TAIL(&dev->wr_sq, io, queue_entry); @@ -201,6 +369,38 @@ spdk_ftl_writev(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_chan return queue_io(dev, io); } +int +spdk_ftl_readv(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch, + uint64_t lba, uint64_t lba_cnt, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg) +{ + int rc; + + if (iov_cnt == 0) { + return -EINVAL; + } + + if (lba_cnt == 0) { + return -EINVAL; + } + + if (lba_cnt != ftl_iovec_num_blocks(iov, iov_cnt)) { + FTL_ERRLOG(dev, "Invalid IO vector to handle, device %s, LBA %"PRIu64"\n", + dev->conf.name, lba); + return -EINVAL; + } + + if (!dev->initialized) { + return -EBUSY; + } + + rc = ftl_io_init(ch, io, lba, lba_cnt, iov, iov_cnt, cb_fn, cb_arg, FTL_IO_READ); + if (rc) { + return rc; + } + + return queue_io(dev, io); +} + #define FTL_IO_QUEUE_BATCH 16 int ftl_io_channel_poll(void *arg) @@ -245,6 +445,16 @@ ftl_process_io_queue(struct spdk_ftl_dev *dev) struct ftl_io_channel *ioch; struct ftl_io *io; + /* TODO: Try to figure out a mechanism to batch more requests at the same time, + * with keeping enough resources (pinned pages), between reads, writes and gc/compaction + */ + if (!TAILQ_EMPTY(&dev->rd_sq)) { + io = TAILQ_FIRST(&dev->rd_sq); + TAILQ_REMOVE(&dev->rd_sq, io, queue_entry); + assert(io->type == FTL_IO_READ); + ftl_io_pin(io); + } + if (!ftl_nv_cache_full(&dev->nv_cache) && !TAILQ_EMPTY(&dev->wr_sq)) { io = TAILQ_FIRST(&dev->wr_sq); TAILQ_REMOVE(&dev->wr_sq, io, queue_entry); diff --git a/lib/ftl/spdk_ftl.map b/lib/ftl/spdk_ftl.map index d9d4483b4..5bc082e93 100644 --- a/lib/ftl/spdk_ftl.map +++ b/lib/ftl/spdk_ftl.map @@ -13,6 +13,7 @@ spdk_ftl_conf_deinit; spdk_ftl_get_io_channel; spdk_ftl_io_size; + spdk_ftl_readv; spdk_ftl_writev; local: *; diff --git a/module/bdev/ftl/bdev_ftl.c b/module/bdev/ftl/bdev_ftl.c index 7bc537bb4..2bac7ec9a 100644 --- a/module/bdev/ftl/bdev_ftl.c +++ b/module/bdev/ftl/bdev_ftl.c @@ -105,6 +105,31 @@ bdev_ftl_cb(void *arg, int rc) spdk_bdev_io_complete(bdev_io, status); } +static void +bdev_ftl_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, + bool success) +{ + struct ftl_bdev *ftl_bdev; + int rc; + + ftl_bdev = bdev_io->bdev->ctxt; + + if (!success) { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + rc = spdk_ftl_readv(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, + ch, + bdev_io->u.bdev.offset_blocks, + bdev_io->u.bdev.num_blocks, + bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); + + if (spdk_unlikely(rc != 0)) { + spdk_bdev_io_complete(bdev_io, rc); + } +} + static int _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) { @@ -112,7 +137,8 @@ _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_i switch (bdev_io->type) { case SPDK_BDEV_IO_TYPE_READ: - spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + spdk_bdev_io_get_buf(bdev_io, bdev_ftl_get_buf_cb, + bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); return 0; case SPDK_BDEV_IO_TYPE_WRITE: diff --git a/test/ftl/config/fio/randr.fio b/test/ftl/config/fio/randr.fio new file mode 100644 index 000000000..f3f644476 --- /dev/null +++ b/test/ftl/config/fio/randr.fio @@ -0,0 +1,19 @@ +[global] +ioengine=spdk_bdev +spdk_json_conf=${FTL_JSON_CONF} +filename=${FTL_BDEV_NAME} +direct=1 +thread=1 +buffered=0 +size=100% +randrepeat=0 +time_based +norandommap + +[test] +stonewall +bs=4k +numjobs=4 +rw=randread +iodepth=128 +runtime=1200 diff --git a/test/ftl/config/fio/randw-verify-depth128.fio b/test/ftl/config/fio/randw-verify-depth128.fio index a932d1b36..9adee6cab 100644 --- a/test/ftl/config/fio/randw-verify-depth128.fio +++ b/test/ftl/config/fio/randw-verify-depth128.fio @@ -7,7 +7,7 @@ direct=1 iodepth=128 rw=randwrite verify=crc32c -do_verify=0 +do_verify=1 verify_dump=0 verify_state_save=0 verify_fatal=1 diff --git a/test/ftl/config/fio/randw-verify-j2.fio b/test/ftl/config/fio/randw-verify-j2.fio index fbbf284be..4610efa3f 100644 --- a/test/ftl/config/fio/randw-verify-j2.fio +++ b/test/ftl/config/fio/randw-verify-j2.fio @@ -7,7 +7,7 @@ direct=1 iodepth=128 rw=randwrite verify=crc32c -do_verify=0 +do_verify=1 verify_dump=0 verify_state_save=0 verify_backlog=5000 diff --git a/test/ftl/config/fio/randw-verify-qd2048-ext.fio b/test/ftl/config/fio/randw-verify-qd2048-ext.fio index f12e41532..4a59fead3 100644 --- a/test/ftl/config/fio/randw-verify-qd2048-ext.fio +++ b/test/ftl/config/fio/randw-verify-qd2048-ext.fio @@ -11,7 +11,7 @@ size=100% numjobs=1 rw=randwrite verify=crc32c -do_verify=0 +do_verify=1 verify_dump=0 verify_state_save=0 verify_fatal=1 diff --git a/test/ftl/config/fio/randw-verify.fio b/test/ftl/config/fio/randw-verify.fio index efb8432a1..edca6c618 100644 --- a/test/ftl/config/fio/randw-verify.fio +++ b/test/ftl/config/fio/randw-verify.fio @@ -8,7 +8,7 @@ iodepth=1 rw=randwrite size=256M verify=crc32c -do_verify=0 +do_verify=1 verify_dump=0 verify_state_save=0 verify_backlog=16 diff --git a/test/ftl/fio.sh b/test/ftl/fio.sh index 5d942d2ac..878cc0223 100755 --- a/test/ftl/fio.sh +++ b/test/ftl/fio.sh @@ -7,7 +7,7 @@ source $testdir/common.sh declare -A suite suite['basic']='randw-verify randw-verify-j2 randw-verify-depth128' -suite['extended']='randw-verify-qd2048-ext' +suite['extended']='randw-verify-qd2048-ext randr' rpc_py=$rootdir/scripts/rpc.py