From 1d29386066dd2b09c684777843bca6b25428bed8 Mon Sep 17 00:00:00 2001 From: Konrad Sztyber Date: Thu, 9 May 2019 16:04:26 +0200 Subject: [PATCH] lib/ftl: stop splitting internal IO data buffers Some of the internal IOs split the contiguous data buffer into multiple iovecs to separate the requests. It's unnecessary and can be replaced by checking the position within the data buffer. Change-Id: I9255ea0072fee6c4e6a7dca21e4008780bc45610 Signed-off-by: Konrad Sztyber Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/455519 Tested-by: SPDK CI Jenkins Reviewed-by: Ben Walker Reviewed-by: Wojciech Malikowski Reviewed-by: Shuhei Matsumoto Reviewed-by: Darek Stojaczyk --- lib/ftl/ftl_band.c | 14 ++--- lib/ftl/ftl_core.c | 22 +++---- lib/ftl/ftl_io.c | 60 ++++---------------- lib/ftl/ftl_io.h | 9 +-- lib/ftl/ftl_reloc.c | 3 +- test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c | 2 +- 6 files changed, 31 insertions(+), 79 deletions(-) diff --git a/lib/ftl/ftl_band.c b/lib/ftl/ftl_band.c index c6b7a0af0..d56a01440 100644 --- a/lib/ftl/ftl_band.c +++ b/lib/ftl/ftl_band.c @@ -734,8 +734,7 @@ ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_md *md, void *data, str .size = sizeof(*io), .flags = FTL_IO_MD | FTL_IO_PPA_MODE, .type = FTL_IO_READ, - .iov_cnt = 1, - .req_size = lbk_cnt, + .lbk_cnt = lbk_cnt, .fn = ftl_read_md_cb, .data = data, }; @@ -756,7 +755,7 @@ ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_md *md, void *data, str static struct ftl_io * ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, - void *data, size_t req_cnt, spdk_ftl_fn cb) + void *data, size_t lbk_cnt, spdk_ftl_fn cb) { struct ftl_io_init_opts opts = { .dev = dev, @@ -766,8 +765,7 @@ ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, .size = sizeof(struct ftl_io), .flags = FTL_IO_MD | FTL_IO_PPA_MODE, .type = FTL_IO_WRITE, - .iov_cnt = req_cnt, - .req_size = dev->xfer_size, + .lbk_cnt = lbk_cnt, .fn = cb, .data = data, .md = NULL, @@ -784,8 +782,7 @@ ftl_band_write_md(struct ftl_band *band, void *data, size_t lbk_cnt, struct ftl_io *io; int rc; - io = ftl_io_init_md_write(dev, band, data, - spdk_divide_round_up(lbk_cnt, dev->xfer_size), cb); + io = ftl_io_init_md_write(dev, band, data, lbk_cnt, cb); if (!io) { return -ENOMEM; } @@ -842,8 +839,7 @@ ftl_band_read_md(struct ftl_band *band, struct ftl_md *md, void *data, size_t lb return -ENOENT; } - io = ftl_io_init_md_read(dev, md, data, start_ppa, band, lbk_cnt, - unpack_fn, cb); + io = ftl_io_init_md_read(dev, md, data, start_ppa, band, lbk_cnt, unpack_fn, cb); if (!io) { return -ENOMEM; } diff --git a/lib/ftl/ftl_core.c b/lib/ftl/ftl_core.c index 37b26b6c5..6311755cc 100644 --- a/lib/ftl/ftl_core.c +++ b/lib/ftl/ftl_core.c @@ -923,9 +923,8 @@ ftl_alloc_io_nv_cache(struct ftl_io *parent, size_t num_lbks) struct ftl_io_init_opts opts = { .dev = parent->dev, .parent = parent, - .iov_cnt = 1, .data = ftl_io_iovec_addr(parent), - .req_size = num_lbks, + .lbk_cnt = num_lbks, .flags = FTL_IO_CACHE, }; @@ -1009,7 +1008,7 @@ _ftl_write_nv_cache(void *ctx) /* Shrink the IO if there isn't enough room in the cache to fill the whole iovec */ if (spdk_unlikely(num_lbks != ftl_io_iovec_len_left(io))) { - ftl_io_shrink_iovec(child, ftl_io_iovec_addr(child), 1, num_lbks); + ftl_io_shrink_iovec(child, num_lbks); } ftl_submit_nv_cache(child); @@ -1171,8 +1170,7 @@ ftl_io_init_child_write(struct ftl_io *parent, struct ftl_ppa ppa, .size = sizeof(struct ftl_io), .flags = 0, .type = FTL_IO_WRITE, - .iov_cnt = 1, - .req_size = dev->xfer_size, + .lbk_cnt = dev->xfer_size, .fn = cb, .data = data, .md = md, @@ -1203,11 +1201,10 @@ ftl_submit_child_write(struct ftl_wptr *wptr, struct ftl_io *io, int lbk_cnt) { struct spdk_ftl_dev *dev = io->dev; struct ftl_io *child; - struct iovec *iov = ftl_io_iovec(io); int rc; /* Split IO to child requests and release chunk immediately after child is completed */ - child = ftl_io_init_child_write(io, wptr->ppa, iov[io->iov_pos].iov_base, + child = ftl_io_init_child_write(io, wptr->ppa, ftl_io_iovec_addr(io), ftl_io_get_md(io), ftl_io_child_write_cb); if (!child) { return -EAGAIN; @@ -1238,12 +1235,11 @@ ftl_submit_write(struct ftl_wptr *wptr, struct ftl_io *io) struct spdk_ftl_dev *dev = io->dev; struct iovec *iov = ftl_io_iovec(io); int rc = 0; - size_t lbk_cnt; + + assert(io->lbk_cnt % dev->xfer_size == 0); while (io->iov_pos < io->iov_cnt) { - lbk_cnt = iov[io->iov_pos].iov_len / PAGE_SIZE; assert(iov[io->iov_pos].iov_len > 0); - assert(lbk_cnt == dev->xfer_size); /* There are no guarantees of the order of completion of NVMe IO submission queue */ /* so wait until chunk is not busy before submitting another write */ @@ -1253,7 +1249,7 @@ ftl_submit_write(struct ftl_wptr *wptr, struct ftl_io *io) break; } - rc = ftl_submit_child_write(wptr, io, lbk_cnt); + rc = ftl_submit_child_write(wptr, io, dev->xfer_size); if (rc == -EAGAIN) { wptr->current_io = io; @@ -1263,8 +1259,8 @@ ftl_submit_write(struct ftl_wptr *wptr, struct ftl_io *io) break; } - ftl_trace_submission(dev, io, wptr->ppa, lbk_cnt); - ftl_wptr_advance(wptr, lbk_cnt); + ftl_trace_submission(dev, io, wptr->ppa, dev->xfer_size); + ftl_wptr_advance(wptr, dev->xfer_size); } if (ftl_io_done(io)) { diff --git a/lib/ftl/ftl_io.c b/lib/ftl/ftl_io.c index 4d45eddf7..570f6ea9d 100644 --- a/lib/ftl/ftl_io.c +++ b/lib/ftl/ftl_io.c @@ -160,49 +160,24 @@ ftl_io_iovec_len_left(struct ftl_io *io) } static void -_ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t iov_cnt, size_t req_size) +ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t lbk_cnt) { - struct iovec *iov; - size_t i; - io->iov_pos = 0; - io->iov_cnt = iov_cnt; - io->lbk_cnt = iov_cnt * req_size; + io->lbk_cnt = lbk_cnt; + io->iov_cnt = 1; - iov = ftl_io_iovec(io); - for (i = 0; i < iov_cnt; ++i) { - iov[i].iov_base = (char *)buf + i * req_size * PAGE_SIZE; - iov[i].iov_len = req_size * PAGE_SIZE; - } -} - -static int -ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t iov_cnt, size_t req_size) -{ - if (iov_cnt > 1) { - io->iov.vector = calloc(iov_cnt, sizeof(struct iovec)); - if (!io->iov.vector) { - return -ENOMEM; - } - } - - _ftl_io_init_iovec(io, buf, iov_cnt, req_size); - - return 0; + io->iov.single.iov_base = buf; + io->iov.single.iov_len = lbk_cnt * PAGE_SIZE; } void -ftl_io_shrink_iovec(struct ftl_io *io, char *buf, size_t iov_cnt, size_t req_size) +ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt) { - assert(io->iov_cnt >= iov_cnt); - assert(io->lbk_cnt >= iov_cnt * req_size); + assert(io->iov_cnt == 1); + assert(io->lbk_cnt >= lbk_cnt); assert(io->pos == 0 && io->iov_pos == 0 && io->iov_off == 0); - if (iov_cnt == 1 && io->iov_cnt > 1) { - free(io->iov.vector); - } - - _ftl_io_init_iovec(io, buf, iov_cnt, req_size); + ftl_io_init_iovec(io, ftl_io_iovec_addr(io), lbk_cnt); } static void @@ -244,12 +219,7 @@ ftl_io_init_internal(const struct ftl_io_init_opts *opts) io->band = opts->band; io->md = opts->md; - if (ftl_io_init_iovec(io, opts->data, opts->iov_cnt, opts->req_size)) { - if (!opts->io) { - ftl_io_free(io); - } - return NULL; - } + ftl_io_init_iovec(io, opts->data, opts->lbk_cnt); return io; } @@ -266,8 +236,7 @@ ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band, .size = sizeof(struct ftl_io), .flags = 0, .type = FTL_IO_WRITE, - .iov_cnt = 1, - .req_size = dev->xfer_size, + .lbk_cnt = dev->xfer_size, .fn = cb, .data = ftl_rwb_batch_get_data(batch), .md = ftl_rwb_batch_get_md(batch), @@ -288,8 +257,7 @@ ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb) .size = sizeof(struct ftl_io), .flags = FTL_IO_PPA_MODE, .type = FTL_IO_ERASE, - .iov_cnt = 0, - .req_size = 1, + .lbk_cnt = 1, .fn = cb, .data = NULL, .md = NULL, @@ -336,10 +304,6 @@ _ftl_io_free(struct ftl_io *io) assert(LIST_EMPTY(&io->children)); - if ((io->flags & FTL_IO_INTERNAL) && io->iov_cnt > 1) { - free(io->iov.vector); - } - if (pthread_spin_destroy(&io->lock)) { SPDK_ERRLOG("pthread_spin_destroy failed\n"); } diff --git a/lib/ftl/ftl_io.h b/lib/ftl/ftl_io.h index bd32d7c41..0381acf47 100644 --- a/lib/ftl/ftl_io.h +++ b/lib/ftl/ftl_io.h @@ -98,17 +98,14 @@ struct ftl_io_init_opts { /* IO type */ enum ftl_io_type type; - /* Number of split requests */ - size_t iov_cnt; - /* RWB entry */ struct ftl_rwb_batch *rwb_batch; /* Band to which the IO is directed */ struct ftl_band *band; - /* Request size */ - size_t req_size; + /* Number of logical blocks */ + size_t lbk_cnt; /* Data */ void *data; @@ -285,7 +282,7 @@ void ftl_io_user_init(struct spdk_ftl_dev *dev, struct ftl_io *io, uint64_t lba, spdk_ftl_fn fn, void *cb_arg, int type); void *ftl_io_get_md(const struct ftl_io *io); void ftl_io_complete(struct ftl_io *io); -void ftl_io_shrink_iovec(struct ftl_io *io, char *buf, size_t iov_cnt, size_t req_size); +void ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt); void ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status); void ftl_io_reset(struct ftl_io *io); diff --git a/lib/ftl/ftl_reloc.c b/lib/ftl/ftl_reloc.c index 5318c31f6..09c29aff4 100644 --- a/lib/ftl/ftl_reloc.c +++ b/lib/ftl/ftl_reloc.c @@ -443,8 +443,7 @@ ftl_reloc_io_init(struct ftl_band_reloc *breloc, struct ftl_io *io, .size = sizeof(*io), .flags = FTL_IO_KEEP_ALIVE | FTL_IO_INTERNAL | FTL_IO_PPA_MODE, .type = FTL_IO_READ, - .iov_cnt = 1, - .req_size = num_lbks, + .lbk_cnt = num_lbks, .fn = ftl_reloc_read_cb, }; diff --git a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c b/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c index 1079a64b7..825a32b2b 100644 --- a/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c +++ b/test/unit/lib/ftl/ftl_reloc.c/ftl_reloc_ut.c @@ -155,7 +155,7 @@ ftl_io_init_internal(const struct ftl_io_init_opts *opts) io->flags = opts->flags; io->cb.fn = opts->fn; io->cb.ctx = io; - io->lbk_cnt = opts->req_size; + io->lbk_cnt = opts->lbk_cnt; io->iov.single.iov_base = opts->data; return io; }