diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 65554c632..d2b1d5f5e 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -2921,6 +2921,55 @@ bdev_io_submit(struct spdk_bdev_io *bdev_io) } } +static inline void +_bdev_io_copy_ext_opts(struct spdk_bdev_io *bdev_io, struct spdk_bdev_ext_io_opts *opts) +{ + struct spdk_bdev_ext_io_opts *opts_copy = &bdev_io->internal.ext_opts_copy; + + memcpy(opts_copy, opts, opts->size); + bdev_io->internal.ext_opts_copy.metadata = bdev_io->u.bdev.md_buf; + /* Save pointer to the copied ext_opts which will be used by bdev modules */ + bdev_io->u.bdev.ext_opts = opts_copy; +} + +static inline void +_bdev_io_ext_use_bounce_buffer(struct spdk_bdev_io *bdev_io) +{ + /* bdev doesn't support memory domains, thereby buffers in this IO request can't + * be accessed directly. It is needed to allocate buffers before issuing IO operation. + * For write operation we need to pull buffers from memory domain before submitting IO. + * Once read operation completes, we need to use memory_domain push functionality to + * update data in original memory domain IO buffer + * This IO request will go through a regular IO flow, so clear memory domains pointers in + * the copied ext_opts */ + bdev_io->internal.ext_opts_copy.memory_domain = NULL; + bdev_io->internal.ext_opts_copy.memory_domain_ctx = NULL; + _bdev_memory_domain_io_get_buf(bdev_io, _bdev_memory_domain_get_io_cb, + bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); +} + +static inline void +_bdev_io_submit_ext(struct spdk_bdev_desc *desc, struct spdk_bdev_io *bdev_io, + struct spdk_bdev_ext_io_opts *opts, bool copy_opts) +{ + if (opts) { + bool use_pull_push = opts->memory_domain && !desc->memory_domains_supported; + assert(opts->size <= sizeof(*opts)); + /* + * copy if size is smaller than opts struct to avoid having to check size + * on every access to bdev_io->u.bdev.ext_opts + */ + if (copy_opts || use_pull_push) { + _bdev_io_copy_ext_opts(bdev_io, opts); + if (use_pull_push) { + _bdev_io_ext_use_bounce_buffer(bdev_io); + return; + } + } + } + bdev_io_submit(bdev_io); +} + static void bdev_io_submit_reset(struct spdk_bdev_io *bdev_io) { @@ -4091,33 +4140,6 @@ _bdev_io_check_md_buf(const struct iovec *iovs, const void *md_buf) return _is_buf_allocated(iovs) == (md_buf != NULL); } -static inline void -_bdev_io_copy_ext_opts(struct spdk_bdev_io *bdev_io, struct spdk_bdev_ext_io_opts *opts) -{ - struct spdk_bdev_ext_io_opts *opts_copy = &bdev_io->internal.ext_opts_copy; - - memcpy(opts_copy, opts, opts->size); - bdev_io->internal.ext_opts_copy.metadata = bdev_io->u.bdev.md_buf; - /* Save pointer to the copied ext_opts which will be used by bdev modules */ - bdev_io->u.bdev.ext_opts = opts_copy; -} - -static inline void -_bdev_io_ext_use_bounce_buffer(struct spdk_bdev_io *bdev_io) -{ - /* bdev doesn't support memory domains, thereby buffers in this IO request can't - * be accessed directly. It is needed to allocate buffers before issuing IO operation. - * For write operation we need to pull buffers from memory domain before submitting IO. - * Once read operation completes, we need to use memory_domain push functionality to - * update data in original memory domain IO buffer - * This IO request will go through a regular IO flow, so clear memory domains pointers in - * the copied ext_opts */ - bdev_io->internal.ext_opts_copy.memory_domain = NULL; - bdev_io->internal.ext_opts_copy.memory_domain_ctx = NULL; - _bdev_memory_domain_io_get_buf(bdev_io, _bdev_memory_domain_get_io_cb, - bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); -} - static int bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks, @@ -4244,20 +4266,7 @@ bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *c bdev_io->internal.ext_opts = opts; bdev_io->u.bdev.ext_opts = opts; - if (opts) { - bool use_pull_push = opts->memory_domain && !desc->memory_domains_supported; - - assert(opts->size <= sizeof(*opts)); - if (copy_opts || use_pull_push) { - _bdev_io_copy_ext_opts(bdev_io, opts); - if (use_pull_push) { - _bdev_io_ext_use_bounce_buffer(bdev_io); - return 0; - } - } - } - - bdev_io_submit(bdev_io); + _bdev_io_submit_ext(desc, bdev_io, opts, copy_opts); return 0; } @@ -4289,6 +4298,15 @@ spdk_bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_chann num_blocks, cb, cb_arg, NULL, false); } +static inline bool +_bdev_io_check_opts(struct spdk_bdev_ext_io_opts *opts, struct iovec *iov) +{ + return opts->size > 0 && + opts->size <= sizeof(*opts) && + /* When memory domain is used, the user must provide data buffers */ + (!opts->memory_domain || (iov && iov[0].iov_base)); +} + int spdk_bdev_readv_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, struct iovec *iov, int iovcnt, @@ -4299,11 +4317,7 @@ spdk_bdev_readv_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel * void *md = NULL; if (opts) { - if (spdk_unlikely(!opts->size || opts->size > sizeof(struct spdk_bdev_ext_io_opts))) { - return -EINVAL; - } - if (spdk_unlikely(opts->memory_domain && !(iov && iov[0].iov_base))) { - /* When memory domain is used, the user must provide data buffers */ + if (spdk_unlikely(!_bdev_io_check_opts(opts, iov))) { return -EINVAL; } md = opts->metadata; @@ -4441,20 +4455,7 @@ bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel * bdev_io->internal.ext_opts = opts; bdev_io->u.bdev.ext_opts = opts; - if (opts) { - bool use_pull_push = opts->memory_domain && !desc->memory_domains_supported; - - assert(opts->size <= sizeof(*opts)); - if (copy_opts || use_pull_push) { - _bdev_io_copy_ext_opts(bdev_io, opts); - if (use_pull_push) { - _bdev_io_ext_use_bounce_buffer(bdev_io); - return 0; - } - } - } - - bdev_io_submit(bdev_io); + _bdev_io_submit_ext(desc, bdev_io, opts, copy_opts); return 0; } @@ -4513,11 +4514,7 @@ spdk_bdev_writev_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel void *md = NULL; if (opts) { - if (spdk_unlikely(!opts->size || opts->size > sizeof(struct spdk_bdev_ext_io_opts))) { - return -EINVAL; - } - if (spdk_unlikely(opts->memory_domain && !(iov && iov[0].iov_base))) { - /* When memory domain is used, the user must provide data buffers */ + if (spdk_unlikely(!_bdev_io_check_opts(opts, iov))) { return -EINVAL; } md = opts->metadata;