diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index 24f691cd3..4d590261b 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -1397,12 +1397,28 @@ _spdk_bdev_qos_set_ops(struct spdk_bdev_qos *qos) } } +static inline void +_spdk_bdev_io_do_submit(struct spdk_bdev_channel *bdev_ch, struct spdk_bdev_io *bdev_io) +{ + struct spdk_bdev *bdev = bdev_io->bdev; + struct spdk_io_channel *ch = bdev_ch->channel; + struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource; + + if (spdk_likely(TAILQ_EMPTY(&shared_resource->nomem_io))) { + bdev_ch->io_outstanding++; + shared_resource->io_outstanding++; + bdev_io->internal.in_submit_request = true; + bdev->fn_table->submit_request(ch, bdev_io); + bdev_io->internal.in_submit_request = false; + } else { + TAILQ_INSERT_TAIL(&shared_resource->nomem_io, bdev_io, internal.link); + } +} + static int _spdk_bdev_qos_io_submit(struct spdk_bdev_channel *ch, struct spdk_bdev_qos *qos) { struct spdk_bdev_io *bdev_io = NULL, *tmp = NULL; - struct spdk_bdev *bdev = ch->bdev; - struct spdk_bdev_shared_resource *shared_resource = ch->shared_resource; int i, submitted_ios = 0; TAILQ_FOREACH_SAFE(bdev_io, &qos->queued, internal.link, tmp) { @@ -1427,11 +1443,7 @@ _spdk_bdev_qos_io_submit(struct spdk_bdev_channel *ch, struct spdk_bdev_qos *qos } TAILQ_REMOVE(&qos->queued, bdev_io, internal.link); - ch->io_outstanding++; - shared_resource->io_outstanding++; - bdev_io->internal.in_submit_request = true; - bdev->fn_table->submit_request(ch->channel, bdev_io); - bdev_io->internal.in_submit_request = false; + _spdk_bdev_io_do_submit(ch, bdev_io); submitted_ios++; } @@ -1682,25 +1694,22 @@ _spdk_bdev_io_submit(void *ctx) struct spdk_bdev_io *bdev_io = ctx; struct spdk_bdev *bdev = bdev_io->bdev; struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch; - struct spdk_io_channel *ch = bdev_ch->channel; struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource; uint64_t tsc; tsc = spdk_get_ticks(); bdev_io->internal.submit_tsc = tsc; spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_START, 0, 0, (uintptr_t)bdev_io, bdev_io->type); + + if (spdk_likely(bdev_ch->flags == 0)) { + _spdk_bdev_io_do_submit(bdev_ch, bdev_io); + return; + } + bdev_ch->io_outstanding++; shared_resource->io_outstanding++; bdev_io->internal.in_submit_request = true; - if (spdk_likely(bdev_ch->flags == 0)) { - if (spdk_likely(TAILQ_EMPTY(&shared_resource->nomem_io))) { - bdev->fn_table->submit_request(ch, bdev_io); - } else { - bdev_ch->io_outstanding--; - shared_resource->io_outstanding--; - TAILQ_INSERT_TAIL(&shared_resource->nomem_io, bdev_io, internal.link); - } - } else if (bdev_ch->flags & BDEV_CH_RESET_IN_PROGRESS) { + if (bdev_ch->flags & BDEV_CH_RESET_IN_PROGRESS) { spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); } else if (bdev_ch->flags & BDEV_CH_QOS_ENABLED) { bdev_ch->io_outstanding--;