From 1450c5470bd6eb80015bf44bc866cb0795311138 Mon Sep 17 00:00:00 2001 From: GangCao Date: Wed, 14 Dec 2022 02:51:26 -0500 Subject: [PATCH] lib/bdev: send back the eligible QoS IO to the original thread Fix issue: #2815 Change-Id: Ic1533b9ed055734a721be0fd7159754e5db1791b Signed-off-by: GangCao Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15917 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Shuhei Matsumoto --- lib/bdev/bdev.c | 32 +++++++++++++++++--------- test/unit/lib/bdev/mt/bdev.c/bdev_ut.c | 7 +++--- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c index f757fff04..72f710825 100644 --- a/lib/bdev/bdev.c +++ b/lib/bdev/bdev.c @@ -2176,6 +2176,15 @@ bdev_qos_queue_io(struct spdk_bdev_qos *qos, struct spdk_bdev_io *bdev_io) return false; } +static inline void +_bdev_io_do_submit(void *ctx) +{ + struct spdk_bdev_io *bdev_io = ctx; + struct spdk_bdev_channel *ch = bdev_io->internal.ch; + + bdev_io_do_submit(ch, bdev_io); +} + static int bdev_qos_io_submit(struct spdk_bdev_channel *ch, struct spdk_bdev_qos *qos) { @@ -2185,7 +2194,17 @@ bdev_qos_io_submit(struct spdk_bdev_channel *ch, struct spdk_bdev_qos *qos) TAILQ_FOREACH_SAFE(bdev_io, &qos->queued, internal.link, tmp) { if (!bdev_qos_queue_io(qos, bdev_io)) { TAILQ_REMOVE(&qos->queued, bdev_io, internal.link); - bdev_io_do_submit(ch, bdev_io); + + if (bdev_io->internal.io_submit_ch) { + /* Send back the IO to the original thread for the actual processing. */ + bdev_io->internal.ch = bdev_io->internal.io_submit_ch; + bdev_io->internal.io_submit_ch = NULL; + spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io), + _bdev_io_do_submit, bdev_io); + } else { + bdev_io_do_submit(ch, bdev_io); + } + submitted_ios++; } } @@ -6252,16 +6271,7 @@ bdev_io_complete(void *ctx) struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch; uint64_t tsc, tsc_diff; - if (spdk_unlikely(bdev_io->internal.in_submit_request || bdev_io->internal.io_submit_ch)) { - /* - * Send the completion to the thread that originally submitted the I/O, - * which may not be the current thread in the case of QoS. - */ - if (bdev_io->internal.io_submit_ch) { - bdev_io->internal.ch = bdev_io->internal.io_submit_ch; - bdev_io->internal.io_submit_ch = NULL; - } - + if (spdk_unlikely(bdev_io->internal.in_submit_request)) { /* * Defer completion to avoid potential infinite recursion if the * user's completion callback issues a new I/O. diff --git a/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c b/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c index 5610be842..3763e7edd 100644 --- a/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c +++ b/test/unit/lib/bdev/mt/bdev.c/bdev_ut.c @@ -930,12 +930,13 @@ basic_qos(void) CU_ASSERT(rc == 0); CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); poll_threads(); - /* Complete I/O on thread 1. This should not complete the I/O we submitted */ + /* Complete I/O on thread 0. This should not complete the I/O we submitted. */ + set_thread(0); stub_complete_io(g_bdev.io_target, 0); poll_threads(); CU_ASSERT(status == SPDK_BDEV_IO_STATUS_PENDING); - /* Now complete I/O on thread 0 */ - set_thread(0); + /* Now complete I/O on original thread 1. */ + set_thread(1); poll_threads(); stub_complete_io(g_bdev.io_target, 0); poll_threads();