test/scheduler: Add test for verifying idle state

This also introduces scheduler tests in the autotest pool.

Signed-off-by: Michal Berger <michalx.berger@intel.com>
Change-Id: I892a2374e21dab0e349c655dcafaa4cc2e45fdc1
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5741
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

autotest.sh

@@ -320,6 +320,10 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 			run_test "blockdev_crypto_qat" ./test/bdev/blockdev.sh "crypto_qat"
 		fi
 	fi
+
+	if [[ $SPDK_TEST_SCHEDULER -eq 1 ]]; then
+		run_test "scheduler" ./test/scheduler/scheduler.sh
+	fi
 fi
 
 timing_enter cleanup

test/common/autotest_common.sh

@@ -133,6 +133,8 @@ export SPDK_TEST_RAID5
 export SPDK_TEST_URING
 : ${SPDK_TEST_USE_IGB_UIO:=0}
 export SPDK_TEST_USE_IGB_UIO
+: ${SPDK_TEST_SCHEDULER:=0}
+export SPDK_TEST_SCHEDULER
 
 export DPDK_LIB_DIR="${SPDK_RUN_EXTERNAL_DPDK:-$rootdir/dpdk/build}/lib"
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SPDK_LIB_DIR:$DPDK_LIB_DIR

test/scheduler/common.sh

@@ -324,3 +324,33 @@ set_cpufreq() {
 			;;
 	esac
 }
+
+exec_under_dynamic_scheduler() {
+	"$@" --wait-for-rpc &
+	spdk_pid=$!
+	# Give some time for the app to init itself
+	waitforlisten "$spdk_pid"
+	"$rootdir/scripts/rpc.py" framework_set_scheduler dynamic
+	"$rootdir/scripts/rpc.py" framework_start_init
+}
+
+get_thread_stats() {
+	xtrace_disable
+	_get_thread_stats busy idle
+	xtrace_restore
+}
+
+_get_thread_stats() {
+	local list_busy=$1
+	local list_idle=$2
+	local thread threads stats
+
+	stats=$(rpc_cmd thread_get_stats | jq -r '.threads[]')
+	threads=($(jq -r '.id' <<< "$stats"))
+
+	for thread in "${threads[@]}"; do
+		eval "${list_busy}[$thread]=\$(jq -r \"select(.id == $thread) | .busy\" <<< \$stats)"
+		eval "${list_idle}[$thread]=\$(jq -r \"select(.id == $thread) | .idle\" <<< \$stats)"
+		thread_map[thread]=$(jq -r "select(.id == $thread) | .name" <<< "$stats")
+	done
+}

test/scheduler/idle.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../../")
+
+source "$rootdir/test/common/autotest_common.sh"
+source "$testdir/common.sh"
+
+trap 'killprocess "$spdk_pid"' EXIT
+
+thread_stats() {
+	local thread
+	local busy_threads=0
+
+	get_thread_stats
+
+	# Simply verify if threads stay idle
+	for thread in "${!thread_map[@]}"; do
+		if ((idle[thread] < busy[thread])); then
+			printf 'Waiting for %s to become idle\n' "${thread_map[thread]}"
+			((++busy_threads))
+		elif ((idle[thread] > busy[thread])); then
+			printf '%s is idle\n' "${thread_map[thread]}"
+		fi
+	done
+
+	((busy_threads == 0))
+}
+
+idle() {
+	local reactor_framework
+	local reactors thread
+	local cpusmask thread_cpumask
+	local threads
+
+	exec_under_dynamic_scheduler "${SPDK_APP[@]}" -m "$spdk_cpusmask" --main-core "$spdk_main_core"
+
+	# The expectation here is that when SPDK app is idle the following is true:
+	# - all threads are assigned to main lcore
+	# - threads are not being moved between lcores
+	# - each thread has a mask pinned to a single cpu
+
+	local all_set
+
+	xtrace_disable
+	while ((samples++ < 5)); do
+		all_set=0 cpusmask=0
+		reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]')
+		threads=($(
+			jq -r "select(.lcore == $spdk_main_core) | .lw_threads[].name" <<< "$reactor_framework"
+		))
+
+		for thread in "${threads[@]}"; do
+			thread_cpumask=0x$(jq -r "select(.lcore == $spdk_main_core) | .lw_threads[] | select(.name == \"$thread\") | .cpumask" <<< "$reactor_framework")
+			((cpusmask |= thread_cpumask))
+		done
+
+		printf 'SPDK cpumask: %x Threads cpumask: %x\n' "$spdk_cpusmask" "$cpusmask"
+		thread_stats && ((cpusmask == spdk_cpusmask)) && all_set=1
+	done
+
+	((all_set == 1))
+	xtrace_restore
+}
+
+idle

test/scheduler/scheduler.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../../")
+
+source "$rootdir/test/common/autotest_common.sh"
+source "$testdir/isolate_cores.sh"
+
+"$rootdir/scripts/setup.sh"
+
+run_test "idle" "$testdir/idle.sh"