test/scheduler: Add basic tests for verifying interrupt mode

Also, move all the functions wrapping the scheduler app into
common.sh.

Signed-off-by: Michal Berger <michalx.berger@intel.com>
Change-Id: I63c1002f867128851cff04a035c6c51c50e10383
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6053
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

test/scheduler/common.sh

@@ -4,6 +4,11 @@ declare -r sysfs_system=/sys/devices/system
 declare -r sysfs_cpu=$sysfs_system/cpu
 declare -r sysfs_node=$sysfs_system/node
 
+export PYTHONPATH=$rootdir/test/event/scheduler
+
+declare -r scheduler=$rootdir/test/event/scheduler/scheduler
+declare -r plugin=scheduler_plugin
+
 fold_list_onto_array() {
 	local array=$1
 	local elem
@@ -364,3 +369,29 @@ _get_thread_stats() {
 		thread_map[thread]=$(jq -r "select(.id == $thread) | .name" <<< "$stats")
 	done
 }
+
+get_cpu_stat() {
+	local cpu_idx=$1
+	local stat=$2 stats
+
+	while read -r cpu stats; do
+		[[ $cpu == "cpu$cpu_idx" ]] && stats=($stats)
+	done < /proc/stat
+
+	case "$stat" in
+		idle) echo "${stats[3]}" ;;
+		*) ;;
+	esac
+}
+
+create_thread() {
+	"$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_create "$@"
+}
+
+destroy_thread() {
+	"$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_delete "$@"
+}
+
+active_thread() {
+	"$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_set_active "$@"
+}

test/scheduler/interrupt.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../../")
+
+source "$rootdir/test/common/autotest_common.sh"
+source "$testdir/common.sh"
+
+trap 'killprocess "$spdk_pid"' EXIT
+
+declare -a cpus=()
+declare -a cpus_to_collect=()
+
+fold_list_onto_array cpus $(parse_cpu_list <(echo "$spdk_cpus_csv"))
+# Normalize the indexes
+cpus=("${cpus[@]}")
+
+collect_cpu_stat() {
+	local list=$1
+	local stat=$2
+
+	for cpu in "${cpus_to_collect[@]}"; do
+		eval "${list}[cpu]=\$(get_cpu_stat $cpu $stat)"
+	done
+}
+
+collect_cpu_idle() {
+	xtrace_disable
+
+	local sample_time=${1:-5} samples=0
+	local cpu bool inc user_hz
+
+	# idle scales to USER_HZ so we use that in order to determine the expected
+	# value it should have been increased to (more or less).
+	user_hz=100
+	# Expected increase of the idle stat
+	inc=$((user_hz * sample_time))
+
+	bool[0]="not" bool[1]="is"
+
+	init_idle_samples=() idle_samples=() is_idle=()
+
+	collect_cpu_stat init_idle_samples idle
+
+	printf 'Collecting cpu idle stats (cpus: %s) for %u seconds...\n' \
+		"${cpus_to_collect[*]}" "$sample_time"
+
+	while ((++samples <= sample_time)) && sleep 1s; do
+		collect_cpu_stat idle_samples idle
+	done
+
+	for cpu in "${!idle_samples[@]}"; do
+		# We start to collect after the spdk app is initialized hence if the interrupt
+		# mode is not working as expected, the idle time of given cpu will not have a
+		# chance to increase. If it does work correctly, then it should change even for
+		# a fraction, depending on how much time we spent on collecting this data.
+		# If idle time is over 70% of expected increase then we consider this cpu as
+		# idle. This is done in order to take into consideration time window the app
+		# needs to actually spin up|down the cpu. It's also taken for granted that
+		# there is no extra load on the target cpus which may be coming from other
+		# processes.
+		if ((idle_samples[cpu] > init_idle_samples[cpu] + (inc * 70 / 100))); then
+			is_idle[cpu]=1
+		else
+			is_idle[cpu]=0
+		fi
+		printf 'cpu%u %s idle (%u %u)\n' \
+			"$cpu" "${bool[is_idle[cpu]]}" "${init_idle_samples[cpu]}" "${idle_samples[cpu]}"
+	done
+
+	xtrace_restore
+}
+
+interrupt() {
+	local busy_cpus
+	local cpu thread
+
+	local reactor_framework
+
+	cpus_to_collect=("${cpus[@]}")
+	collect_cpu_idle
+
+	# Standard scenario - spdk app is idle, all cpus, except the main cpu, should be
+	# switched into interrupt mode. main cpu should remain busy, all remaining cpus
+	# should become idle.
+	for cpu in "${!is_idle[@]}"; do
+		if ((cpu == spdk_main_core)); then
+			((is_idle[cpu] == 0)) # main cpu must not be idle
+		fi
+		if ((cpu != spdk_main_core)); then
+			((is_idle[cpu] == 1)) # all cpus except the main cpu must be idle
+		fi
+	done
+
+	# While cpus are still idle, verify that each cpu, except the main cpu, has no threads assigned
+	reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]')
+	for cpu in "${cpus[@]:1}"; do
+		[[ -z $(jq -r "select(.lcore == $cpu) | .lw_threads[].id" <<< "$reactor_framework") ]]
+	done
+
+	# select 3 cpus except the main one
+	busy_cpus=("${cpus[@]:1:3}") threads=()
+
+	# Create busy thread on each of the selected cpus, then verify if given cpu has become busy
+	# and that newly created thread was assigned to a proper lcore.
+	for cpu in "${busy_cpus[@]}"; do
+		threads[cpu]=$(create_thread -n "thread$cpu" -m "$(mask_cpus "$cpu")" -a 100) cpus_to_collect=("$cpu")
+		collect_cpu_idle
+		((is_idle[cpu] == 0))
+		reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]')
+		[[ -n $(jq -r "select(.lcore == $cpu) | .lw_threads[] | select(.name == \"thread$cpu\")" <<< "$reactor_framework") ]]
+	done
+
+	# Make all the threads idle and verify if their cpus have become idle as well and if they were
+	# moved away out of their lcores.
+	for cpu in "${!threads[@]}"; do
+		active_thread "${threads[cpu]}" 0
+		cpus_to_collect=("$cpu")
+		# Give some extra time for the cpu to spin down
+		collect_cpu_idle 10
+		((is_idle[cpu] == 1))
+		reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]')
+		[[ -z $(jq -r "select(.lcore == $cpu) | .lw_threads[].id" <<< "$reactor_framework") ]]
+	done
+
+	for cpu in "${!threads[@]}"; do
+		destroy_thread "${threads[cpu]}"
+	done
+}
+
+exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpusmask" --main-core "$spdk_main_core"
+
+interrupt

test/scheduler/load_balancing.sh

@@ -8,27 +8,10 @@ source "$testdir/common.sh"
 
 trap 'killprocess "$spdk_pid"' EXIT
 
-export PYTHONPATH=$rootdir/test/event/scheduler
-
-declare -r scheduler=$rootdir/test/event/scheduler/scheduler
-declare -r plugin=scheduler_plugin
-
 fold_list_onto_array cpus $(parse_cpu_list <(echo "$spdk_cpus_csv"))
 # Normalize the indexes
 cpus=("${cpus[@]}")
 
-create_thread() {
-	"$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_create "$@"
-}
-
-destroy_thread() {
-	"$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_delete "$@"
-}
-
-active_thread() {
-	"$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_set_active "$@"
-}
-
 busy() {
 	local selected_cpus cpu
 	local reactor_framework

test/scheduler/scheduler.sh

@@ -11,3 +11,4 @@ source "$testdir/isolate_cores.sh"
 run_test "idle" "$testdir/idle.sh"
 run_test "load_balancing" "$testdir/load_balancing.sh"
 run_test "dpdk_governor" "$testdir/governor.sh"
+run_test "interrupt_mode" "$testdir/interrupt.sh"