From 049d587e50c836a169bfc537f27b1c804530c405 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Fri, 22 Jan 2021 17:30:42 +0100 Subject: [PATCH] test/scheduler: Add basic tests for verifying interrupt mode Also, move all the functions wrapping the scheduler app into common.sh. Signed-off-by: Michal Berger Change-Id: I63c1002f867128851cff04a035c6c51c50e10383 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6053 Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins --- test/scheduler/common.sh | 31 +++++++ test/scheduler/interrupt.sh | 133 +++++++++++++++++++++++++++++++ test/scheduler/load_balancing.sh | 17 ---- test/scheduler/scheduler.sh | 1 + 4 files changed, 165 insertions(+), 17 deletions(-) create mode 100755 test/scheduler/interrupt.sh diff --git a/test/scheduler/common.sh b/test/scheduler/common.sh index cfcd65e69..bd712f415 100644 --- a/test/scheduler/common.sh +++ b/test/scheduler/common.sh @@ -4,6 +4,11 @@ declare -r sysfs_system=/sys/devices/system declare -r sysfs_cpu=$sysfs_system/cpu declare -r sysfs_node=$sysfs_system/node +export PYTHONPATH=$rootdir/test/event/scheduler + +declare -r scheduler=$rootdir/test/event/scheduler/scheduler +declare -r plugin=scheduler_plugin + fold_list_onto_array() { local array=$1 local elem @@ -364,3 +369,29 @@ _get_thread_stats() { thread_map[thread]=$(jq -r "select(.id == $thread) | .name" <<< "$stats") done } + +get_cpu_stat() { + local cpu_idx=$1 + local stat=$2 stats + + while read -r cpu stats; do + [[ $cpu == "cpu$cpu_idx" ]] && stats=($stats) + done < /proc/stat + + case "$stat" in + idle) echo "${stats[3]}" ;; + *) ;; + esac +} + +create_thread() { + "$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_create "$@" +} + +destroy_thread() { + "$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_delete "$@" +} + +active_thread() { + "$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_set_active "$@" +} diff --git a/test/scheduler/interrupt.sh b/test/scheduler/interrupt.sh new file mode 100755 index 000000000..a6744fb95 --- /dev/null +++ b/test/scheduler/interrupt.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f "$(dirname "$0")") +rootdir=$(readlink -f "$testdir/../../") + +source "$rootdir/test/common/autotest_common.sh" +source "$testdir/common.sh" + +trap 'killprocess "$spdk_pid"' EXIT + +declare -a cpus=() +declare -a cpus_to_collect=() + +fold_list_onto_array cpus $(parse_cpu_list <(echo "$spdk_cpus_csv")) +# Normalize the indexes +cpus=("${cpus[@]}") + +collect_cpu_stat() { + local list=$1 + local stat=$2 + + for cpu in "${cpus_to_collect[@]}"; do + eval "${list}[cpu]=\$(get_cpu_stat $cpu $stat)" + done +} + +collect_cpu_idle() { + xtrace_disable + + local sample_time=${1:-5} samples=0 + local cpu bool inc user_hz + + # idle scales to USER_HZ so we use that in order to determine the expected + # value it should have been increased to (more or less). + user_hz=100 + # Expected increase of the idle stat + inc=$((user_hz * sample_time)) + + bool[0]="not" bool[1]="is" + + init_idle_samples=() idle_samples=() is_idle=() + + collect_cpu_stat init_idle_samples idle + + printf 'Collecting cpu idle stats (cpus: %s) for %u seconds...\n' \ + "${cpus_to_collect[*]}" "$sample_time" + + while ((++samples <= sample_time)) && sleep 1s; do + collect_cpu_stat idle_samples idle + done + + for cpu in "${!idle_samples[@]}"; do + # We start to collect after the spdk app is initialized hence if the interrupt + # mode is not working as expected, the idle time of given cpu will not have a + # chance to increase. If it does work correctly, then it should change even for + # a fraction, depending on how much time we spent on collecting this data. + # If idle time is over 70% of expected increase then we consider this cpu as + # idle. This is done in order to take into consideration time window the app + # needs to actually spin up|down the cpu. It's also taken for granted that + # there is no extra load on the target cpus which may be coming from other + # processes. + if ((idle_samples[cpu] > init_idle_samples[cpu] + (inc * 70 / 100))); then + is_idle[cpu]=1 + else + is_idle[cpu]=0 + fi + printf 'cpu%u %s idle (%u %u)\n' \ + "$cpu" "${bool[is_idle[cpu]]}" "${init_idle_samples[cpu]}" "${idle_samples[cpu]}" + done + + xtrace_restore +} + +interrupt() { + local busy_cpus + local cpu thread + + local reactor_framework + + cpus_to_collect=("${cpus[@]}") + collect_cpu_idle + + # Standard scenario - spdk app is idle, all cpus, except the main cpu, should be + # switched into interrupt mode. main cpu should remain busy, all remaining cpus + # should become idle. + for cpu in "${!is_idle[@]}"; do + if ((cpu == spdk_main_core)); then + ((is_idle[cpu] == 0)) # main cpu must not be idle + fi + if ((cpu != spdk_main_core)); then + ((is_idle[cpu] == 1)) # all cpus except the main cpu must be idle + fi + done + + # While cpus are still idle, verify that each cpu, except the main cpu, has no threads assigned + reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') + for cpu in "${cpus[@]:1}"; do + [[ -z $(jq -r "select(.lcore == $cpu) | .lw_threads[].id" <<< "$reactor_framework") ]] + done + + # select 3 cpus except the main one + busy_cpus=("${cpus[@]:1:3}") threads=() + + # Create busy thread on each of the selected cpus, then verify if given cpu has become busy + # and that newly created thread was assigned to a proper lcore. + for cpu in "${busy_cpus[@]}"; do + threads[cpu]=$(create_thread -n "thread$cpu" -m "$(mask_cpus "$cpu")" -a 100) cpus_to_collect=("$cpu") + collect_cpu_idle + ((is_idle[cpu] == 0)) + reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') + [[ -n $(jq -r "select(.lcore == $cpu) | .lw_threads[] | select(.name == \"thread$cpu\")" <<< "$reactor_framework") ]] + done + + # Make all the threads idle and verify if their cpus have become idle as well and if they were + # moved away out of their lcores. + for cpu in "${!threads[@]}"; do + active_thread "${threads[cpu]}" 0 + cpus_to_collect=("$cpu") + # Give some extra time for the cpu to spin down + collect_cpu_idle 10 + ((is_idle[cpu] == 1)) + reactor_framework=$(rpc_cmd framework_get_reactors | jq -r '.reactors[]') + [[ -z $(jq -r "select(.lcore == $cpu) | .lw_threads[].id" <<< "$reactor_framework") ]] + done + + for cpu in "${!threads[@]}"; do + destroy_thread "${threads[cpu]}" + done +} + +exec_under_dynamic_scheduler "$scheduler" -m "$spdk_cpusmask" --main-core "$spdk_main_core" + +interrupt diff --git a/test/scheduler/load_balancing.sh b/test/scheduler/load_balancing.sh index ebd86d1c9..f02e8044f 100755 --- a/test/scheduler/load_balancing.sh +++ b/test/scheduler/load_balancing.sh @@ -8,27 +8,10 @@ source "$testdir/common.sh" trap 'killprocess "$spdk_pid"' EXIT -export PYTHONPATH=$rootdir/test/event/scheduler - -declare -r scheduler=$rootdir/test/event/scheduler/scheduler -declare -r plugin=scheduler_plugin - fold_list_onto_array cpus $(parse_cpu_list <(echo "$spdk_cpus_csv")) # Normalize the indexes cpus=("${cpus[@]}") -create_thread() { - "$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_create "$@" -} - -destroy_thread() { - "$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_delete "$@" -} - -active_thread() { - "$rootdir/scripts/rpc.py" --plugin "$plugin" scheduler_thread_set_active "$@" -} - busy() { local selected_cpus cpu local reactor_framework diff --git a/test/scheduler/scheduler.sh b/test/scheduler/scheduler.sh index bc5f90c92..dab769a8a 100755 --- a/test/scheduler/scheduler.sh +++ b/test/scheduler/scheduler.sh @@ -11,3 +11,4 @@ source "$testdir/isolate_cores.sh" run_test "idle" "$testdir/idle.sh" run_test "load_balancing" "$testdir/load_balancing.sh" run_test "dpdk_governor" "$testdir/governor.sh" +run_test "interrupt_mode" "$testdir/interrupt.sh"