From 7832b34f948dd1a4708272a84b701d875e425561 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Fri, 24 Feb 2023 10:58:10 +0100 Subject: [PATCH] test/scheduler: Trigger actual freq drop In order to do so we need to make sure that freq is lowered for all thread siblings of a given core. Since DPDK and/or dynamic scheduler do not take that into the account we need to do this on our own. Find thread sibling of the main cpu and imitate the DPDK's governor work by adjusting its freq settings. Signed-off-by: Michal Berger Change-Id: I154a2a789903b66c2722160d7e252221083f5e3c Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16930 Reviewed-by: Konrad Sztyber Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris --- test/scheduler/common.sh | 10 +++++++--- test/scheduler/governor.sh | 34 +++++++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/test/scheduler/common.sh b/test/scheduler/common.sh index 54c532a7d..d9682e4c6 100644 --- a/test/scheduler/common.sh +++ b/test/scheduler/common.sh @@ -64,6 +64,9 @@ map_cpus_node() { core_idx=$(< "$sysfs_cpu/cpu$cpu_idx/topology/core_id") local -n _cpu_core_map=node_${node_idx}_core_${core_idx} _cpu_core_map+=("$cpu_idx") cpu_core_map[cpu_idx]=$core_idx + local -n _cpu_siblings=node_${node_idx}_core_${core_idx}_thread_${cpu_idx} + _cpu_siblings=($(parse_cpu_list "$sysfs_cpu/cpu$cpu_idx/topology/thread_siblings_list")) + cpu_siblings[cpu_idx]="node_${node_idx}_core_${core_idx}_thread_${cpu_idx}[@]" fi _cpu_node_map[cpu_idx]=$cpu_idx cpu_node_map[cpu_idx]=$node_idx cpus+=("$cpu_idx") @@ -74,6 +77,7 @@ map_cpus_node() { map_cpus() { local -g cpus=() + local -g cpu_siblings=() local -g nodes=() local -g cpu_node_map=() local -g cpu_core_map=() @@ -354,12 +358,12 @@ set_cpufreq() { echo "$min_freq" > "$cpufreq/scaling_setspeed" ;; intel_pstate | intel_cpufreq) - if ((min_freq <= cpufreq_max_freqs[cpu])); then - echo "$min_freq" > "$cpufreq/scaling_min_freq" - fi if [[ -n $max_freq ]] && ((max_freq >= min_freq)); then echo "$max_freq" > "$cpufreq/scaling_max_freq" fi + if ((min_freq <= cpufreq_max_freqs[cpu])); then + echo "$min_freq" > "$cpufreq/scaling_min_freq" + fi ;; esac } diff --git a/test/scheduler/governor.sh b/test/scheduler/governor.sh index a5067bbde..78b232c25 100755 --- a/test/scheduler/governor.sh +++ b/test/scheduler/governor.sh @@ -43,11 +43,23 @@ update_main_core_cpufreq() { intel_pstate | intel_cpufreq) main_core_setspeed=$main_core_set_max_freq ;; cppc_cpufreq) main_core_setspeed=${cpufreq_setspeed[spdk_main_core]} ;; esac + + local thread + for thread in "${!cpu_siblings[spdk_main_core]}"; do + ((thread == spdk_main_core)) && continue # handled by DPDK/scheduler + # While assigning cpus to SPDK, we took every thread from given core, + # hence the cpufreq governor should already be properly set by the + # DPDK. So the only thing we need to take care of is {max,min} freq. + # The expectation here is to see actual freq drop on the main cpu in + # next iterations. Both, max and min, should be set to the same value. + set_cpufreq "$thread" "$main_core_set_min_freq" "$main_core_set_max_freq" + done } verify_dpdk_governor() { xtrace_disable + map_cpus # Run the app and see if DPDK's PM subsystem is doing the proper thing. Aim at the main core. # What we expect to happen is based mainly on what cpufreq driver is in use. The main assumption # is that with all SPDK threads being idle the governor should start lowering the frequency for @@ -77,6 +89,7 @@ verify_dpdk_governor() { unset -v "cpus[spdk_main_core]" local samples=0 all_set=0 dir=-1 old_main_core_setspeed=0 + local old_main_core_set_cur_freq=0 first_main_core_set_cur_freq=0 exec_under_dynamic_scheduler "${SPDK_APP[@]}" -m "$spdk_cpumask" --main-core "$spdk_main_core" @@ -94,9 +107,13 @@ verify_dpdk_governor() { elif ((main_core_setspeed < old_main_core_setspeed)); then dir=0 elif ((main_core_setspeed == old_main_core_setspeed)); then - # Frequency didn't change, skip and wait for a bit + # Frequency didn't change, wait for a bit, but then fall to the main check to + # see if cur freq actually changed or not. sleep 0.5s - continue + fi + + if ((first_main_core_set_cur_freq == 0)); then + first_main_core_set_cur_freq=$main_core_set_cur_freq fi case "$main_core_driver" in @@ -119,16 +136,10 @@ verify_dpdk_governor() { && ((main_core_setspeed == main_core_freqs[-1])) \ && ((dir == 0)) ;; - esac && all_set=1 + esac && ((main_core_set_cur_freq < old_main_core_set_cur_freq)) && all_set=1 # Print stats after first sane sample was taken if ((old_main_core_setspeed != 0 && dir != -1)); then - # Currently, we only determine if DPDK set sysfs attributes to min frequency supported by given cpu. - # We don't act on actual frequency cpu ends up spinning at. The reason being all sibling threads of - # the main cpu would need to be adjusted manually (i.e. their cpufreq attributes) in order for freq - # to start going down. This is deemed as not necessary for the moment since the core objective of - # the test is to determine if DPDK's governor did its job, within its current scope, not how it - # impacts the system overall. printf 'MAIN DPDK cpu%u current frequency at %u KHz (%u-%u KHz), set frequency %u KHz %s %u KHz\n' \ "$spdk_main_core" "$main_core_set_cur_freq" "$main_core_min_freq" "$main_core_max_freq" \ "$main_core_setspeed" "${dir_map[dir]}" "$old_main_core_setspeed" @@ -137,10 +148,15 @@ verify_dpdk_governor() { fi old_main_core_setspeed=$main_core_setspeed + old_main_core_set_cur_freq=$main_core_set_cur_freq done ((all_set == 1)) + printf 'Main cpu%u frequency dropped by %u%%\n' \ + "$spdk_main_core" \ + $(((first_main_core_set_cur_freq - main_core_set_cur_freq) * 100 / (first_main_core_set_cur_freq - main_core_min_freq))) + xtrace_restore }