test/scheduler: Trigger actual freq drop

In order to do so we need to make sure that freq is lowered for
all thread siblings of a given core. Since DPDK and/or dynamic
scheduler do not take that into the account we need to do this
on our own.

Find thread sibling of the main cpu and imitate the DPDK's governor
work by adjusting its freq settings.

Signed-off-by: Michal Berger <michal.berger@intel.com>
Change-Id: I154a2a789903b66c2722160d7e252221083f5e3c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16930
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

test/scheduler/common.sh

@@ -64,6 +64,9 @@ map_cpus_node() {
 			core_idx=$(< "$sysfs_cpu/cpu$cpu_idx/topology/core_id")
 			local -n _cpu_core_map=node_${node_idx}_core_${core_idx}
 			_cpu_core_map+=("$cpu_idx") cpu_core_map[cpu_idx]=$core_idx
+			local -n _cpu_siblings=node_${node_idx}_core_${core_idx}_thread_${cpu_idx}
+			_cpu_siblings=($(parse_cpu_list "$sysfs_cpu/cpu$cpu_idx/topology/thread_siblings_list"))
+			cpu_siblings[cpu_idx]="node_${node_idx}_core_${core_idx}_thread_${cpu_idx}[@]"
 		fi
 		_cpu_node_map[cpu_idx]=$cpu_idx cpu_node_map[cpu_idx]=$node_idx
 		cpus+=("$cpu_idx")
@@ -74,6 +77,7 @@ map_cpus_node() {
 
 map_cpus() {
 	local -g cpus=()
+	local -g cpu_siblings=()
 	local -g nodes=()
 	local -g cpu_node_map=()
 	local -g cpu_core_map=()
@@ -354,12 +358,12 @@ set_cpufreq() {
 			echo "$min_freq" > "$cpufreq/scaling_setspeed"
 			;;
 		intel_pstate | intel_cpufreq)
-			if ((min_freq <= cpufreq_max_freqs[cpu])); then
-				echo "$min_freq" > "$cpufreq/scaling_min_freq"
-			fi
 			if [[ -n $max_freq ]] && ((max_freq >= min_freq)); then
 				echo "$max_freq" > "$cpufreq/scaling_max_freq"
 			fi
+			if ((min_freq <= cpufreq_max_freqs[cpu])); then
+				echo "$min_freq" > "$cpufreq/scaling_min_freq"
+			fi
 			;;
 	esac
 }

test/scheduler/governor.sh

@@ -43,11 +43,23 @@ update_main_core_cpufreq() {
 		intel_pstate | intel_cpufreq) main_core_setspeed=$main_core_set_max_freq ;;
 		cppc_cpufreq) main_core_setspeed=${cpufreq_setspeed[spdk_main_core]} ;;
 	esac
+
+	local thread
+	for thread in "${!cpu_siblings[spdk_main_core]}"; do
+		((thread == spdk_main_core)) && continue # handled by DPDK/scheduler
+		# While assigning cpus to SPDK, we took every thread from given core,
+		# hence the cpufreq governor should already be properly set by the
+		# DPDK. So the only thing we need to take care of is {max,min} freq.
+		# The expectation here is to see actual freq drop on the main cpu in
+		# next iterations. Both, max and min, should be set to the same value.
+		set_cpufreq "$thread" "$main_core_set_min_freq" "$main_core_set_max_freq"
+	done
 }
 
 verify_dpdk_governor() {
 	xtrace_disable
 
+	map_cpus
 	# Run the app and see if DPDK's PM subsystem is doing the proper thing. Aim at the main core.
 	# What we expect to happen is based mainly on what cpufreq driver is in use. The main assumption
 	# is that with all SPDK threads being idle the governor should start lowering the frequency for
@@ -77,6 +89,7 @@ verify_dpdk_governor() {
 	unset -v "cpus[spdk_main_core]"
 
 	local samples=0 all_set=0 dir=-1 old_main_core_setspeed=0
+	local old_main_core_set_cur_freq=0 first_main_core_set_cur_freq=0
 
 	exec_under_dynamic_scheduler "${SPDK_APP[@]}" -m "$spdk_cpumask" --main-core "$spdk_main_core"
 
@@ -94,9 +107,13 @@ verify_dpdk_governor() {
 		elif ((main_core_setspeed < old_main_core_setspeed)); then
 			dir=0
 		elif ((main_core_setspeed == old_main_core_setspeed)); then
-			# Frequency didn't change, skip and wait for a bit
+			# Frequency didn't change, wait for a bit, but then fall to the main check to
+			# see if cur freq actually changed or not.
 			sleep 0.5s
-			continue
+		fi
+
+		if ((first_main_core_set_cur_freq == 0)); then
+			first_main_core_set_cur_freq=$main_core_set_cur_freq
 		fi
 
 		case "$main_core_driver" in
@@ -119,16 +136,10 @@ verify_dpdk_governor() {
 					&& ((main_core_setspeed == main_core_freqs[-1])) \
 					&& ((dir == 0))
 				;;
-		esac && all_set=1
+		esac && ((main_core_set_cur_freq < old_main_core_set_cur_freq)) && all_set=1
 
 		# Print stats after first sane sample was taken
 		if ((old_main_core_setspeed != 0 && dir != -1)); then
-			# Currently, we only determine if DPDK set sysfs attributes to min frequency supported by given cpu.
-			# We don't act on actual frequency cpu ends up spinning at. The reason being all sibling threads of
-			# the main cpu would need to be adjusted manually (i.e. their cpufreq attributes) in order for freq
-			# to start going down. This is deemed as not necessary for the moment since the core objective of
-			# the test is to determine if DPDK's governor did its job, within its current scope, not how it
-			# impacts the system overall.
 			printf 'MAIN DPDK cpu%u current frequency at %u KHz (%u-%u KHz), set frequency %u KHz %s %u KHz\n' \
 				"$spdk_main_core" "$main_core_set_cur_freq" "$main_core_min_freq" "$main_core_max_freq" \
 				"$main_core_setspeed" "${dir_map[dir]}" "$old_main_core_setspeed"
@@ -137,10 +148,15 @@ verify_dpdk_governor() {
 		fi
 
 		old_main_core_setspeed=$main_core_setspeed
+		old_main_core_set_cur_freq=$main_core_set_cur_freq
 	done
 
 	((all_set == 1))
 
+	printf 'Main cpu%u frequency dropped by %u%%\n' \
+		"$spdk_main_core" \
+		$(((first_main_core_set_cur_freq - main_core_set_cur_freq) * 100 / (first_main_core_set_cur_freq - main_core_min_freq)))
+
 	xtrace_restore
 }