From d60bc7714266885841667392efd726a918995147 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Fri, 27 Nov 2020 09:30:26 +0100 Subject: [PATCH] test/setup: Add tests for hugepages allocation These tests are meant to "predict" the right number of hugepages given setup.sh instance will set, depending on its environment options (HUGENODE, HUGEMEM, etc.). Change-Id: Iaf441e69d25dd59941891e1d5c3c2b0dbd77644d Signed-off-by: Michal Berger Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5309 Tested-by: SPDK CI Jenkins Reviewed-by: Tomasz Zawadzki Reviewed-by: Ben Walker Reviewed-by: Jim Harris Reviewed-by: Karol Latecki --- test/setup/common.sh | 24 +++++ test/setup/hugepages.sh | 187 +++++++++++++++++++++++++++++++++++++++ test/setup/test-setup.sh | 1 + 3 files changed, 212 insertions(+) create mode 100755 test/setup/hugepages.sh diff --git a/test/setup/common.sh b/test/setup/common.sh index d7913e802..98f8b351e 100644 --- a/test/setup/common.sh +++ b/test/setup/common.sh @@ -7,3 +7,27 @@ setup() { "$rootdir/scripts/setup.sh" "$@" &> /dev/null fi } + +get_meminfo() { + xtrace_disable + + local get=$1 + local node=$2 + local var val + local mem_f mem + + mem_f=/proc/meminfo + if [[ -e /sys/devices/system/node/node$node/meminfo ]]; then + mem_f=/sys/devices/system/node/node$node/meminfo + fi + mapfile -t mem < "$mem_f" + mem=("${mem[@]#Node +([0-9]) }") + + while IFS=": " read -r var val _; do + [[ $var == "$get" ]] || continue + echo "$val" && return 0 + done < <(printf '%s\n' "${mem[@]}") + return 1 + + xtrace_restore +} diff --git a/test/setup/hugepages.sh b/test/setup/hugepages.sh new file mode 100755 index 000000000..a8460c07d --- /dev/null +++ b/test/setup/hugepages.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +testdir=$(readlink -f "$(dirname "$0")") +rootdir=$(readlink -f "$testdir/../../") +source "$testdir/common.sh" + +shopt -s extglob nullglob + +declare -a nodes_sys=() + +declare -i default_huges=0 +declare -i no_nodes=0 +declare -i nr_hugepages=0 + +default_huges=$(get_meminfo Hugepagesize) +default_huge_nr=/sys/kernel/mm/hugepages/hugepages-${default_huges}kB/nr_hugepages +global_huge_nr=/proc/sys/vm/nr_hugepages + +get_nodes() { + local node + + for node in /sys/devices/system/node/node+([0-9]); do + nodes_sys[${node##*node}]=$(< "$node/hugepages/hugepages-${default_huges}kB/nr_hugepages") + done + no_nodes=${#nodes_sys[@]} + ((no_nodes > 0)) +} + +clear_hp() { + local node hp + + for node in "${!nodes_sys[@]}"; do + for hp in "/sys/devices/system/node/node$node/hugepages/hugepages-"*; do + echo 0 > "$hp/nr_hugepages" + done + done + + export CLEAR_HUGE=yes +} + +get_test_nr_hugepages() { + local size=$1 # kB + if (($# > 1)); then + shift + local node_ids=("$@") + fi + + ((size >= default_huges)) + + nr_hugepages=$(((size + default_huges - 1) / default_huges)) + get_test_nr_hugepages_per_node "${node_ids[@]}" +} + +get_test_nr_hugepages_per_node() { + local user_nodes=("$@") + + local _nr_hugepages=$nr_hugepages + local _no_nodes=$no_nodes + + local -g nodes_test=() + + if ((${#user_nodes[@]} > 0)); then + for _no_nodes in "${user_nodes[@]}"; do + nodes_test[_no_nodes]=$nr_hugepages + done + return 0 + elif ((${#nodes_hp[@]} > 0)); then + for _no_nodes in "${!nodes_hp[@]}"; do + nodes_test[_no_nodes]=${nodes_hp[_no_nodes]} + done + return 0 + fi + + while ((_no_nodes > 0)); do + nodes_test[_no_nodes - 1]=$((_nr_hugepages / _no_nodes)) + : $((_nr_hugepages -= nodes_test[_no_nodes - 1])) + : $((--_no_nodes)) + done +} + +verify_nr_hugepages() { + local node + local sorted_t + local sorted_s + + echo "nr_hugepages=$nr_hugepages" + (($(< "$default_huge_nr") == nr_hugepages)) + (($(< "$global_huge_nr") == nr_hugepages)) + (($(get_meminfo HugePages_Total) == nr_hugepages)) + + get_nodes + + # There's no obvious way of determining which NUMA node is going to end + # up with an odd number of hugepages in case such number was actually + # allocated by the kernel. Considering that, let's simply check if our + # expaction is met by sorting and comparing it with nr of hugepages that + # was actually allocated on each node. + + for node in "${!nodes_test[@]}"; do + sorted_t[nodes_test[node]]=1 sorted_s[nodes_sys[node]]=1 + echo "node$node=${nodes_sys[node]}" + done + [[ ${!sorted_s[*]} == "${!sorted_t[*]}" ]] +} + +# Test cases +default_setup() { + # Default HUGEMEM (8G) alloc on node0 + get_test_nr_hugepages $((HUGEMEM * 1024)) 0 + setup + verify_nr_hugepages +} + +per_node_2G_alloc() { + # 2G alloc per node, total N*2G pages + local IFS="," + + get_test_nr_hugepages $((2048 * 1024)) "${!nodes_sys[@]}" + NRHUGE=$nr_hugepages HUGENODE="${!nodes_sys[*]}" setup + nr_hugepages=$((nr_hugepages * ${#nodes_sys[@]})) verify_nr_hugepages +} + +even_2G_alloc() { + # 2G alloc spread across N nodes + get_test_nr_hugepages $((2048 * 1024)) + NRHUGE=$nr_hugepages HUGE_EVEN_ALLOC=yes setup + verify_nr_hugepages +} + +odd_alloc() { + # Odd 2049MB alloc across N nodes + get_test_nr_hugepages $((2049 * 1024)) + HUGEMEM=2049 HUGE_EVEN_ALLOC=yes setup + verify_nr_hugepages +} + +custom_alloc() { + # Custom alloc: node0 == 512 pages [node1 == 1024 pages] + + local IFS="," + + local node + local nodes_hp=() + + local nr_hugepages=0 + + nodes_hp[0]=512 + if ((${#nodes_sys[@]} > 1)); then + nodes_hp[1]=1024 + fi + + for node in "${!nodes_hp[@]}"; do + HUGENODE+=("nodes_hp[$node]=${nodes_hp[node]}") + ((nr_hugepages += nodes_hp[node])) + done + + get_test_nr_hugepages_per_node + HUGENODE="${HUGENODE[*]}" setup + verify_nr_hugepages +} + +hp_status() { + # Parse status from last verification + + local node + local size free total + + ((${#nodes_sys[@]} > 0)) + + while read -r node size free _ total; do + size=${size/kB/} node=${node#node} + ((size == default_huges)) || continue + ((free == nodes_test[node])) + ((total == nodes_test[node])) + done < <(setup output status |& grep "node[0-9]") +} + +get_nodes +clear_hp + +run_test "default_setup" default_setup +run_test "per_node_2G_alloc" per_node_2G_alloc +run_test "even_2G_alloc" even_2G_alloc +run_test "odd_alloc" odd_alloc +run_test "custom_alloc" custom_alloc +run_test "hp_status" hp_status + +clear_hp diff --git a/test/setup/test-setup.sh b/test/setup/test-setup.sh index e5a09c80f..1ff0b937e 100755 --- a/test/setup/test-setup.sh +++ b/test/setup/test-setup.sh @@ -6,3 +6,4 @@ source "$testdir/common.sh" [[ $(uname -s) == Linux ]] || exit 0 run_test "acl" "$testdir/acl.sh" +run_test "hugepages" "$testdir/hugepages.sh"