From c3dc7fb5f2b7ec2edbb21889b046ec9291b5872e Mon Sep 17 00:00:00 2001 From: Karol Latecki Date: Wed, 8 Jul 2020 17:43:39 +0200 Subject: [PATCH] test/vhost_perf: parse fio json results to csv Parse fio output to CSV. Calculate average read and write metrics while parsing and present final results. Change-Id: I8f6d7f84205be409f0ccbba7a42eb7888b2cb03d Signed-off-by: Karol Latecki Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3253 Tested-by: SPDK CI Jenkins Community-CI: Mellanox Build Bot Reviewed-by: Shuhei Matsumoto Reviewed-by: Maciej Wawryk Reviewed-by: Tomasz Zawadzki Reviewed-by: Michal Berger --- test/vhost/common.sh | 75 +++++++++++++++++++++++++++++ test/vhost/perf_bench/vhost_perf.sh | 2 + 2 files changed, 77 insertions(+) diff --git a/test/vhost/common.sh b/test/vhost/common.sh index 45332d91a..2d1a142f6 100644 --- a/test/vhost/common.sh +++ b/test/vhost/common.sh @@ -1155,6 +1155,81 @@ function run_fio() { fi } +# Parsing fio results for json output and client-server mode only! +function parse_fio_results() { + local fio_log_dir=$1 + local fio_log_filename=$2 + local fio_csv_filename + + # Variables used in parsing loop + local log_file + local rwmode mixread mixwrite + local lat_key lat_divisor + local client_stats iops bw + local read_avg_lat read_min_lat read_max_lat + local write_avg_lat write_min_lat write_min_lat + + declare -A results + results["iops"]=0 + results["bw"]=0 + results["avg_lat"]=0 + results["min_lat"]=0 + results["max_lat"]=0 + + # Loop using the log filename to see if there are any other + # matching files. This is in case we ran fio test multiple times. + log_files=("$fio_log_dir/$fio_log_filename"*) + for log_file in "${log_files[@]}"; do + rwmode=$(jq -r '.["client_stats"][0]["job options"]["rw"]' "$log_file") + mixread=1 + mixwrite=1 + if [[ $rwmode = *"rw"* ]]; then + mixread=$(jq -r '.["client_stats"][0]["job options"]["rwmixread"]' "$log_file") + mixread=$(bc -l <<< "scale=3; $mixread/100") + mixwrite=$(bc -l <<< "scale=3; 1-$mixread") + fi + + client_stats=$(jq -r '.["client_stats"][] | select(.jobname == "All clients")' "$log_file") + + # Check latency unit and later normalize to microseconds + lat_key="lat_us" + lat_divisor=1 + if jq -er '.read["lat_ns"]' &> /dev/null <<< $client_stats; then + lat_key="lat_ns" + lat_divisor=1000 + fi + + # Horrific bash float point arithmetic oprations below. + # Viewer discretion is advised. + iops=$(jq -r '[.read["iops"],.write["iops"]] | add' <<< $client_stats) + bw=$(jq -r '[.read["bw"],.write["bw"]] | add' <<< $client_stats) + read_avg_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["mean"]' <<< $client_stats) + read_min_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["min"]' <<< $client_stats) + read_max_lat=$(jq -r --arg lat_key $lat_key '.read[$lat_key]["max"]' <<< $client_stats) + write_avg_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["mean"]' <<< $client_stats) + write_min_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["min"]' <<< $client_stats) + write_max_lat=$(jq -r --arg lat_key $lat_key '.write[$lat_key]["max"]' <<< $client_stats) + + results["iops"]=$(bc -l <<< "${results[iops]} + $iops") + results["bw"]=$(bc -l <<< "${results[bw]} + $bw") + results["avg_lat"]=$(bc -l <<< "${results[avg_lat]} + ($mixread*$read_avg_lat + $mixwrite*$write_avg_lat)/$lat_divisor") + results["min_lat"]=$(bc -l <<< "${results[min_lat]} + ($mixread*$read_min_lat + $mixwrite*$write_min_lat)/$lat_divisor") + results["max_lat"]=$(bc -l <<< "${results[max_lat]} + ($mixread*$read_max_lat + $mixwrite*$write_max_lat)/$lat_divisor") + done + + results["iops"]=$(bc -l <<< "scale=3; ${results[iops]} / ${#log_files[@]}") + results["bw"]=$(bc -l <<< "scale=3; ${results[bw]} / ${#log_files[@]}") + results["avg_lat"]=$(bc -l <<< "scale=3; ${results[avg_lat]} / ${#log_files[@]}") + results["min_lat"]=$(bc -l <<< "scale=3; ${results[min_lat]} / ${#log_files[@]}") + results["max_lat"]=$(bc -l <<< "scale=3; ${results[max_lat]} / ${#log_files[@]}") + + fio_csv_filename="${fio_log_filename%%.*}.csv" + cat <<- EOF > "$fio_log_dir/$fio_csv_filename" + iops,bw,avg_lat,min_lat,max_lat + ${results["iops"]},${results["bw"]},${results["avg_lat"]},${results["min_lat"]},${results["max_lat"]} + EOF +} + # Shutdown or kill any running VM and SPDK APP. # function at_app_exit() { diff --git a/test/vhost/perf_bench/vhost_perf.sh b/test/vhost/perf_bench/vhost_perf.sh index 068ab5001..6b8f67d30 100755 --- a/test/vhost/perf_bench/vhost_perf.sh +++ b/test/vhost/perf_bench/vhost_perf.sh @@ -446,6 +446,8 @@ for fio_job in ${fio_jobs//,/ }; do mv $VHOST_DIR/fio_results/$fio_log_fname $VHOST_DIR/fio_results/$fio_log_fname.$i sleep 1 done + + parse_fio_results "$VHOST_DIR/fio_results" "$fio_log_fname" done notice "Shutting down virtual machines..."