nvme/perf: add bdevperf

Adding capability to use bdevperf to this script. We
need to use bdevperf to demonstrate performance of up
to 10 million IOPS per core. fio overhead limits that
number to less than 3 million IOPS/core.

Signed-off-by: John Kariuki <John.K.Kariuki@intel.com>
Change-Id: Iad81fc73a961b05e168befbed01bf800b9888564
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/459695
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
John Kariuki 2019-06-27 15:02:31 -07:00 committed by Darek Stojaczyk
parent ed486dc4ef
commit 4af44c234f
2 changed files with 75 additions and 35 deletions

View File

@ -5,6 +5,7 @@ BASE_DIR=$(readlink -f $(dirname $0))
ROOT_DIR=$(readlink -f $BASE_DIR/../../..)
PLUGIN_DIR_NVME=$ROOT_DIR/examples/nvme/fio_plugin
PLUGIN_DIR_BDEV=$ROOT_DIR/examples/bdev/fio_plugin
BDEVPERF_DIR=$ROOT_DIR/test/bdev/bdevperf
. $ROOT_DIR/scripts/common.sh || exit 1
. $ROOT_DIR/test/common/autotest_common.sh
NVME_FIO_RESULTS=$BASE_DIR/result.json
@ -50,7 +51,7 @@ function get_numa_node(){
echo $(cat /sys/bus/pci/devices/$bdf/numa_node)
fi
done
elif [ "$plugin" = "bdev" ]; then
elif [ "$plugin" = "bdev" ] || [ "$plugin" = "bdevperf" ]; then
local bdevs=$(discover_bdevs $ROOT_DIR $BASE_DIR/bdev.conf)
for name in $disks; do
local bdev_bdf=$(jq -r ".[] | select(.name==\"$name\").driver_specific.nvme.pci_address" <<< $bdevs)
@ -72,7 +73,7 @@ function get_disks(){
echo "$bdf"
fi
done
elif [ "$plugin" = "bdev" ]; then
elif [ "$plugin" = "bdev" ] || [ "$plugin" = "bdevperf" ]; then
local bdevs=$(discover_bdevs $ROOT_DIR $BASE_DIR/bdev.conf)
echo $(jq -r '.[].name' <<< $bdevs)
else
@ -239,6 +240,21 @@ function get_results(){
esac
}
function get_bdevperf_results(){
case "$1" in
iops)
iops=$(grep Total $NVME_FIO_RESULTS | awk -F 'Total' '{print $2}' | awk '{print $2}')
iops=${iops%.*}
echo $iops
;;
bw_Kibs)
bw_MBs=$(grep Total $NVME_FIO_RESULTS | awk -F 'Total' '{print $2}' | awk '{print $4}')
bw_MBs=${bw_MBs%.*}
echo $(( $bw_MBs * 1024 ))
;;
esac
}
function run_spdk_nvme_fio(){
local plugin=$1
echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting."
@ -259,6 +275,12 @@ function run_nvme_fio(){
sleep 1
}
function run_bdevperf(){
echo "** Running bdevperf test, this can take a while, depending on the run-time setting."
echo $($BDEVPERF_DIR/bdevperf -c $BASE_DIR/bdev.conf -q $IODEPTH -o $BLK_SIZE -w $RW -M $MIX -t $RUNTIME)
sleep 1
}
function usage()
{
set +x
@ -316,7 +338,7 @@ done
trap 'rm -f *.state $BASE_DIR/bdev.conf; print_backtrace' ERR SIGTERM SIGABRT
mkdir -p $BASE_DIR/results
date="$(date +'%m_%d_%Y_%H%M%S')"
if [ $PLUGIN = "bdev" ]; then
if [[ $PLUGIN == "bdev" ]] || [[ $PLUGIN == "bdevperf" ]]; then
$ROOT_DIR/scripts/gen_nvme.sh >> $BASE_DIR/bdev.conf
fi

View File

@ -88,35 +88,42 @@ do
qd=$IODEPTH
fi
create_fio_config $k $PLUGIN "$disk_names" "$disks_numa" "$cores"
desc="Running Test: Blocksize=${BLK_SIZE} Workload=$RW MIX=${MIX} qd=${IODEPTH} io_plugin/driver=$PLUGIN"
if [ $PLUGIN = "nvme" ] || [ $PLUGIN = "bdev" ]; then
run_spdk_nvme_fio $PLUGIN "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_$${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}"
if [ $PLUGIN = "bdevperf" ]; then
run_bdevperf > $NVME_FIO_RESULTS
iops_disks[$k]=$((${iops_disks[$k]} + $(get_bdevperf_results iops)))
bw[$k]=$((${bw[$k]} + $(get_bdevperf_results bw_Kibs)))
cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.output
else
run_nvme_fio $fio_ioengine_opt "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}"
create_fio_config $k $PLUGIN "$disk_names" "$disks_numa" "$cores"
desc="Running Test: Blocksize=${BLK_SIZE} Workload=$RW MIX=${MIX} qd=${IODEPTH} io_plugin/driver=$PLUGIN"
if [ $PLUGIN = "nvme" ] || [ $PLUGIN = "bdev" ]; then
run_spdk_nvme_fio $PLUGIN "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_$${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}"
else
run_nvme_fio $fio_ioengine_opt "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}"
fi
#Store values for every number of used disks
iops_disks[$k]=$((${iops_disks[$k]} + $(get_results iops $MIX)))
mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + $(get_results mean_lat_usec $MIX)))
p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} + $(get_results p99_lat_usec $MIX)))
p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} + $(get_results p99_99_lat_usec $MIX)))
stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} + $(get_results stdev_usec $MIX)))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} + $(get_results mean_slat_usec $MIX)))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} + $(get_results mean_clat_usec $MIX)))
bw[$k]=$((${bw[$k]} + $(get_results bw_Kibs $MIX)))
cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.json
cp $BASE_DIR/config.fio $BASE_DIR/results/$result_dir/config_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.fio
rm -f $BASE_DIR/config.fio
fi
#Store values for every number of used disks
iops_disks[$k]=$((${iops_disks[$k]} + $(get_results iops $MIX)))
mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + $(get_results mean_lat_usec $MIX)))
p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} + $(get_results p99_lat_usec $MIX)))
p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} + $(get_results p99_99_lat_usec $MIX)))
stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} + $(get_results stdev_usec $MIX)))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} + $(get_results mean_slat_usec $MIX)))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} + $(get_results mean_clat_usec $MIX)))
bw[$k]=$((${bw[$k]} + $(get_results bw_Kibs $MIX)))
cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.json
cp $BASE_DIR/config.fio $BASE_DIR/results/$result_dir/config_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.fio
rm -f $BASE_DIR/config.fio
#if tested on only one number of disk
if $ONEWORKLOAD; then
break
@ -127,12 +134,23 @@ done
for (( k=$DISKNO; k >= 1; k-=2 ))
do
iops_disks[$k]=$((${iops_disks[$k]} / $REPEAT_NO))
mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / $REPEAT_NO))
p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / $REPEAT_NO))
p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / $REPEAT_NO))
stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / $REPEAT_NO))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / $REPEAT_NO))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / $REPEAT_NO))
if [ $PLUGIN != "bdevperf" ]; then
mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / $REPEAT_NO))
p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / $REPEAT_NO))
p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / $REPEAT_NO))
stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / $REPEAT_NO))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / $REPEAT_NO))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / $REPEAT_NO))
else
mean_lat_disks_usec[$k]=0
p99_lat_disks_usec[$k]=0
p99_99_lat_disks_usec[$k]=0
stdev_disks_usec[$k]=0
mean_slat_disks_usec[$k]=0
mean_clat_disks_usec[$k]=0
fi
bw[$k]=$((${bw[$k]} / $REPEAT_NO))
printf "%s,%s,%s,%s,%s,%s,%s,%s,%s\n" ${k} ${iops_disks[$k]} ${mean_lat_disks_usec[$k]} ${p99_lat_disks_usec[$k]}\