ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvme/perf: add bdevperf

Browse Source 
Adding capability to use bdevperf to this script. We
need to use bdevperf to demonstrate performance of up
to 10 million IOPS per core. fio overhead limits that
number to less than 3 million IOPS/core.

Signed-off-by: John Kariuki <John.K.Kariuki@intel.com>
Change-Id: Iad81fc73a961b05e168befbed01bf800b9888564
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/459695
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
John Kariuki 2019-06-27 15:02:31 -07:00 committed by Darek Stojaczyk
parent ed486dc4ef
commit 4af44c234f
2 changed files with 75 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
test/nvme/perf/common.sh
View File

@ -5,6 +5,7 @@ BASE_DIR=$(readlink -f $(dirname $0))
ROOT_DIR=$(readlink -f $BASE_DIR/../../..) ROOT_DIR=$(readlink -f $BASE_DIR/../../..)
PLUGIN_DIR_NVME=$ROOT_DIR/examples/nvme/fio_plugin PLUGIN_DIR_NVME=$ROOT_DIR/examples/nvme/fio_plugin
PLUGIN_DIR_BDEV=$ROOT_DIR/examples/bdev/fio_plugin PLUGIN_DIR_BDEV=$ROOT_DIR/examples/bdev/fio_plugin
BDEVPERF_DIR=$ROOT_DIR/test/bdev/bdevperf
. $ROOT_DIR/scripts/common.sh || exit 1 . $ROOT_DIR/scripts/common.sh || exit 1
. $ROOT_DIR/test/common/autotest_common.sh . $ROOT_DIR/test/common/autotest_common.sh
NVME_FIO_RESULTS=$BASE_DIR/result.json NVME_FIO_RESULTS=$BASE_DIR/result.json
@ -50,7 +51,7 @@ function get_numa_node(){
echo $(cat /sys/bus/pci/devices/$bdf/numa_node) echo $(cat /sys/bus/pci/devices/$bdf/numa_node)
fi fi
done done
elif [ "$plugin" = "bdev" ]; then elif [ "$plugin" = "bdev" ] || [ "$plugin" = "bdevperf" ]; then
local bdevs=$(discover_bdevs $ROOT_DIR $BASE_DIR/bdev.conf) local bdevs=$(discover_bdevs $ROOT_DIR $BASE_DIR/bdev.conf)
for name in $disks; do for name in $disks; do
local bdev_bdf=$(jq -r ".[] | select(.name==\"$name\").driver_specific.nvme.pci_address" <<< $bdevs) local bdev_bdf=$(jq -r ".[] | select(.name==\"$name\").driver_specific.nvme.pci_address" <<< $bdevs)
@ -72,7 +73,7 @@ function get_disks(){
echo "$bdf" echo "$bdf"
fi fi
done done
elif [ "$plugin" = "bdev" ]; then elif [ "$plugin" = "bdev" ] || [ "$plugin" = "bdevperf" ]; then
local bdevs=$(discover_bdevs $ROOT_DIR $BASE_DIR/bdev.conf) local bdevs=$(discover_bdevs $ROOT_DIR $BASE_DIR/bdev.conf)
echo $(jq -r '.[].name' <<< $bdevs) echo $(jq -r '.[].name' <<< $bdevs)
else else
@ -239,6 +240,21 @@ function get_results(){
esac esac
} }
function get_bdevperf_results(){
case "$1" in
iops)
iops=$(grep Total $NVME_FIO_RESULTS | awk -F 'Total' '{print $2}' | awk '{print $2}')
iops=${iops%.*}
echo $iops
;;
bw_Kibs)
bw_MBs=$(grep Total $NVME_FIO_RESULTS | awk -F 'Total' '{print $2}' | awk '{print $4}')
bw_MBs=${bw_MBs%.*}
echo $(( $bw_MBs * 1024 ))
;;
esac
}
function run_spdk_nvme_fio(){ function run_spdk_nvme_fio(){
local plugin=$1 local plugin=$1
echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting." echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting."
@ -259,6 +275,12 @@ function run_nvme_fio(){
sleep 1 sleep 1
} }
function run_bdevperf(){
echo "** Running bdevperf test, this can take a while, depending on the run-time setting."
echo $($BDEVPERF_DIR/bdevperf -c $BASE_DIR/bdev.conf -q $IODEPTH -o $BLK_SIZE -w $RW -M $MIX -t $RUNTIME)
sleep 1
}
function usage() function usage()
{ {
set +x set +x
@ -316,7 +338,7 @@ done
trap 'rm -f *.state $BASE_DIR/bdev.conf; print_backtrace' ERR SIGTERM SIGABRT trap 'rm -f *.state $BASE_DIR/bdev.conf; print_backtrace' ERR SIGTERM SIGABRT
mkdir -p $BASE_DIR/results mkdir -p $BASE_DIR/results
date="$(date +'%m_%d_%Y_%H%M%S')" date="$(date +'%m_%d_%Y_%H%M%S')"
if [ $PLUGIN = "bdev" ]; then if [[ $PLUGIN == "bdev" ]] || [[ $PLUGIN == "bdevperf" ]]; then
$ROOT_DIR/scripts/gen_nvme.sh >> $BASE_DIR/bdev.conf $ROOT_DIR/scripts/gen_nvme.sh >> $BASE_DIR/bdev.conf
fi fi

82
test/nvme/perf/run_perf.sh
View File

@ -88,35 +88,42 @@ do
qd=$IODEPTH qd=$IODEPTH
fi fi
create_fio_config $k $PLUGIN "$disk_names" "$disks_numa" "$cores" if [ $PLUGIN = "bdevperf" ]; then
desc="Running Test: Blocksize=${BLK_SIZE} Workload=$RW MIX=${MIX} qd=${IODEPTH} io_plugin/driver=$PLUGIN" run_bdevperf > $NVME_FIO_RESULTS
iops_disks[$k]=$((${iops_disks[$k]} + $(get_bdevperf_results iops)))
if [ $PLUGIN = "nvme" ] || [ $PLUGIN = "bdev" ]; then bw[$k]=$((${bw[$k]} + $(get_bdevperf_results bw_Kibs)))
run_spdk_nvme_fio $PLUGIN "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\ cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.output
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_$${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}"
else else
run_nvme_fio $fio_ioengine_opt "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\ create_fio_config $k $PLUGIN "$disk_names" "$disks_numa" "$cores"
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\ desc="Running Test: Blocksize=${BLK_SIZE} Workload=$RW MIX=${MIX} qd=${IODEPTH} io_plugin/driver=$PLUGIN"
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}" if [ $PLUGIN = "nvme" ] || [ $PLUGIN = "bdev" ]; then
run_spdk_nvme_fio $PLUGIN "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_$${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}"
else
run_nvme_fio $fio_ioengine_opt "--runtime=$RUNTIME" "--ramp_time=$RAMP_TIME" "--bs=$BLK_SIZE"\
"--rw=$RW" "--rwmixread=$MIX" "--iodepth=$qd" "--output=$NVME_FIO_RESULTS" "--time_based=1"\
"--numjobs=$NUMJOBS" "--description=$desc" "-log_avg_msec=250"\
"--write_lat_log=$BASE_DIR/results/$result_dir/perf_lat_${BLK_SIZE}BS_${IODEPTH}QD_${RW}_${MIX}MIX_${PLUGIN}_${date}_${k}disks_${j}"
fi
#Store values for every number of used disks
iops_disks[$k]=$((${iops_disks[$k]} + $(get_results iops $MIX)))
mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + $(get_results mean_lat_usec $MIX)))
p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} + $(get_results p99_lat_usec $MIX)))
p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} + $(get_results p99_99_lat_usec $MIX)))
stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} + $(get_results stdev_usec $MIX)))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} + $(get_results mean_slat_usec $MIX)))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} + $(get_results mean_clat_usec $MIX)))
bw[$k]=$((${bw[$k]} + $(get_results bw_Kibs $MIX)))
cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.json
cp $BASE_DIR/config.fio $BASE_DIR/results/$result_dir/config_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.fio
rm -f $BASE_DIR/config.fio
fi fi
#Store values for every number of used disks
iops_disks[$k]=$((${iops_disks[$k]} + $(get_results iops $MIX)))
mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + $(get_results mean_lat_usec $MIX)))
p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} + $(get_results p99_lat_usec $MIX)))
p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} + $(get_results p99_99_lat_usec $MIX)))
stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} + $(get_results stdev_usec $MIX)))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} + $(get_results mean_slat_usec $MIX)))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} + $(get_results mean_clat_usec $MIX)))
bw[$k]=$((${bw[$k]} + $(get_results bw_Kibs $MIX)))
cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.json
cp $BASE_DIR/config.fio $BASE_DIR/results/$result_dir/config_${MIX}_${PLUGIN}_${no_cores}cpus_${date}_${k}_disks_${j}.fio
rm -f $BASE_DIR/config.fio
#if tested on only one number of disk #if tested on only one number of disk
if $ONEWORKLOAD; then if $ONEWORKLOAD; then
break break
@ -127,12 +134,23 @@ done
for (( k=$DISKNO; k >= 1; k-=2 )) for (( k=$DISKNO; k >= 1; k-=2 ))
do do
iops_disks[$k]=$((${iops_disks[$k]} / $REPEAT_NO)) iops_disks[$k]=$((${iops_disks[$k]} / $REPEAT_NO))
mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / $REPEAT_NO))
p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / $REPEAT_NO)) if [ $PLUGIN != "bdevperf" ]; then
p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / $REPEAT_NO)) mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / $REPEAT_NO))
stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / $REPEAT_NO)) p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / $REPEAT_NO))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / $REPEAT_NO)) p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / $REPEAT_NO))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / $REPEAT_NO)) stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / $REPEAT_NO))
mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / $REPEAT_NO))
mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / $REPEAT_NO))
else
mean_lat_disks_usec[$k]=0
p99_lat_disks_usec[$k]=0
p99_99_lat_disks_usec[$k]=0
stdev_disks_usec[$k]=0
mean_slat_disks_usec[$k]=0
mean_clat_disks_usec[$k]=0
fi
bw[$k]=$((${bw[$k]} / $REPEAT_NO)) bw[$k]=$((${bw[$k]} / $REPEAT_NO))
printf "%s,%s,%s,%s,%s,%s,%s,%s,%s\n" ${k} ${iops_disks[$k]} ${mean_lat_disks_usec[$k]} ${p99_lat_disks_usec[$k]}\ printf "%s,%s,%s,%s,%s,%s,%s,%s,%s\n" ${k} ${iops_disks[$k]} ${mean_lat_disks_usec[$k]} ${p99_lat_disks_usec[$k]}\