From 18fbf2e15de01b2b0ebf1dd6db7ec3bd6a5f569a Mon Sep 17 00:00:00 2001 From: Karol Latecki Date: Thu, 9 Jan 2020 13:19:05 +0100 Subject: [PATCH] test/nvme-perf: add nvme perf option Allow running tests with nvme/perf. This will be needed for comparison with bdevperf and/or fio nvme plugin. Change-Id: Ie351d1b3c9d8eae430161779f9818c04d87e6ed7 Signed-off-by: Karol Latecki Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/479821 Tested-by: SPDK CI Jenkins Community-CI: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Tomasz Zawadzki --- test/nvme/perf/common.sh | 43 ++++++++++++++++++++++++++++++++++++-- test/nvme/perf/run_perf.sh | 22 +++++++++++++++++-- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/test/nvme/perf/common.sh b/test/nvme/perf/common.sh index 48dbd2b29..eaf79cf2b 100755 --- a/test/nvme/perf/common.sh +++ b/test/nvme/perf/common.sh @@ -6,6 +6,7 @@ ROOT_DIR=$(readlink -f $BASE_DIR/../../..) PLUGIN_DIR_NVME=$ROOT_DIR/examples/nvme/fio_plugin PLUGIN_DIR_BDEV=$ROOT_DIR/examples/bdev/fio_plugin BDEVPERF_DIR=$ROOT_DIR/test/bdev/bdevperf +NVMEPERF_DIR=$ROOT_DIR/examples/nvme/perf . $ROOT_DIR/scripts/common.sh || exit 1 . $ROOT_DIR/test/common/autotest_common.sh NVME_FIO_RESULTS=$BASE_DIR/result.json @@ -53,7 +54,7 @@ function get_cores_numa_node(){ function get_numa_node(){ local plugin=$1 local disks=$2 - if [[ "$plugin" == "spdk-plugin-nvme" ]]; then + if [[ "$plugin" =~ "nvme" ]]; then for bdf in $disks; do local driver driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}') @@ -82,7 +83,7 @@ function get_numa_node(){ function get_disks(){ local plugin=$1 - if [[ "$plugin" == "spdk-plugin-nvme" ]]; then + if [[ "$plugin" =~ "nvme" ]]; then for bdf in $(iter_pci_class_code 01 08 02); do driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}') if [ "$driver" = "vfio-pci" ] || [ "$driver" = "uio_pci_generic" ]; then @@ -288,6 +289,26 @@ function get_bdevperf_results(){ esac } +function get_nvmeperf_results() { + local iops + local bw_MBs + local mean_lat_usec + local max_lat_usec + local min_lat_usec + + read -r iops bw_MBs mean_lat_usec min_lat_usec max_lat_usec<<< $(cat $NVME_FIO_RESULTS | tr -s " " | grep -oP "(?<=Total : )(.*+)") + + # We need to get rid of the decimal spaces due + # to use of arithmetic expressions instead of "bc" for calculations + iops=${iops%.*} + bw_MBs=${bw_MBs%.*} + mean_lat_usec=${mean_lat_usec%.*} + min_lat_usec=${min_lat_usec%.*} + max_lat_usec=${max_lat_usec%.*} + + echo "$iops $(bc <<< "$bw_MBs * 1024") $mean_lat_usec $min_lat_usec $max_lat_usec" +} + function run_spdk_nvme_fio(){ local plugin=$1 echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting." @@ -314,6 +335,23 @@ function run_bdevperf(){ sleep 1 } +function run_nvmeperf() { + # Prepare -r argument string for nvme perf command + local r_opt + local disks + + # Limit the number of disks to $1 if needed + disks=( $(get_disks nvme) ) + disks=( "${disks[@]:0:$1}" ) + r_opt=$(printf -- ' -r "trtype:PCIe traddr:%s"' "${disks[@]}") + + echo "** Running nvme perf test, this can take a while, depending on the run-time setting." + + # Run command in separate shell as this solves quoting issues related to r_opt var + $SHELL -c "$NVMEPERF_DIR/perf $r_opt -q $IODEPTH -o $BLK_SIZE -w $RW -M $MIX -t $RUNTIME -c [$CPUS_ALLOWED]" + sleep 1 +} + function wait_for_nvme_reload() { local nvmes=$1 @@ -368,6 +406,7 @@ function usage() echo echo "Test setup parameters:" echo " --driver=STR Selects tool used for testing. Choices available:" + echo " - spdk-perf-nvme (SPDK nvme perf)" echo " - spdk-perf-bdev (SPDK bdev perf)" echo " - spdk-plugin-nvme (SPDK nvme fio plugin)" echo " - spdk-plugin-bdev (SPDK bdev fio plugin)" diff --git a/test/nvme/perf/run_perf.sh b/test/nvme/perf/run_perf.sh index bdae39417..98c921752 100755 --- a/test/nvme/perf/run_perf.sh +++ b/test/nvme/perf/run_perf.sh @@ -123,6 +123,17 @@ do run_bdevperf > $NVME_FIO_RESULTS iops_disks[$k]=$((${iops_disks[$k]} + $(get_bdevperf_results iops))) bw[$k]=$((${bw[$k]} + $(get_bdevperf_results bw_Kibs))) + cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output + elif [ $PLUGIN = "spdk-perf-nvme" ]; then + run_nvmeperf $k > $NVME_FIO_RESULTS + read -r iops bandwidth mean_lat min_lat max_lat <<< $(get_nvmeperf_results) + + iops_disks[$k]=$((${iops_disks[$k]} + iops)) + bw[$k]=$((${bw[$k]} + bandwidth)) + mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + mean_lat)) + min_lat_disks_usec[$k]=$((${min_lat_disks_usec[$k]} + min_lat)) + max_lat_disks_usec[$k]=$((${max_lat_disks_usec[$k]} + max_lat)) + cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output else create_fio_config $k $PLUGIN "$DISK_NAMES" "$DISKS_NUMA" "$CORES" @@ -178,20 +189,27 @@ for (( k=DISKNO; k >= 1; k-=2 )) do iops_disks[$k]=$((${iops_disks[$k]} / REPEAT_NO)) - if [[ $PLUGIN != "spdk-perf-bdev" ]]; then + if [[ "$PLUGIN" =~ "plugin" ]]; then mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO)) p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / REPEAT_NO)) p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / REPEAT_NO)) stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / REPEAT_NO)) mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / REPEAT_NO)) mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / REPEAT_NO)) - else + elif [[ "$PLUGIN" == "spdk-perf-bdev" ]]; then mean_lat_disks_usec[$k]=0 p99_lat_disks_usec[$k]=0 p99_99_lat_disks_usec[$k]=0 stdev_disks_usec[$k]=0 mean_slat_disks_usec[$k]=0 mean_clat_disks_usec[$k]=0 + elif [[ "$PLUGIN" == "spdk-perf-nvme" ]]; then + mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO)) + p99_lat_disks_usec[$k]=0 + p99_99_lat_disks_usec[$k]=0 + stdev_disks_usec[$k]=0 + mean_slat_disks_usec[$k]=0 + mean_clat_disks_usec[$k]=0 fi bw[$k]=$((${bw[$k]} / REPEAT_NO))