test/nvme-perf: add nvme perf option

Allow running tests with nvme/perf. This will be needed
for comparison with bdevperf and/or fio nvme plugin.

Change-Id: Ie351d1b3c9d8eae430161779f9818c04d87e6ed7
Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/479821
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>

test/nvme/perf/common.sh

@@ -6,6 +6,7 @@ ROOT_DIR=$(readlink -f $BASE_DIR/../../..)
 PLUGIN_DIR_NVME=$ROOT_DIR/examples/nvme/fio_plugin
 PLUGIN_DIR_BDEV=$ROOT_DIR/examples/bdev/fio_plugin
 BDEVPERF_DIR=$ROOT_DIR/test/bdev/bdevperf
+NVMEPERF_DIR=$ROOT_DIR/examples/nvme/perf
 . $ROOT_DIR/scripts/common.sh || exit 1
 . $ROOT_DIR/test/common/autotest_common.sh
 NVME_FIO_RESULTS=$BASE_DIR/result.json
@@ -53,7 +54,7 @@ function get_cores_numa_node(){
 function get_numa_node(){
 	local plugin=$1
 	local disks=$2
-	if [[ "$plugin" == "spdk-plugin-nvme" ]]; then
+	if [[ "$plugin" =~ "nvme" ]]; then
 		for bdf in $disks; do
 			local driver
 			driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}')
@@ -82,7 +83,7 @@ function get_numa_node(){
 
 function get_disks(){
 	local plugin=$1
-	if [[ "$plugin" == "spdk-plugin-nvme" ]]; then
+	if [[ "$plugin" =~ "nvme" ]]; then
 		for bdf in $(iter_pci_class_code 01 08 02); do
 			driver=$(grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}')
 			if [ "$driver" = "vfio-pci" ] || [ "$driver" = "uio_pci_generic" ]; then
@@ -288,6 +289,26 @@ function get_bdevperf_results(){
 	esac
 }
 
+function get_nvmeperf_results() {
+	local iops
+	local bw_MBs
+	local mean_lat_usec
+	local max_lat_usec
+	local min_lat_usec
+
+	read -r iops bw_MBs mean_lat_usec min_lat_usec max_lat_usec<<< $(cat $NVME_FIO_RESULTS | tr -s " " | grep -oP "(?<=Total : )(.*+)")
+
+	# We need to get rid of the decimal spaces due
+	# to use of arithmetic expressions instead of "bc" for calculations
+	iops=${iops%.*}
+	bw_MBs=${bw_MBs%.*}
+	mean_lat_usec=${mean_lat_usec%.*}
+	min_lat_usec=${min_lat_usec%.*}
+	max_lat_usec=${max_lat_usec%.*}
+
+	echo "$iops $(bc <<< "$bw_MBs * 1024") $mean_lat_usec $min_lat_usec $max_lat_usec"
+}
+
 function run_spdk_nvme_fio(){
 	local plugin=$1
 	echo "** Running fio test, this can take a while, depending on the run-time and ramp-time setting."
@@ -314,6 +335,23 @@ function run_bdevperf(){
 	sleep 1
 }
 
+function run_nvmeperf() {
+	# Prepare -r argument string for nvme perf command
+	local r_opt
+	local disks
+
+	# Limit the number of disks to $1 if needed
+	disks=( $(get_disks nvme) )
+	disks=( "${disks[@]:0:$1}" )
+	r_opt=$(printf -- ' -r "trtype:PCIe traddr:%s"' "${disks[@]}")
+
+	echo "** Running nvme perf test, this can take a while, depending on the run-time setting."
+
+	# Run command in separate shell as this solves quoting issues related to r_opt var
+	$SHELL -c "$NVMEPERF_DIR/perf $r_opt -q $IODEPTH -o $BLK_SIZE -w $RW -M $MIX -t $RUNTIME -c [$CPUS_ALLOWED]"
+	sleep 1
+}
+
 function wait_for_nvme_reload() {
 	local nvmes=$1
 
@@ -368,6 +406,7 @@ function usage()
 	echo
 	echo "Test setup parameters:"
 	echo "    --driver=STR          Selects tool used for testing. Choices available:"
+	echo "                             - spdk-perf-nvme (SPDK nvme perf)"
 	echo "                             - spdk-perf-bdev (SPDK bdev perf)"
 	echo "                             - spdk-plugin-nvme (SPDK nvme fio plugin)"
 	echo "                             - spdk-plugin-bdev (SPDK bdev fio plugin)"

test/nvme/perf/run_perf.sh

@@ -123,6 +123,17 @@ do
 			run_bdevperf > $NVME_FIO_RESULTS
 			iops_disks[$k]=$((${iops_disks[$k]} + $(get_bdevperf_results iops)))
 			bw[$k]=$((${bw[$k]} + $(get_bdevperf_results bw_Kibs)))
+			cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output
+		elif [ $PLUGIN = "spdk-perf-nvme" ]; then
+			run_nvmeperf $k > $NVME_FIO_RESULTS
+			read -r iops bandwidth mean_lat min_lat max_lat <<< $(get_nvmeperf_results)
+
+			iops_disks[$k]=$((${iops_disks[$k]} + iops))
+			bw[$k]=$((${bw[$k]} + bandwidth))
+			mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} + mean_lat))
+			min_lat_disks_usec[$k]=$((${min_lat_disks_usec[$k]} + min_lat))
+			max_lat_disks_usec[$k]=$((${max_lat_disks_usec[$k]} + max_lat))
+
 			cp $NVME_FIO_RESULTS $BASE_DIR/results/$result_dir/perf_results_${MIX}_${PLUGIN}_${NO_CORES}cpus_${DATE}_${k}_disks_${j}.output
 		else
 			create_fio_config $k $PLUGIN "$DISK_NAMES" "$DISKS_NUMA" "$CORES"
@@ -178,20 +189,27 @@ for (( k=DISKNO; k >= 1; k-=2 ))
 do
 	iops_disks[$k]=$((${iops_disks[$k]} / REPEAT_NO))
 
-	if [[ $PLUGIN != "spdk-perf-bdev" ]]; then
+	if [[ "$PLUGIN" =~ "plugin" ]]; then
 		mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO))
 		p99_lat_disks_usec[$k]=$((${p99_lat_disks_usec[$k]} / REPEAT_NO))
 		p99_99_lat_disks_usec[$k]=$((${p99_99_lat_disks_usec[$k]} / REPEAT_NO))
 		stdev_disks_usec[$k]=$((${stdev_disks_usec[$k]} / REPEAT_NO))
 		mean_slat_disks_usec[$k]=$((${mean_slat_disks_usec[$k]} / REPEAT_NO))
 		mean_clat_disks_usec[$k]=$((${mean_clat_disks_usec[$k]} / REPEAT_NO))
-	else
+	elif [[ "$PLUGIN" == "spdk-perf-bdev" ]]; then
 		mean_lat_disks_usec[$k]=0
 		p99_lat_disks_usec[$k]=0
 		p99_99_lat_disks_usec[$k]=0
 		stdev_disks_usec[$k]=0
 		mean_slat_disks_usec[$k]=0
 		mean_clat_disks_usec[$k]=0
+	elif [[ "$PLUGIN" == "spdk-perf-nvme" ]]; then
+		mean_lat_disks_usec[$k]=$((${mean_lat_disks_usec[$k]} / REPEAT_NO))
+		p99_lat_disks_usec[$k]=0
+		p99_99_lat_disks_usec[$k]=0
+		stdev_disks_usec[$k]=0
+		mean_slat_disks_usec[$k]=0
+		mean_clat_disks_usec[$k]=0
 	fi
 
 	bw[$k]=$((${bw[$k]} / REPEAT_NO))