test/vhost: Gather IRQ stats from the VM

Signed-off-by: Michal Berger <michal.berger@intel.com>
Change-Id: I4351d812b9b9da127b6daf46b0f44ce237e33ee9
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15460
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

test/vhost/common.sh

@@ -1289,3 +1289,57 @@ function error_exit() {
 	at_app_exit
 	exit 1
 }
+
+function lookup_dev_irqs() {
+	local vm=$1 irqs=() cpus=()
+	local script_get_irqs script_get_cpus
+
+	mkdir -p "$VHOST_DIR/irqs"
+
+	# All vhost tests depend either on virtio_blk or virtio_scsi drivers on the VM side.
+	# Considering that, simply iterate over virtio bus and pick pci device corresponding
+	# to each virtio device.
+	# For vfio-user setup, look for bare nvme devices.
+
+	script_get_irqs=$(
+		cat <<- 'SCRIPT'
+			shopt -s nullglob
+			for virtio in /sys/bus/virtio/devices/virtio*; do
+			  irqs+=("$(readlink -f "$virtio")/../msi_irqs/"*)
+			done
+			irqs+=(/sys/class/nvme/nvme*/device/msi_irqs/*)
+			printf '%u\n' "${irqs[@]##*/}"
+		SCRIPT
+	)
+
+	script_get_cpus=$(
+		cat <<- 'SCRIPT'
+			cpus=(/sys/devices/system/cpu/cpu[0-9]*)
+			printf '%u\n' "${cpus[@]##*cpu}"
+		SCRIPT
+	)
+
+	irqs=($(vm_exec "$vm" "$script_get_irqs"))
+	cpus=($(vm_exec "$vm" "$script_get_cpus"))
+	((${#irqs[@]} > 0 && ${#cpus[@]} > 0))
+
+	printf '%u\n' "${irqs[@]}" > "$VHOST_DIR/irqs/$vm.irqs"
+	printf '%u\n' "${cpus[@]}" > "$VHOST_DIR/irqs/$vm.cpus"
+}
+
+function irqs() {
+	local vm
+	for vm; do
+		vm_exec "$vm" "while :; do cat /proc/interrupts; sleep 1s; done" > "$VHOST_DIR/irqs/$vm.interrupts" &
+		irqs_pids+=($!)
+	done
+}
+
+function parse_irqs() {
+	local iter=${1:-1}
+	"$rootdir/test/vhost/parse_irqs.sh" "$VHOST_DIR/irqs/"*.interrupts
+	rm "$VHOST_DIR/irqs/"*.interrupts
+
+	mkdir -p "$VHOST_DIR/irqs/$iter"
+	mv "$VHOST_DIR/irqs/"*.parsed "$VHOST_DIR/irqs/$iter/"
+}

test/vhost/parse_irqs.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#  SPDX-License-Identifier: BSD-3-Clause
+#  Copyright (C) 2022 Intel Corporation.
+#  All rights reserved.
+
+curdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$curdir/../../")
+set -e
+
+source "$curdir/irqs.sh"
+
+vm_interrupts=("$@")
+SHOW_ALL_IRQS=yes
+
+for interrupt in "${vm_interrupts[@]}"; do
+	reset_irqs
+	irqs=${interrupt%.*}.irqs
+	cpus=${interrupt%.*}.cpus
+	[[ -e $irqs ]]
+	[[ -e $cpus ]]
+	for irq in $(< "$irqs"); do
+		irqs_to_lookup[irq]=$irq
+	done
+	cpus_override=($(< "$cpus"))
+	update_irqs_procfs "$interrupt"
+	get_irqs "${irqs_to_lookup[@]}" > "$interrupt.parsed"
+done

test/vhost/perf_bench/vhost_perf.sh

@@ -32,6 +32,8 @@ fio_iterations=1
 fio_gtod=""
 precond_fio_bin=$CONFIG_FIO_SOURCE_DIR/fio
 disk_map=""
+enable_irq=0
+irqs_pids=()
 
 disk_cfg_bdfs=()
 disk_cfg_spdk_names=()
@@ -88,6 +90,7 @@ function usage() {
 	echo "    --iobuf-small-pool-count=INT   number of small buffers in the global pool"
 	echo "    --iobuf-large-pool-count=INT   number of large buffers in the global pool"
 	echo "-x                          set -x for script debug"
+	echo "-i                          Collect IRQ stats from each VM"
 	exit 0
 }
 
@@ -174,7 +177,7 @@ function create_spdk_controller() {
 	fi
 }
 
-while getopts 'xh-:' optchar; do
+while getopts 'xhi-:' optchar; do
 	case "$optchar" in
 		-)
 			case "$OPTARG" in
@@ -210,6 +213,7 @@ while getopts 'xh-:' optchar; do
 			set -x
 			x="-x"
 			;;
+		i) enable_irq=1 ;;
 		*) usage $0 "Invalid argument '$OPTARG'" ;;
 	esac
 done
@@ -446,6 +450,7 @@ fi
 
 # Run FIO
 fio_disks=""
+
 for vm_num in $used_vms; do
 	host_name="VM-$vm_num"
 	vm_exec $vm_num "hostname $host_name"
@@ -484,10 +489,12 @@ for vm_num in $used_vms; do
 	fi
 
 	fio_disks+=" --vm=${vm_num}$(printf ':/dev/%s' $SCSI_DISK)"
+	((enable_irq == 1)) && lookup_dev_irqs "$vm_num"
 done
 
 # Run FIO traffic
 for fio_job in ${fio_jobs//,/ }; do
+	((enable_irq == 1)) && irqs $used_vms
 	fio_job_fname=$(basename $fio_job)
 	fio_log_fname="${fio_job_fname%%.*}.log"
 	for i in $(seq 1 $fio_iterations); do
@@ -528,7 +535,10 @@ for fio_job in ${fio_jobs//,/ }; do
 		sleep 1
 	done
 
+	((enable_irq == 1)) && kill "${irqs_pids[@]}"
+
 	parse_fio_results "$VHOST_DIR/fio_results" "$fio_log_fname"
+	((enable_irq == 1)) && parse_irqs $((++iter))
 done
 
 notice "Shutting down virtual machines..."