test/nvme_perf: use config file to select test disks

Use a simple config file to select NVMes for test
instead of selecting them all via gen_nvme.sh.
HW platforms may differ significantly in setup and
capabilities, so using all disks in sequential order
is not always a good way.
(e.g. using "first" four available NVMes for test
may yield not optimal results if it turns out that
NVMes are connected to oversubscibed PCIe switch)

Change-Id: I55dd85799a5859c4764e94e1d8058e01ad9b84f0
Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3296
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

test/nvme/perf/common.sh

@@ -28,7 +28,8 @@ NUMJOBS=1
 REPEAT_NO=3
 FIO_BIN=$CONFIG_FIO_SOURCE_DIR/fio
 PLUGIN="nvme"
-DISKNO=1
+DISKCFG=""
+DISKNO="ALL"
 CPUS_ALLOWED=1
 NOIOSCALING=false
 PRECONDITIONING=true
@@ -70,6 +71,44 @@ function discover_bdevs() {
 	rm -f /var/run/spdk_bdev0
 }
 
+function create_spdk_bdev_conf() {
+	local output
+	local disk_cfg
+	local bdev_json_cfg
+
+	disk_cfg=($(grep -vP "^\s*#" "$DISKCFG"))
+
+	bdev_json_cfg=()
+	for i in "${!disk_cfg[@]}"; do
+		bdev_json_cfg+=("$(
+			cat <<- JSON
+				{
+					"method": "bdev_nvme_attach_controller",
+					"params": {
+						"trtype": "PCIe",
+						"name":"Nvme${i}",
+						"traddr":"${disk_cfg[i]}"
+					}
+				}
+			JSON
+		)")
+	done
+
+	local IFS=","
+	jq -r '.' <<- JSON > $BASE_DIR/bdev.conf
+		{
+			"subsystems": [
+				{
+					"subsystem": "bdev",
+					"config": [
+						${bdev_json_cfg[*]}
+					]
+				}
+			]
+		}
+	JSON
+}
+
 function is_bdf_not_mounted() {
 	local bdf=$1
 	local blkname
@@ -114,28 +153,36 @@ function get_numa_node() {
 			cat /sys/bus/pci/devices/$bdev_bdf/numa_node
 		done
 	else
-		# Only target not mounted NVMes
+		for name in $disks; do
-		for bdf in $(get_nvme_bdfs); do
+			local bdf
-			if is_bdf_not_mounted $bdf; then
+			# Not reading directly from /sys/block/nvme* because of a kernel bug
-				cat /sys/bus/pci/devices/$bdf/numa_node
+			# which results in NUMA 0 always getting reported.
-			fi
+			bdf=$(cat /sys/block/$name/device/address)
+			cat /sys/bus/pci/devices/$bdf/numa_node
 		done
 	fi
 }
 
 function get_disks() {
 	local plugin=$1
+	local disk_cfg
+
+	disk_cfg=($(grep -vP "^\s*#" "$DISKCFG"))
 	if [[ "$plugin" =~ "nvme" ]]; then
-		for bdf in $(get_nvme_bdfs); do
+		# PCI BDF address is enough for nvme-perf and nvme-fio-plugin,
-			echo "$bdf"
+		# so just print them from configuration file
-		done
+		echo "${disk_cfg[*]}"
 	elif [[ "$plugin" =~ "bdev" ]]; then
+		# Generate NvmeXn1 bdev name configuration file for bdev-perf
+		# and bdev-fio-plugin
 		local bdevs
-		bdevs=$(discover_bdevs $ROOT_DIR $BASE_DIR/bdev.conf --json)
+		local disk_no
-		jq -r '.[].name' <<< $bdevs
+		disk_no=${#disk_cfg[@]}
+		eval echo "Nvme{0..$((disk_no - 1))}n1"
 	else
-		# Only target not mounted NVMes
+		# Find nvme block devices and only use the ones which
-		for bdf in $(get_nvme_bdfs); do
+		# are not mounted
+		for bdf in "${disk_cfg[@]}"; do
 			if is_bdf_not_mounted $bdf; then
 				local blkname
 				blkname=$(ls -l /sys/block/ | grep $bdf | awk '{print $9}')
@@ -403,14 +450,6 @@ function wait_for_nvme_reload() {
 
 function verify_disk_number() {
 	# Check if we have appropriate number of disks to carry out the test
-	if [[ "$PLUGIN" =~ "bdev" ]]; then
-		cat <<- JSON > "$BASE_DIR/bdev.conf"
-			{"subsystems":[
-			$("$ROOT_DIR/scripts/gen_nvme.sh" --json)
-			]}
-		JSON
-	fi
-
 	disks=($(get_disks $PLUGIN))
 	if [[ $DISKNO == "ALL" ]] || [[ $DISKNO == "all" ]]; then
 		DISKNO=${#disks[@]}
@@ -455,6 +494,10 @@ function usage() {
 	echo "                             - kernel-hybrid-polling"
 	echo "                             - kernel-libaio"
 	echo "                             - kernel-io-uring"
+	echo "    --disk-config         Configuration file containing PCI BDF addresses of NVMe disks to use in test."
+	echo "                          It consists a single column of PCI addresses. SPDK Bdev names will be assigned"
+	echo "                          and Kernel block device names detected."
+	echo "                          Lines starting with # are ignored as comments."
 	echo "    --max-disk=INT,ALL    Number of disks to test on, this will run multiple workloads with increasing number of disk each run."
 	echo "                          If =ALL then test on all found disk. [default=$DISKNO]"
 	echo "    --cpu-allowed=INT     Comma-separated list of CPU cores used to run the workload. [default=$CPUS_ALLOWED]"
@@ -481,6 +524,13 @@ while getopts 'h-:' optchar; do
 				repeat-no=*) REPEAT_NO="${OPTARG#*=}" ;;
 				fio-bin=*) FIO_BIN="${OPTARG#*=}" ;;
 				driver=*) PLUGIN="${OPTARG#*=}" ;;
+				disk-config=*)
+					DISKCFG="${OPTARG#*=}"
+					if [[ ! -f "$DISKCFG" ]]; then
+						echo "Disk confiuration file $DISKCFG does not exist!"
+						exit 1
+					fi
+					;;
 				max-disk=*) DISKNO="${OPTARG#*=}" ;;
 				cpu-allowed=*) CPUS_ALLOWED="${OPTARG#*=}" ;;
 				no-preconditioning) PRECONDITIONING=false ;;

test/nvme/perf/run_perf.sh

@@ -35,8 +35,11 @@ BASE_DIR=$(readlink -f $(dirname $0))
 . $BASE_DIR/common.sh
 
 trap 'rm -f *.state $BASE_DIR/bdev.conf; print_backtrace' ERR SIGTERM SIGABRT
-verify_disk_number
 
+if [[ "$PLUGIN" =~ "bdev" ]]; then
+	create_spdk_bdev_conf
+fi
+verify_disk_number
 DISK_NAMES=$(get_disks $PLUGIN)
 DISKS_NUMA=$(get_numa_node $PLUGIN "$DISK_NAMES")
 CORES=$(get_cores "$CPUS_ALLOWED")