scripts/qat_setup: Add support for dh895xcc devices

These can be found under CYP platform.

To that end, refactor qat_setup.sh so it can support devices based on
their dedicated driver rather than the specific device ID - this will
allow for easier addition of new devices in the future.

Also, configure number of VFs based on total number given ctrl
supports - this is exactly what qat_service is doing while enabling
VFs.

Drop warning about old bug in the uio_pci_generic driver - in
practice we don't hit that under CI since quite a long time.

Slap errexit on top to make sure we exit immediately when writing
to sysfs fails.

Last but not least, run blockdev_crypto_qat test without any top
condition - if qat_setup.sh succeeds, the test should be able to pass.

Signed-off-by: Michal Berger <michal.berger@intel.com>
Change-Id: I37c4fd319ad7002017f9baf9fdcf3890429aac75
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17086
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Pawel Piatek <pawelx.piatek@intel.com>

autotest.sh

@@ -335,11 +335,8 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
 	if [ $SPDK_TEST_CRYPTO -eq 1 ]; then
 		run_test "blockdev_crypto_aesni" $rootdir/test/bdev/blockdev.sh "crypto_aesni"
 		run_test "blockdev_crypto_sw" $rootdir/test/bdev/blockdev.sh "crypto_sw"
-		# Proceed with the test only if QAT devices are in place
-		if [[ $(lspci -d:37c8) ]]; then
 		run_test "blockdev_crypto_qat" $rootdir/test/bdev/blockdev.sh "crypto_qat"
 	fi
-	fi
 
 	if [[ $SPDK_TEST_SCHEDULER -eq 1 ]]; then
 		run_test "scheduler" $rootdir/test/scheduler/scheduler.sh

scripts/qat_setup.sh

@@ -4,6 +4,7 @@
 #  All rights reserved.
 #
 shopt -s nullglob
+set -e
 
 rootdir=$(readlink -f $(dirname $0))/..
 allowed_drivers=("igb_uio" "uio_pci_generic")
@@ -58,9 +59,16 @@ reload_intel_qat() {
 
 bad_driver=true
 driver_to_bind=uio_pci_generic
-num_vfs=16
 
-qat_pci_bdfs=($(lspci -Dd:37c8 | awk '{print $1}'))
+reload_intel_qat
+
+_qat_pci_bdfs=(
+	/sys/bus/pci/drivers/c6xx/0000*
+	/sys/bus/pci/drivers/dh895xcc/0000*
+)
+
+qat_pci_bdfs=("${_qat_pci_bdfs[@]#*drivers/}")
+
 if [ ${#qat_pci_bdfs[@]} -eq 0 ]; then
 	echo "No QAT devices found. Exiting"
 	exit 1
@@ -82,45 +90,39 @@ if $bad_driver; then
 	exit 1
 fi
 
-reload_intel_qat
-
 # try starting the qat service. If this doesn't work, just treat it as a warning for now.
 if ! service qat_service start; then
 	echo "failed to start the qat service. Something may be wrong with your 01.org driver."
 fi
 
+expected_num_vfs=0 set_num_vfs=0
 # configure virtual functions for the QAT cards.
 for qat_bdf in "${qat_pci_bdfs[@]}"; do
-	if [[ ! -e /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs ]]; then
+	if [[ ! -e /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs ]]; then
-		echo "($qat_bdf) sriov_numvfs interface missing, is SR-IOV enabled?"
+		echo "(${qat_bdf##*/}) sriov_numvfs interface missing, is SR-IOV enabled?"
 		continue
 	fi
-	echo "$num_vfs" > /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs
+	num_vfs=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_totalvfs")
-	num_vfs_set=$(cat /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs)
+	echo "$num_vfs" > /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs
+	num_vfs_set=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_numvfs")
 	if ((num_vfs != num_vfs_set)); then
 		echo "Number of VFs set to $num_vfs_set, expected $num_vfs"
 	else
-		echo "$qat_bdf set to $num_vfs VFs"
+		echo "${qat_bdf##*/} set to $num_vfs VFs"
 	fi
+	((expected_num_vfs += num_vfs))
+	((set_num_vfs += num_vfs_set))
 done
 
-# Confirm we have all of the virtual functions we asked for.
+# Build VF list out of qat_pci_bdfs[@] by slapping /virtfn* to the path to know if we got'em all.
+# shellcheck disable=SC2206 # <- intentional
+qat_vf_bdfs=(${_qat_pci_bdfs[@]/%//virtfn*})
 
-qat_vf_bdfs=($(lspci -Dd:37c9 | awk '{print $1}'))
+if ((expected_num_vfs != set_num_vfs || ${#qat_vf_bdfs[@]} != expected_num_vfs)); then
-if ((${#qat_vf_bdfs[@]} != ${#qat_pci_bdfs[@]} * num_vfs)); then
+	echo "Failed to prepare the VFs. Aborting" >&2
-	echo "Failed to prepare the VFs. Aborting"
 	exit 1
 fi
 
-# Unbind old driver if necessary.
-for vf in "${qat_vf_bdfs[@]}"; do
-	old_driver=$(basename $(readlink -f /sys/bus/pci/devices/${vf}/driver))
-	if [ $old_driver != "driver" ]; then
-		echo "unbinding driver $old_driver from qat VF at BDF $vf"
-		echo -n $vf > /sys/bus/pci/drivers/$old_driver/unbind
-	fi
-done
-
 modprobe uio
 
 # Insert the dpdk uio kernel module.
@@ -136,18 +138,18 @@ else
 	exit 1
 fi
 
-echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/remove_id 2> /dev/null
+# Unbind old driver if necessary.
-echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/new_id
 for vf in "${qat_vf_bdfs[@]}"; do
-	if ! ls -l /sys/bus/pci/devices/$vf/driver | grep -q $driver_to_bind; then
+	vf=$(readlink -f "$vf")
-		echo "unable to bind the driver to the device at bdf $vf"
+	if [[ -e $vf/driver ]]; then
-		if [ "$driver_to_bind" == "uio_pci_generic" ]; then
+		old_driver=$(basename "$(readlink -f "$vf/driver")")
-			echo "Your kernel's uio_pci_generic module does not support binding to virtual functions."
+		if [[ $old_driver != "$driver_to_bind" ]]; then
-			echo "It likely is missing Linux git commit ID acec09e67 which is needed to bind"
+			echo "unbinding driver $old_driver from qat VF at BDF ${vf##*/}"
-			echo "uio_pci_generic to virtual functions which have no legacy interrupt vector."
+			echo "${vf##*/}" > "/sys/bus/pci/drivers/$old_driver/unbind"
-			echo "Please build DPDK kernel module for igb_uio and re-run this script specifying the igb_uio driver."
 		fi
-		exit 1
 	fi
+	echo "$driver_to_bind" > "$vf/driver_override"
+	echo "${vf##*/}" > /sys/bus/pci/drivers_probe
 done
+
 echo "Properly configured the qat device with driver $driver_to_bind."