diff --git a/autotest.sh b/autotest.sh index 65e7098f8..b2a785e98 100755 --- a/autotest.sh +++ b/autotest.sh @@ -335,10 +335,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then if [ $SPDK_TEST_CRYPTO -eq 1 ]; then run_test "blockdev_crypto_aesni" $rootdir/test/bdev/blockdev.sh "crypto_aesni" run_test "blockdev_crypto_sw" $rootdir/test/bdev/blockdev.sh "crypto_sw" - # Proceed with the test only if QAT devices are in place - if [[ $(lspci -d:37c8) ]]; then - run_test "blockdev_crypto_qat" $rootdir/test/bdev/blockdev.sh "crypto_qat" - fi + run_test "blockdev_crypto_qat" $rootdir/test/bdev/blockdev.sh "crypto_qat" fi if [[ $SPDK_TEST_SCHEDULER -eq 1 ]]; then diff --git a/scripts/qat_setup.sh b/scripts/qat_setup.sh index 3250e5ef8..83d5caad4 100755 --- a/scripts/qat_setup.sh +++ b/scripts/qat_setup.sh @@ -4,6 +4,7 @@ # All rights reserved. # shopt -s nullglob +set -e rootdir=$(readlink -f $(dirname $0))/.. allowed_drivers=("igb_uio" "uio_pci_generic") @@ -58,9 +59,16 @@ reload_intel_qat() { bad_driver=true driver_to_bind=uio_pci_generic -num_vfs=16 -qat_pci_bdfs=($(lspci -Dd:37c8 | awk '{print $1}')) +reload_intel_qat + +_qat_pci_bdfs=( + /sys/bus/pci/drivers/c6xx/0000* + /sys/bus/pci/drivers/dh895xcc/0000* +) + +qat_pci_bdfs=("${_qat_pci_bdfs[@]#*drivers/}") + if [ ${#qat_pci_bdfs[@]} -eq 0 ]; then echo "No QAT devices found. Exiting" exit 1 @@ -82,45 +90,39 @@ if $bad_driver; then exit 1 fi -reload_intel_qat - # try starting the qat service. If this doesn't work, just treat it as a warning for now. if ! service qat_service start; then echo "failed to start the qat service. Something may be wrong with your 01.org driver." fi +expected_num_vfs=0 set_num_vfs=0 # configure virtual functions for the QAT cards. for qat_bdf in "${qat_pci_bdfs[@]}"; do - if [[ ! -e /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs ]]; then - echo "($qat_bdf) sriov_numvfs interface missing, is SR-IOV enabled?" + if [[ ! -e /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs ]]; then + echo "(${qat_bdf##*/}) sriov_numvfs interface missing, is SR-IOV enabled?" continue fi - echo "$num_vfs" > /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs - num_vfs_set=$(cat /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs) + num_vfs=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_totalvfs") + echo "$num_vfs" > /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs + num_vfs_set=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_numvfs") if ((num_vfs != num_vfs_set)); then echo "Number of VFs set to $num_vfs_set, expected $num_vfs" else - echo "$qat_bdf set to $num_vfs VFs" + echo "${qat_bdf##*/} set to $num_vfs VFs" fi + ((expected_num_vfs += num_vfs)) + ((set_num_vfs += num_vfs_set)) done -# Confirm we have all of the virtual functions we asked for. +# Build VF list out of qat_pci_bdfs[@] by slapping /virtfn* to the path to know if we got'em all. +# shellcheck disable=SC2206 # <- intentional +qat_vf_bdfs=(${_qat_pci_bdfs[@]/%//virtfn*}) -qat_vf_bdfs=($(lspci -Dd:37c9 | awk '{print $1}')) -if ((${#qat_vf_bdfs[@]} != ${#qat_pci_bdfs[@]} * num_vfs)); then - echo "Failed to prepare the VFs. Aborting" +if ((expected_num_vfs != set_num_vfs || ${#qat_vf_bdfs[@]} != expected_num_vfs)); then + echo "Failed to prepare the VFs. Aborting" >&2 exit 1 fi -# Unbind old driver if necessary. -for vf in "${qat_vf_bdfs[@]}"; do - old_driver=$(basename $(readlink -f /sys/bus/pci/devices/${vf}/driver)) - if [ $old_driver != "driver" ]; then - echo "unbinding driver $old_driver from qat VF at BDF $vf" - echo -n $vf > /sys/bus/pci/drivers/$old_driver/unbind - fi -done - modprobe uio # Insert the dpdk uio kernel module. @@ -136,18 +138,18 @@ else exit 1 fi -echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/remove_id 2> /dev/null -echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/new_id +# Unbind old driver if necessary. for vf in "${qat_vf_bdfs[@]}"; do - if ! ls -l /sys/bus/pci/devices/$vf/driver | grep -q $driver_to_bind; then - echo "unable to bind the driver to the device at bdf $vf" - if [ "$driver_to_bind" == "uio_pci_generic" ]; then - echo "Your kernel's uio_pci_generic module does not support binding to virtual functions." - echo "It likely is missing Linux git commit ID acec09e67 which is needed to bind" - echo "uio_pci_generic to virtual functions which have no legacy interrupt vector." - echo "Please build DPDK kernel module for igb_uio and re-run this script specifying the igb_uio driver." + vf=$(readlink -f "$vf") + if [[ -e $vf/driver ]]; then + old_driver=$(basename "$(readlink -f "$vf/driver")") + if [[ $old_driver != "$driver_to_bind" ]]; then + echo "unbinding driver $old_driver from qat VF at BDF ${vf##*/}" + echo "${vf##*/}" > "/sys/bus/pci/drivers/$old_driver/unbind" fi - exit 1 fi + echo "$driver_to_bind" > "$vf/driver_override" + echo "${vf##*/}" > /sys/bus/pci/drivers_probe done + echo "Properly configured the qat device with driver $driver_to_bind."