ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scripts/qat_setup: Add support for dh895xcc devices

Browse Source 
These can be found under CYP platform.

To that end, refactor qat_setup.sh so it can support devices based on
their dedicated driver rather than the specific device ID - this will
allow for easier addition of new devices in the future.

Also, configure number of VFs based on total number given ctrl
supports - this is exactly what qat_service is doing while enabling
VFs.

Drop warning about old bug in the uio_pci_generic driver - in
practice we don't hit that under CI since quite a long time.

Slap errexit on top to make sure we exit immediately when writing
to sysfs fails.

Last but not least, run blockdev_crypto_qat test without any top
condition - if qat_setup.sh succeeds, the test should be able to pass.

Signed-off-by: Michal Berger <michal.berger@intel.com>
Change-Id: I37c4fd319ad7002017f9baf9fdcf3890429aac75
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17086
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Pawel Piatek <pawelx.piatek@intel.com>
This commit is contained in:
Michal Berger 2023-03-08 11:00:04 +01:00 committed by David Ko
parent 8d3118f156
commit 6b7ec13c8f
2 changed files with 35 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
autotest.sh
View File

@ -335,10 +335,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
if [ $SPDK_TEST_CRYPTO -eq 1 ]; then
run_test "blockdev_crypto_aesni" $rootdir/test/bdev/blockdev.sh "crypto_aesni"
run_test "blockdev_crypto_sw" $rootdir/test/bdev/blockdev.sh "crypto_sw"
# Proceed with the test only if QAT devices are in place
if [[ $(lspci -d:37c8) ]]; then
run_test "blockdev_crypto_qat" $rootdir/test/bdev/blockdev.sh "crypto_qat"
fi
run_test "blockdev_crypto_qat" $rootdir/test/bdev/blockdev.sh "crypto_qat"
fi
if [[ $SPDK_TEST_SCHEDULER -eq 1 ]]; then

66
scripts/qat_setup.sh
View File

@ -4,6 +4,7 @@
# All rights reserved.
#
shopt -s nullglob
set -e
rootdir=$(readlink -f $(dirname $0))/..
allowed_drivers=("igb_uio" "uio_pci_generic")
@ -58,9 +59,16 @@ reload_intel_qat() {
bad_driver=true
driver_to_bind=uio_pci_generic
num_vfs=16
qat_pci_bdfs=($(lspci -Dd:37c8 | awk '{print $1}'))
reload_intel_qat
_qat_pci_bdfs=(
/sys/bus/pci/drivers/c6xx/0000*
/sys/bus/pci/drivers/dh895xcc/0000*
)
qat_pci_bdfs=("${_qat_pci_bdfs[@]#*drivers/}")
if [ ${#qat_pci_bdfs[@]} -eq 0 ]; then
echo "No QAT devices found. Exiting"
exit 1
@ -82,45 +90,39 @@ if $bad_driver; then
exit 1
fi
reload_intel_qat
# try starting the qat service. If this doesn't work, just treat it as a warning for now.
if ! service qat_service start; then
echo "failed to start the qat service. Something may be wrong with your 01.org driver."
fi
expected_num_vfs=0 set_num_vfs=0
# configure virtual functions for the QAT cards.
for qat_bdf in "${qat_pci_bdfs[@]}"; do
if [[ ! -e /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs ]]; then
echo "($qat_bdf) sriov_numvfs interface missing, is SR-IOV enabled?"
if [[ ! -e /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs ]]; then
echo "(${qat_bdf##*/}) sriov_numvfs interface missing, is SR-IOV enabled?"
continue
fi
echo "$num_vfs" > /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs
num_vfs_set=$(cat /sys/bus/pci/drivers/c6xx/$qat_bdf/sriov_numvfs)
num_vfs=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_totalvfs")
echo "$num_vfs" > /sys/bus/pci/drivers/$qat_bdf/sriov_numvfs
num_vfs_set=$(< "/sys/bus/pci/drivers/$qat_bdf/sriov_numvfs")
if ((num_vfs != num_vfs_set)); then
echo "Number of VFs set to $num_vfs_set, expected $num_vfs"
else
echo "$qat_bdf set to $num_vfs VFs"
echo "${qat_bdf##*/} set to $num_vfs VFs"
fi
((expected_num_vfs += num_vfs))
((set_num_vfs += num_vfs_set))
done
# Confirm we have all of the virtual functions we asked for.
# Build VF list out of qat_pci_bdfs[@] by slapping /virtfn* to the path to know if we got'em all.
# shellcheck disable=SC2206 # <- intentional
qat_vf_bdfs=(${_qat_pci_bdfs[@]/%//virtfn*})
qat_vf_bdfs=($(lspci -Dd:37c9 | awk '{print $1}'))
if ((${#qat_vf_bdfs[@]} != ${#qat_pci_bdfs[@]} * num_vfs)); then
echo "Failed to prepare the VFs. Aborting"
if ((expected_num_vfs != set_num_vfs || ${#qat_vf_bdfs[@]} != expected_num_vfs)); then
echo "Failed to prepare the VFs. Aborting" >&2
exit 1
fi
# Unbind old driver if necessary.
for vf in "${qat_vf_bdfs[@]}"; do
old_driver=$(basename $(readlink -f /sys/bus/pci/devices/${vf}/driver))
if [ $old_driver != "driver" ]; then
echo "unbinding driver $old_driver from qat VF at BDF $vf"
echo -n $vf > /sys/bus/pci/drivers/$old_driver/unbind
fi
done
modprobe uio
# Insert the dpdk uio kernel module.
@ -136,18 +138,18 @@ else
exit 1
fi
echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/remove_id 2> /dev/null
echo -n "8086 37c9" > /sys/bus/pci/drivers/$driver_to_bind/new_id
# Unbind old driver if necessary.
for vf in "${qat_vf_bdfs[@]}"; do
if ! ls -l /sys/bus/pci/devices/$vf/driver | grep -q $driver_to_bind; then
echo "unable to bind the driver to the device at bdf $vf"
if [ "$driver_to_bind" == "uio_pci_generic" ]; then
echo "Your kernel's uio_pci_generic module does not support binding to virtual functions."
echo "It likely is missing Linux git commit ID acec09e67 which is needed to bind"
echo "uio_pci_generic to virtual functions which have no legacy interrupt vector."
echo "Please build DPDK kernel module for igb_uio and re-run this script specifying the igb_uio driver."
vf=$(readlink -f "$vf")
if [[ -e $vf/driver ]]; then
old_driver=$(basename "$(readlink -f "$vf/driver")")
if [[ $old_driver != "$driver_to_bind" ]]; then
echo "unbinding driver $old_driver from qat VF at BDF ${vf##*/}"
echo "${vf##*/}" > "/sys/bus/pci/drivers/$old_driver/unbind"
fi
exit 1
fi
echo "$driver_to_bind" > "$vf/driver_override"
echo "${vf##*/}" > /sys/bus/pci/drivers_probe
done
echo "Properly configured the qat device with driver $driver_to_bind."