Add vfio support to scripts.
cleanup.sh and unbind.sh have been combined into a single setup.sh that takes one optional parameter (reset). If no parameter is given, the script will automatically bind all NVMe and IOAT devices to either uio_pci_generic or vfio-pci, as appropriate based on IOMMU settings. If the reset parameter is given, the devices will be bound back to the appropriate kernel drivers. Change-Id: I25db3234f1ecfb352a281e5093f4c1aa455152ae Signed-off-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Ben Walker
parent
c0a1cd3827
commit
55dc5f2131
@ -88,4 +88,4 @@ any NVMe and I/OAT devices must be unbound from the native kernel drivers.
|
||||
SPDK includes scripts to automate this process on both Linux and FreeBSD.
|
||||
|
||||
1) scripts/configure_hugepages.sh
|
||||
2) scripts/unbind.sh
|
||||
2) scripts/setup.sh
|
||||
|
||||
@ -10,7 +10,7 @@ if [ $EUID -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
trap "process_core; $rootdir/scripts/cleanup.sh; exit 1" SIGINT SIGTERM EXIT
|
||||
trap "process_core; $rootdir/scripts/setup.sh reset; exit 1" SIGINT SIGTERM EXIT
|
||||
|
||||
timing_enter autotest
|
||||
|
||||
@ -38,7 +38,7 @@ timing_enter afterboot
|
||||
./scripts/configure_hugepages.sh 1024
|
||||
timing_exit afterboot
|
||||
|
||||
./scripts/unbind.sh
|
||||
./scripts/setup.sh
|
||||
|
||||
#####################
|
||||
# Unit Tests
|
||||
@ -52,7 +52,7 @@ time test/lib/ioat/ioat.sh
|
||||
|
||||
timing_exit lib
|
||||
|
||||
./scripts/cleanup.sh
|
||||
./scripts/setup.sh reset
|
||||
./scripts/build_kmod.sh clean
|
||||
|
||||
timing_exit autotest
|
||||
|
||||
@ -448,7 +448,8 @@ spdk_pci_device_has_non_uio_driver(struct spdk_pci_device *dev)
|
||||
driver_begin = driver;
|
||||
}
|
||||
|
||||
return strcmp(driver_begin, "uio_pci_generic") != 0;
|
||||
return (strcmp(driver_begin, "uio_pci_generic") != 0 &&
|
||||
strcmp(driver_begin, "vfio-pci") != 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
function cleanup_linux() {
|
||||
# detach pci devices from uio driver
|
||||
grep -q "^uio_pci_generic" /proc/modules && rmmod uio_pci_generic
|
||||
|
||||
# bind NVMe devices to NVMe driver if no kernel device
|
||||
if [ -d "/sys/bus/pci/drivers/nvme" ]; then
|
||||
device=`find /sys/bus/pci/drivers/nvme -name "0000*" -print`
|
||||
if [ -z "$device" ]; then
|
||||
rmmod nvme
|
||||
modprobe nvme
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function cleanup_freebsd {
|
||||
kldunload contigmem.ko || true
|
||||
kldunload nic_uio.ko || true
|
||||
}
|
||||
|
||||
if [ `uname` = Linux ]; then
|
||||
cleanup_linux
|
||||
else
|
||||
cleanup_freebsd
|
||||
fi
|
||||
138
scripts/setup.sh
Executable file
138
scripts/setup.sh
Executable file
@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
rootdir=$(readlink -f $(dirname $0))/..
|
||||
|
||||
function linux_iter_pci {
|
||||
# Argument is the class code
|
||||
# TODO: More specifically match against only class codes in the grep
|
||||
# step.
|
||||
lspci -mm -n | grep $1 | tr -d '"' | awk -F " " '{print "0000:"$1}'
|
||||
}
|
||||
|
||||
function configure_linux {
|
||||
driver_name=vfio-pci
|
||||
if [ -z "$(ls /sys/kernel/iommu_groups)" ]; then
|
||||
# No IOMMU. Use uio.
|
||||
driver_name=uio_pci_generic
|
||||
fi
|
||||
|
||||
# NVMe
|
||||
modprobe $driver_name || true
|
||||
for bdf in $(linux_iter_pci 0108); do
|
||||
if [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
|
||||
# Unbind the device from whatever driver it is currently bound to
|
||||
echo $bdf > "/sys/bus/pci/devices/$bdf/driver/unbind"
|
||||
fi
|
||||
# Bind this device to the new driver
|
||||
ven_dev_id=$(lspci -n -s $bdf | cut -d' ' -f3 | sed 's/:/ /')
|
||||
echo "Binding $bdf ($ven_dev_id) to $driver_name"
|
||||
echo $ven_dev_id > "/sys/bus/pci/drivers/$driver_name/new_id"
|
||||
done
|
||||
|
||||
|
||||
# IOAT
|
||||
TMP=`mktemp`
|
||||
#collect all the device_id info of ioat devices.
|
||||
grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/lib/ioat/ioat_pci.h \
|
||||
| awk -F"x" '{print $2}' > $TMP
|
||||
|
||||
for dev_id in `cat $TMP`; do
|
||||
# Abuse linux_iter_pci by giving it a device ID instead of a class code
|
||||
for bdf in $(linux_iter_pci $dev_id); do
|
||||
if [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
|
||||
# Unbind the device from whatever driver it is currently bound to
|
||||
echo $bdf > "/sys/bus/pci/devices/$bdf/driver/unbind"
|
||||
fi
|
||||
# Bind this device to the new driver
|
||||
ven_dev_id=$(lspci -n -s $bdf | cut -d' ' -f3 | sed 's/:/ /')
|
||||
echo "Binding $bdf ($ven_dev_id) to $driver_name"
|
||||
echo $ven_dev_id > "/sys/bus/pci/drivers/$driver_name/new_id"
|
||||
done
|
||||
done
|
||||
rm $TMP
|
||||
|
||||
echo "1" > "/sys/bus/pci/rescan"
|
||||
}
|
||||
|
||||
function reset_linux {
|
||||
# NVMe
|
||||
modprobe nvme || true
|
||||
for bdf in $(linux_iter_pci 0108); do
|
||||
ven_dev_id=$(lspci -n -s $bdf | cut -d' ' -f3 | sed 's/:/ /')
|
||||
if [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
|
||||
# Unregister this device from the driver it was bound to.
|
||||
echo $ven_dev_id > "/sys/bus/pci/devices/$bdf/driver/remove_id"
|
||||
# Unbind the device from whatever driver it is currently bound to
|
||||
echo $bdf > "/sys/bus/pci/devices/$bdf/driver/unbind"
|
||||
fi
|
||||
# Bind this device back to the nvme driver
|
||||
echo "Binding $bdf ($ven_dev_id) to nvme"
|
||||
echo $bdf > "/sys/bus/pci/drivers/nvme/bind"
|
||||
done
|
||||
|
||||
|
||||
# IOAT
|
||||
TMP=`mktemp`
|
||||
#collect all the device_id info of ioat devices.
|
||||
grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/lib/ioat/ioat_pci.h \
|
||||
| awk -F"x" '{print $2}' > $TMP
|
||||
|
||||
modprobe ioatdma || true
|
||||
for dev_id in `cat $TMP`; do
|
||||
# Abuse linux_iter_pci by giving it a device ID instead of a class code
|
||||
for bdf in $(linux_iter_pci $dev_id); do
|
||||
ven_dev_id=$(lspci -n -s $bdf | cut -d' ' -f3 | sed 's/:/ /')
|
||||
if [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
|
||||
# Unregister this device from the driver it was bound to.
|
||||
echo $ven_dev_id > "/sys/bus/pci/devices/$bdf/driver/remove_id"
|
||||
# Unbind the device from whatever driver it is currently bound to
|
||||
echo $bdf > "/sys/bus/pci/devices/$bdf/driver/unbind"
|
||||
fi
|
||||
# Bind this device back to the nvme driver
|
||||
echo "Binding $bdf ($ven_dev_id) to ioatdma"
|
||||
echo $bdf > "/sys/bus/pci/drivers/ioatdma/bind"
|
||||
done
|
||||
done
|
||||
rm $TMP
|
||||
|
||||
echo "1" > "/sys/bus/pci/rescan"
|
||||
}
|
||||
|
||||
function configure_freebsd {
|
||||
TMP=`mktemp`
|
||||
AWK_PROG="{if (count > 0) printf \",\"; printf \"%s:%s:%s\",\$2,\$3,\$4; count++}"
|
||||
echo $AWK_PROG > $TMP
|
||||
PCICONF=`pciconf -l | grep 'class=0x010802\|^ioat'`
|
||||
BDFS=`echo $PCICONF | awk -F: -f $TMP`
|
||||
kldunload nic_uio.ko || true
|
||||
kenv hw.nic_uio.bdfs=$BDFS
|
||||
kldload nic_uio.ko
|
||||
rm $TMP
|
||||
}
|
||||
|
||||
function reset_freebsd {
|
||||
kldunload contigmem.ko || true
|
||||
kldunload nic_uio.ko || true
|
||||
}
|
||||
|
||||
mode=$1
|
||||
if [ "$mode" == "" ]; then
|
||||
mode="config"
|
||||
fi
|
||||
|
||||
if [ `uname` = Linux ]; then
|
||||
if [ "$mode" == "config" ]; then
|
||||
configure_linux
|
||||
elif [ "$mode" == "reset" ]; then
|
||||
reset_linux
|
||||
fi
|
||||
else
|
||||
if [ "$mode" == "config" ]; then
|
||||
configure_freebsd
|
||||
elif [ "$mode" == "reset" ]; then
|
||||
reset_freebsd
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
rootdir=$(readlink -f $(dirname $0))/..
|
||||
|
||||
function prep_nvme {
|
||||
TMP=`mktemp`
|
||||
# Get vendor_id:device_id by nvme's class_id and subcalss_id
|
||||
lspci -n | awk -F " " '{if ($2 == "0108:") {print $3}}' \
|
||||
| awk -F ":" '{print $1" "$2}' > $TMP
|
||||
|
||||
cat $TMP | while read device
|
||||
do
|
||||
echo $device > /sys/bus/pci/drivers/uio_pci_generic/new_id
|
||||
done
|
||||
rm $TMP
|
||||
}
|
||||
|
||||
function prep_ioat {
|
||||
TMP=`mktemp`
|
||||
#collect all the device_id info of ioat devices.
|
||||
grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/lib/ioat/ioat_pci.h \
|
||||
| awk -F"x" '{print $2}' > $TMP
|
||||
vendor=8086
|
||||
|
||||
for device in `cat $TMP`
|
||||
do
|
||||
if lspci -n | grep "$vendor:$device"
|
||||
then
|
||||
echo $vendor $device > /sys/bus/pci/drivers/uio_pci_generic/new_id
|
||||
fi
|
||||
done
|
||||
rm $TMP
|
||||
}
|
||||
|
||||
function configure_linux {
|
||||
rmmod nvme || true
|
||||
rmmod uio_pci_generic || true
|
||||
rmmod ioatdma || true
|
||||
modprobe uio_pci_generic || true
|
||||
prep_nvme
|
||||
prep_ioat
|
||||
}
|
||||
|
||||
function configure_freebsd {
|
||||
TMP=`mktemp`
|
||||
AWK_PROG="{if (count > 0) printf \",\"; printf \"%s:%s:%s\",\$2,\$3,\$4; count++}"
|
||||
echo $AWK_PROG > $TMP
|
||||
PCICONF=`pciconf -l | grep 'class=0x010802\|^ioat'`
|
||||
BDFS=`echo $PCICONF | awk -F: -f $TMP`
|
||||
kldunload nic_uio.ko || true
|
||||
kenv hw.nic_uio.bdfs=$BDFS
|
||||
kldload nic_uio.ko
|
||||
rm $TMP
|
||||
}
|
||||
|
||||
if [ `uname` = Linux ]; then
|
||||
configure_linux
|
||||
else
|
||||
configure_freebsd
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user