From a6edaa96005555e472c035b51e6e4ba8909fad1a Mon Sep 17 00:00:00 2001 From: Wojciech Malikowski Date: Fri, 24 May 2019 05:04:51 -0400 Subject: [PATCH] scripts/setup: Added binding VMD devices To bind VMD device it need to be specified in PCI_WHITELIST. Change-Id: If19c47941278e4df9c6157024bba8b76c90695c6 Signed-off-by: Wojciech Malikowski Signed-off-by: Orden Smith Signed-off-by: Darek Stojaczyk Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/455635 Reviewed-by: Ben Walker Reviewed-by: Jim Harris Tested-by: SPDK CI Jenkins --- scripts/setup.sh | 73 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/scripts/setup.sh b/scripts/setup.sh index 337a12edc..b7fcadadf 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -14,9 +14,9 @@ function usage() fi [[ ! -z $2 ]] && ( echo "$2"; echo ""; ) - echo "Helper script for allocating hugepages and binding NVMe, I/OAT and Virtio devices to" - echo "a generic VFIO kernel driver. If VFIO is not available on the system, this script will" - echo "fall back to UIO. NVMe and Virtio devices with active mountpoints will be ignored." + echo "Helper script for allocating hugepages and binding NVMe, I/OAT, VMD and Virtio devices" + echo "to a generic VFIO kernel driver. If VFIO is not available on the system, this script" + echo "will fall back to UIO. NVMe and Virtio devices with active mountpoints will be ignored." echo "All hugepage operations use default hugepage size on the system (hugepagesz)." echo "Usage: $(basename $1) $options" echo @@ -42,7 +42,7 @@ function usage() echo " hugepages on multiple nodes run this script multiple times -" echo " once for each node." echo "PCI_WHITELIST" - echo "PCI_BLACKLIST Whitespace separated list of PCI devices (NVMe, I/OAT, Virtio)." + echo "PCI_BLACKLIST Whitespace separated list of PCI devices (NVMe, I/OAT, VMD, Virtio)." echo " Each device must be specified as a full PCI address." echo " E.g. PCI_WHITELIST=\"0000:01:00.0 0000:02:00.0\"" echo " To blacklist all PCI devices use a non-valid address." @@ -262,6 +262,25 @@ function configure_linux_pci { done rm $TMP + # VMD + TMP=`mktemp` + #collect all the device_id info of vmd devices. + grep "PCI_DEVICE_ID_INTEL_VMD" $rootdir/include/spdk/pci_ids.h \ + | awk -F"x" '{print $2}' > $TMP + + for dev_id in `cat $TMP`; do + for bdf in $(iter_pci_dev_id 8086 $dev_id); do + if [[ -z "$PCI_WHITELIST" ]] || ! pci_can_use $bdf; then + echo "Skipping un-whitelisted VMD device at $bdf" + continue + fi + + linux_bind_driver "$bdf" "$driver_name" + echo " VMD generic kdrv: " "$bdf" "$driver_name" + done + done + rm $TMP + echo "1" > "/sys/bus/pci/rescan" } @@ -440,6 +459,31 @@ function reset_linux_pci { done rm $TMP + # VMD + TMP=`mktemp` + #collect all the device_id info of vmd devices. + grep "PCI_DEVICE_ID_INTEL_VMD" $rootdir/include/spdk/pci_ids.h \ + | awk -F"x" '{print $2}' > $TMP + + set +e + check_for_driver vmd + driver_loaded=$? + set -e + for dev_id in `cat $TMP`; do + for bdf in $(iter_pci_dev_id 8086 $dev_id); do + if ! pci_can_use $bdf; then + echo "Skipping un-whitelisted VMD device at $bdf" + continue + fi + if [ $driver_loaded -ne 0 ]; then + linux_bind_driver "$bdf" vmd + else + linux_unbind_driver "$bdf" + fi + done + done + rm $TMP + echo "1" > "/sys/bus/pci/rescan" } @@ -545,6 +589,20 @@ function status_linux { echo -e "$bdf\t${vendor#0x}\t${device#0x}\t$node\t\t${driver:--}\t\t$blknames" done done + + echo "VMD" + + #collect all the device_id info of vmd devices. + TMP=`grep "PCI_DEVICE_ID_INTEL_VMD" $rootdir/include/spdk/pci_ids.h \ + | awk -F"x" '{print $2}'` + echo -e "BDF\t\tNuma Node\tDriver Name" + for dev_id in $TMP; do + for bdf in $(iter_pci_dev_id 8086 $dev_id); do + driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}'` + node=`cat /sys/bus/pci/devices/$bdf/numa_node`; + echo -e "$bdf\t$node\t\t$driver" + done + done } function configure_freebsd_pci { @@ -560,6 +618,13 @@ function configure_freebsd_pci { GREP_STR="${GREP_STR}\|chip=0x${dev_id}8086" done + # VMD + grep "PCI_DEVICE_ID_INTEL_VMD" $rootdir/include/spdk/pci_ids.h \ + | awk -F"x" '{print $2}' > $TMP + for dev_id in `cat $TMP`; do + GREP_STR="${GREP_STR}\|chip=0x${dev_id}8086" + done + AWK_PROG="{if (count > 0) printf \",\"; printf \"%s:%s:%s\",\$2,\$3,\$4; count++}" echo $AWK_PROG > $TMP