From 904ac49f4c7069f0dd55a6e44a970eda56b1d209 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Fri, 21 Aug 2020 13:41:40 +0200 Subject: [PATCH] scripts/setup: Configure binding of the controllers in parallel There are some devices for which nvme driver takes a long time to finalize the unbind stage. With that in mind, each device would add up a significant amount of time needed for setup.sh to complete. To mitigate such a scenario, make sure the controllers are unbound in a parallel fashion. Examples taken from the system with 19 nvmes on board: [root@supermicro4 spdk]# time ./scripts/setup.sh &>/dev/null real 0m36.250s user 0m1.024s sys 0m1.990s [root@supermicro4 spdk]# time ./scripts/setup.sh &>/dev/null real 0m4.848s user 0m0.867s sys 0m17.605s Also, take note that this is currently done only for the nvme devices since other, i.e., ioatdma, seem to trigger a BUG in the kernel when unbound in parallel. Some details here: https://bugzilla.kernel.org/show_bug.cgi?id=209041 Change-Id: Icaeb2b2ecb306f149587bc5da73743b1519bc5d6 Signed-off-by: Michal Berger Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3893 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Tomasz Zawadzki --- scripts/setup.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/setup.sh b/scripts/setup.sh index 14d048def..0733d6498 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -274,9 +274,19 @@ function configure_linux_pci() { for bdf in "${!all_devices_d[@]}"; do if ((all_devices_d["$bdf"] == 0)); then - linux_bind_driver "$bdf" "$driver_name" + if [[ -n ${nvme_d["$bdf"]} ]]; then + # Some nvme controllers may take significant amount of time while being + # unbound from the driver. Put that task into background to speed up the + # whole process. Currently this is done only for the devices bound to the + # nvme driver as other, i.e., ioatdma's, trigger a kernel BUG when being + # unbound in parallel. See https://bugzilla.kernel.org/show_bug.cgi?id=209041. + linux_bind_driver "$bdf" "$driver_name" & + else + linux_bind_driver "$bdf" "$driver_name" + fi fi done + wait echo "1" > "/sys/bus/pci/rescan" }