From e8a5e965de496a4efc538d1b69b5460ded1e1081 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Tue, 11 May 2021 14:23:27 +0200 Subject: [PATCH] test/nvmf: Refactor hw detection routines Merge detection functions into a single one where all supported devices are picked up and exposed in the environment. Be verbose about what's been found for given tests. Signed-off-by: Michal Berger Change-Id: I780c5e480c301dbb92092074f927a731ed2680b1 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7849 Tested-by: SPDK CI Jenkins Community-CI: Broadcom CI Reviewed-by: Tomasz Zawadzki Reviewed-by: Jim Harris Reviewed-by: Pawel Piatek --- autotest.sh | 6 - test/common/autotest_common.sh | 2 + test/json_config/json_config.sh | 3 +- test/nvmf/common.sh | 280 ++++++++++++++------------------ test/spdkcli/nvmf.sh | 4 +- 5 files changed, 126 insertions(+), 169 deletions(-) diff --git a/autotest.sh b/autotest.sh index a41ebc1d2..74f2e8ef4 100755 --- a/autotest.sh +++ b/autotest.sh @@ -243,15 +243,9 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then # The NVMe-oF run test cases are split out like this so that the parser that compiles the # list of all tests can properly differentiate them. Please do not merge them into one line. if [ "$SPDK_TEST_NVMF_TRANSPORT" = "rdma" ]; then - timing_enter rdma_setup - rdma_device_init - timing_exit rdma_setup run_test "nvmf_rdma" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT run_test "spdkcli_nvmf_rdma" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT elif [ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" ]; then - timing_enter tcp_setup - tcp_device_init - timing_exit tcp_setup run_test "nvmf_tcp" ./test/nvmf/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT if [[ $SPDK_TEST_URING -eq 0 ]]; then run_test "spdkcli_nvmf_tcp" ./test/spdkcli/nvmf.sh --transport=$SPDK_TEST_NVMF_TRANSPORT diff --git a/test/common/autotest_common.sh b/test/common/autotest_common.sh index 7728444b9..f391699e0 100755 --- a/test/common/autotest_common.sh +++ b/test/common/autotest_common.sh @@ -155,6 +155,8 @@ export SPDK_TEST_USE_IGB_UIO export SPDK_TEST_SCHEDULER : ${SPDK_TEST_SCANBUILD:=0} export SPDK_TEST_SCANBUILD +: ${SPDK_TEST_NVMF_DRIVER:=} +export SPDK_TEST_NVMF_DRIVER # always test with SPDK shared objects. export SPDK_LIB_DIR="$rootdir/build/lib" diff --git a/test/json_config/json_config.sh b/test/json_config/json_config.sh index 965f74382..1c38edb4d 100755 --- a/test/json_config/json_config.sh +++ b/test/json_config/json_config.sh @@ -286,8 +286,7 @@ function create_nvmf_subsystem_config() { NVMF_FIRST_TARGET_IP="127.0.0.1" if [[ $SPDK_TEST_NVMF_TRANSPORT == "rdma" ]]; then - rdma_device_init - NVMF_FIRST_TARGET_IP=$(get_available_rdma_ips | head -n 1) + TEST_TRANSPORT=$SPDK_TEST_NVMF_TRANSPORT nvmftestinit fi if [[ -z $NVMF_FIRST_TARGET_IP ]]; then diff --git a/test/nvmf/common.sh b/test/nvmf/common.sh index d68b88b17..b8174d9d8 100644 --- a/test/nvmf/common.sh +++ b/test/nvmf/common.sh @@ -26,6 +26,8 @@ function build_nvmf_app_args() { fi } +source "$rootdir/scripts/common.sh" + : ${NVMF_APP_SHM_ID="0"} export NVMF_APP_SHM_ID build_nvmf_app_args @@ -57,140 +59,6 @@ function detect_soft_roce_nics() { rxe_cfg start } -# args 1 and 2 represent the grep filters for finding our NICS. -# subsequent args are all drivers that should be loaded if we find these NICs. -# Those drivers should be supplied in the correct order. -function detect_nics_and_probe_drivers() { - NIC_VENDOR="$1" - NIC_CLASS="$2" - - nvmf_nic_bdfs=$(lspci | grep Ethernet | grep "$NIC_VENDOR" | grep "$NIC_CLASS" | awk -F ' ' '{print "0000:"$1}') - - if [ -z "$nvmf_nic_bdfs" ]; then - return 0 - fi - - have_pci_nics=1 - if [ $# -ge 2 ]; then - # shift out the first two positional arguments. - shift 2 - # Iterate through the remaining arguments. - for i; do - if [[ $i == irdma ]]; then - # Our tests don't play well with iWARP protocol. Make sure we use RoCEv2 instead. - if [[ -e /sys/module/irdma/parameters/roce_ena ]]; then - # reload the module to re-init the rdma devices - (($(< /sys/module/irdma/parameters/roce_ena) != 1)) && modprobe -r irdma - fi - modprobe "$i" roce_ena=1 - else - modprobe "$i" - fi - done - fi -} - -function pci_rdma_switch() { - local driver=$1 - - local -a driver_args=() - driver_args+=("Mellanox ConnectX-4 mlx5_core mlx5_ib") - driver_args+=("Mellanox ConnectX-5 mlx5_core mlx5_ib") - driver_args+=("Intel E810 ice irdma") - driver_args+=("Intel X722 i40e i40iw") - driver_args+=("Chelsio \"Unified Wire\" cxgb4 iw_cxgb4") - - case $driver in - mlx5_ib) - detect_nics_and_probe_drivers ${driver_args[0]} - detect_nics_and_probe_drivers ${driver_args[1]} - ;; - irdma) - detect_nics_and_probe_drivers ${driver_args[2]} - ;; - i40iw) - detect_nics_and_probe_drivers ${driver_args[3]} - ;; - iw_cxgb4) - detect_nics_and_probe_drivers ${driver_args[4]} - ;; - *) - for d in "${driver_args[@]}"; do - detect_nics_and_probe_drivers $d - done - ;; - esac -} - -function pci_tcp_switch() { - local driver=$1 - - local -a driver_args=() - driver_args+=("Intel E810 ice") - - case $driver in - ice) - detect_nics_and_probe_drivers ${driver_args[0]} - ;; - *) - for d in "${driver_args[@]}"; do - detect_nics_and_probe_drivers $d - done - ;; - esac -} - -function detect_pci_nics() { - - if ! hash lspci; then - return 0 - fi - - local nic_drivers - local found_drivers - - if [[ -z "$TEST_TRANSPORT" ]]; then - TEST_TRANSPORT=$SPDK_TEST_NVMF_TRANSPORT - fi - - if [[ "$TEST_TRANSPORT" == "rdma" ]]; then - nic_drivers="mlx5_ib|irdma|i40iw|iw_cxgb4" - - # Try to find RDMA drivers which are already loded and try to - # use only it's associated NICs, without probing all drivers. - found_drivers=$(lsmod | grep -Eo $nic_drivers | sort -u) - for d in $found_drivers; do - pci_rdma_switch $d - done - - # In case lsmod reported driver, but lspci does not report - # physical NICs - fall back to old approach any try to - # probe all compatible NICs. - ((have_pci_nics == 0)) && pci_rdma_switch "default" - - elif [[ "$TEST_TRANSPORT" == "tcp" ]]; then - nic_drivers="ice" - found_drivers=$(lsmod | grep -Eo $nic_drivers | sort -u) - for d in $found_drivers; do - pci_tcp_switch $d - done - ((have_pci_nics == 0)) && pci_tcp_switch "default" - fi - - # Use softroce if everything else failed. - ((have_pci_nics == 0)) && return 0 - - # Provide time for drivers to properly load. - sleep 5 -} - -function detect_transport_nics() { - detect_pci_nics - if [ "$have_pci_nics" -eq "0" ]; then - detect_soft_roce_nics - fi -} - function allocate_nic_ips() { ((count = NVMF_IP_LEAST_ADDR)) for nic_name in $(get_rdma_if_list); do @@ -212,17 +80,25 @@ function get_available_rdma_ips() { } function get_rdma_if_list() { - rxe_cfg rxe-net -} + local net_dev rxe_net_dev rxe_net_devs -function get_tcp_if_list_by_driver() { - local driver - driver=${1:-ice} + mapfile -t rxe_net_devs < <(rxe_cfg rxe-net) - shopt -s nullglob - tcp_if_list=(/sys/bus/pci/drivers/$driver/0000*/net/*) - shopt -u nullglob - printf '%s\n' "${tcp_if_list[@]##*/}" + if ((${#net_devs[@]} == 0)); then + # No rdma-capable nics on board, using soft-RoCE + printf '%s\n' "${rxe_net_devs[@]}" + return 0 + fi + + # Pick only these devices which were found during gather_supported_nvmf_pci_devs() run + for net_dev in "${net_devs[@]}"; do + for rxe_net_dev in "${rxe_net_devs[@]}"; do + if [[ $net_dev == "$rxe_net_dev" ]]; then + echo "$net_dev" + continue 2 + fi + done + done } function get_ip_address() { @@ -345,8 +221,8 @@ function nvmf_veth_fini() { function nvmf_tcp_init() { NVMF_INITIATOR_IP=10.0.0.1 NVMF_FIRST_TARGET_IP=10.0.0.2 - TCP_INTERFACE_LIST=($(get_tcp_if_list_by_driver)) - if ((${#TCP_INTERFACE_LIST[@]} == 0)) || [ "$TEST_MODE" == "iso" ]; then + TCP_INTERFACE_LIST=("${net_devs[@]}") + if ((${#TCP_INTERFACE_LIST[@]} == 0)); then nvmf_veth_init return 0 fi @@ -400,6 +276,103 @@ function nvmf_tcp_fini() { ip -4 addr flush $NVMF_INITIATOR_INTERFACE || : } +function gather_supported_nvmf_pci_devs() { + # Go through the entire pci bus and gather all ethernet controllers we support for the nvmf tests. + # Focus on the hardware that's currently being tested by the CI. + xtrace_disable + cache_pci_bus_sysfs + xtrace_restore + + local intel=0x8086 mellanox=0x15b3 pci + + local -a pci_devs=() + local -a pci_net_devs=() + local -A pci_drivers=() + + local -ga net_devs=() + local -ga e810=() + local -ga x722=() + local -ga mlx=() + + # E810-XXV + e810+=(${pci_bus_cache["$intel:0x1592"]}) + e810+=(${pci_bus_cache["$intel:0x159b"]}) + # X722 10G + x722+=(${pci_bus_cache["$intel:0x37d2"]}) + # ConnectX-5 + mlx+=(${pci_bus_cache["$mellanox:0x1017"]}) + # ConnectX-4 + mlx+=(${pci_bus_cache["$mellanox:0x1015"]}) + mlx+=(${pci_bus_cache["$mellanox:0x1013"]}) + + pci_devs+=("${e810[@]}") + if [[ $TEST_TRANSPORT == rdma ]]; then + pci_devs+=("${x722[@]}") + pci_devs+=("${mlx[@]}") + fi + + # Try to respect what CI wants to test and override pci_devs[] + if [[ $SPDK_TEST_NVMF_DRIVER == mlx5_ib ]]; then + pci_devs=("${mlx[@]}") + elif [[ $SPDK_TEST_NVMF_DRIVER == ice ]]; then + pci_devs=("${e810[@]}") + fi + + if ((${#pci_devs[@]} == 0)); then + if [[ $TEST_TRANSPORT == rdma ]]; then + echo "WARNING: No pci devices found for the $TEST_TRANSPORT test, falling back to Soft-RoCE" + detect_soft_roce_nics + fi + # tcp fallbacks to veth setup + return 0 + fi + + # Load proper kernel modules if necessary + for pci in "${pci_devs[@]}"; do + echo "Found $pci (${pci_ids_vendor["$pci"]} - ${pci_ids_device["$pci"]})" + if [[ ${pci_mod_resolved["$pci"]} == unknown ]]; then + echo "Unresolved modalias for $pci (${pci_mod_driver["$pci"]}). Driver not installed|builtin?" + continue + fi + if [[ ${pci_bus_driver["$pci"]} == unbound ]]; then + echo "$pci not bound, needs ${pci_mod_resolved["$pci"]}" + pci_drivers["${pci_mod_resolved["$pci"]}"]=1 + fi + done + + if ((${#pci_drivers[@]} > 0)); then + echo "Loading kernel modules: ${!pci_drivers[*]}" + modprobe -a "${!pci_drivers[@]}" + fi + + # E810 cards also need irdma driver to be around. + if ((${#e810[@]} > 0)) && [[ $TEST_TRANSPORT == rdma ]]; then + if [[ -e /sys/module/irdma/parameters/roce_ena ]]; then + # Our tests don't play well with iWARP protocol. Make sure we use RoCEv2 instead. + (($(< /sys/module/irdma/parameters/roce_ena) != 1)) && modprobe -r irdma + fi + modinfo irdma && modprobe irdma roce_ena=1 + fi > /dev/null + + # All devices detected, kernel modules loaded. Now look under net class to see if there + # are any net devices bound to the controllers. + for pci in "${pci_devs[@]}"; do + if [[ ! -e /sys/bus/pci/devices/$pci/net ]]; then + echo "No net devices associated with $pci" + continue + fi + pci_net_devs=("/sys/bus/pci/devices/$pci/net/"*) + pci_net_devs=("${pci_net_devs[@]##*/}") + echo "Found net devices under $pci: ${pci_net_devs[*]}" + net_devs+=("${pci_net_devs[@]}") + done + + if ((${#net_devs[@]} == 0)); then + echo "ERROR: No net devices were found for: ${pci_devs[*]}. Cannot run the $TEST_TRANSPORT test" + return 1 + fi +} + function nvmftestinit() { if [ -z $TEST_TRANSPORT ]; then echo "transport not specified - use --transport= to specify" @@ -408,18 +381,15 @@ function nvmftestinit() { trap 'process_shm --id $NVMF_APP_SHM_ID || :; nvmftestfini' SIGINT SIGTERM EXIT + gather_supported_nvmf_pci_devs + if [ "$TEST_MODE" == "iso" ]; then $rootdir/scripts/setup.sh - if [[ "$TEST_TRANSPORT" == "rdma" ]]; then - rdma_device_init - fi - if [[ "$TEST_TRANSPORT" == "tcp" ]]; then - tcp_device_init - fi fi NVMF_TRANSPORT_OPTS="-t $TEST_TRANSPORT" if [[ "$TEST_TRANSPORT" == "rdma" ]]; then + rdma_device_init RDMA_IP_LIST=$(get_available_rdma_ips) NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1) NVMF_SECOND_TARGET_IP=$(echo "$RDMA_IP_LIST" | tail -n +2 | head -n 1) @@ -458,9 +428,6 @@ function nvmftestfini() { fi if [ "$TEST_MODE" == "iso" ]; then $rootdir/scripts/setup.sh reset - if [[ "$TEST_TRANSPORT" == "rdma" ]]; then - rdma_device_init - fi fi if [[ "$TEST_TRANSPORT" == "tcp" ]]; then nvmf_tcp_fini @@ -469,14 +436,9 @@ function nvmftestfini() { function rdma_device_init() { load_ib_rdma_modules - detect_transport_nics allocate_nic_ips } -function tcp_device_init() { - detect_transport_nics -} - function revert_soft_roce() { rxe_cfg stop } diff --git a/test/spdkcli/nvmf.sh b/test/spdkcli/nvmf.sh index 4bc7899a4..4b8d3246d 100755 --- a/test/spdkcli/nvmf.sh +++ b/test/spdkcli/nvmf.sh @@ -17,8 +17,8 @@ timing_exit run_nvmf_tgt NVMF_TARGET_IP="127.0.0.1" if [[ $TEST_TRANSPORT == "rdma" ]]; then - rdma_device_init - NVMF_TARGET_IP=$(get_available_rdma_ips | head -n 1) + nvmftestinit + NVMF_TARGET_IP=$NVMF_FIRST_TARGET_IP fi timing_enter spdkcli_create_nvmf_config