function install_spdk() { mkdir -p "$GIT_REPOS/spdk_repo/output" || echo "Can not create spdk_repo/output directory." if [[ -d $GIT_REPOS/spdk_repo/spdk ]]; then echo "spdk source already present, not cloning" else git -C "$GIT_REPOS/spdk_repo" clone "${GIT_REPO_SPDK}" fi git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.dpdk.url "${GIT_REPO_DPDK}" git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}" git -C "$GIT_REPOS/spdk_repo/spdk" submodule update --init --recursive } function install_refspdk() { local release local output_dir local config_params local rootdir local version version=$1 # Create a reference SPDK build for ABI tests git -C "$GIT_REPOS/spdk_repo/spdk" fetch --tags --force if [[ "$version" == "latest" ]]; then release=${REFSPDK_TAG:-$(git -C "$GIT_REPOS/spdk_repo/spdk" tag | sort --version-sort | grep -v rc | tail -n 1)} output_dir="$GIT_REPOS/spdk_abi_latest" elif [[ "$version" == "LTS" ]]; then release=$(git -C "$GIT_REPOS/spdk_repo/spdk" describe --tags --exclude=LTS LTS) output_dir="$GIT_REPOS/spdk_abi_lts" fi rm -rf "$output_dir" if [[ ! -d $output_dir ]]; then cp -R "$GIT_REPOS/spdk_repo/spdk" "$output_dir" fi git -C "$output_dir" checkout "$release" git -C "$output_dir" submodule update --init if ((gcc_version >= 11)) && le "${release#v}" 21.04; then git -C "$output_dir" config --global user.name "spdk" git -C "$output_dir" config --global user.email "hotpatch@spdk.io" git -C "$output_dir" cherry-pick 36b5a69bb0699694b53a2f08a13cc0de620450a9 git -C "$output_dir" cherry-pick 2ac152158116a17b863270a4731977d9ddedf50d fi cat > $HOME/autorun-spdk.conf <<- EOF SPDK_BUILD_SHARED_OBJECT=1 SPDK_TEST_AUTOBUILD=1 SPDK_TEST_UNITTEST=1 SPDK_TEST_BLOCKDEV=1 SPDK_TEST_PMDK=1 SPDK_TEST_ISAL=1 SPDK_TEST_REDUCE=1 SPDK_TEST_CRYPTO=1 SPDK_TEST_FTL=1 SPDK_TEST_OCF=1 SPDK_TEST_RAID5=1 SPDK_TEST_RBD=1 SPDK_RUN_ASAN=1 SPDK_RUN_UBSAN=1 SPDK_TEST_NVME_PMR=1 SPDK_TEST_NVME_SCC=1 SPDK_TEST_NVME_BP=1 SPDK_TEST_NVME_CUSE=1 SPDK_TEST_BLOBFS=1 SPDK_TEST_URING=1 SPDK_TEST_VFIOUSER=1 EOF mkdir -p $HOME/output ( rootdir="$output_dir" source $HOME/autorun-spdk.conf source $output_dir/test/common/autotest_common.sh # Prepare separate, fixed, cmdline for the FreeBSD, Issue #1397. if [[ $OSID == freebsd ]]; then config_params="--enable-debug" config_params+=" --without-isal" config_params+=" --with-idxd --disable-unit-tests" MAKE=gmake else config_params="$(get_config_params)" fi $output_dir/configure $(echo $config_params | sed 's/--enable-coverage//g') --without-fio if [[ $OSID != freebsd ]]; then $MAKE -C $output_dir $MAKEFLAGS include/spdk/config.h CONFIG_OCF_PATH="$output_dir/ocf" $MAKE -C $output_dir/lib/env_ocf $MAKEFLAGS exportlib O=$output_dir/ocf.a $output_dir/configure $config_params --with-ocf=$output_dir/ocf.a --with-shared --without-fio fi $MAKE -C $output_dir $MAKEFLAGS ) } function install_qat() { # Disect the kernel version into maj, min, release and local version local kernel_maj kernel_min kernel_rel kernel_loc local kernel_ver in_syms() { local syms if [[ -e /proc/kallsyms ]]; then syms=/proc/kallsyms elif [[ -e /boot/System.map-$(< /proc/sys/kernel/osrelease) ]]; then syms=/boot/System.map-$(< /proc/sys/kernel/osrelease) else return 0 fi grep -q "$1" "$syms" } IFS=".-" read -r kernel_{maj,min,rel,loc} < /proc/sys/kernel/osrelease kernel_ver=$((kernel_maj << 16 | kernel_min << 8 | kernel_rel)) if [[ -e /sys/module/qat_c62x ]]; then sudo modprobe -r qat_c62x || : fi if [[ -d $GIT_REPOS/QAT ]]; then sudo rm -rf "$GIT_REPOS/QAT" fi mkdir "$GIT_REPOS/QAT" tar -C "$GIT_REPOS/QAT" -xzof - < <(wget -O- "$DRIVER_LOCATION_QAT") # Patch use of hidden types in kernels >= 5.6.3. See .patch for details if ((kernel_ver >= 0x050603)); then # Patch only the driver version that was tested [[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1 fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-timespec.patch" # Patch name of the pci_aer function which was renamed in kernels >= 5.7.1. See .patch for details if ((kernel_ver >= 0x050701)) || ! in_syms pci_cleanup_aer_uncorrect_error_status; then # Patch only the driver version that was tested if [[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]]; then patch --dir="$GIT_REPOS/QAT" -p1 fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-pci_aer.patch" if ((kernel_ver < 0x050701)); then # Older kernel build without pci_cleanup_aer_uncorrect_error_status()? This might be # centos8's build. if [[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.12.0-00011.tar.gz ]]; then patch --dir="$GIT_REPOS/QAT" -p1 fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-pci_aer_centos.patch" fi fi # Patch use of cryptohash.h which was removed in favor of crypto/sha.h in kernels >= 5.8. See .patch for details if ((kernel_ver >= 0x050800)); then # Patch only the driver version that was tested [[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1 fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-cryptohash.patch" # Object files need to be passed via obj-m otherwise they won't be built with kernels >= 5.10 if ((kernel_ver >= 0x050a01)); then patch --dir="$GIT_REPOS/QAT" -p1 fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-set-obj-m.patch" # sha.h was split into two separate files, sha1.h and sha2.h in kernels >= 5.11 if ((kernel_ver >= 0x050b01)); then # qat_algs_ablkcipher.c doesn't exist in older versions of the driver so simply force the patch patch --force --dir="$GIT_REPOS/QAT" -p1 fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-sha.patch" || : # cipher routines were moved from crypto.h to crypto/internal/cipher.h in kernels >= 5.12 if ((kernel_ver >= 0x050c00)); then patch --dir="$GIT_REPOS/QAT" -p1 fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-cipher-ns.patch" (cd "$GIT_REPOS/QAT" && sudo ./configure --enable-icp-sriov=host && sudo make install) if ! sudo service qat_service start; then echo "failed to start the qat service. Something may be wrong with your device or package." fi } function install_rocksdb() { # Rocksdb is installed for use with the blobfs tests. if [ ! -d /usr/src/rocksdb ]; then git clone "${GIT_REPO_ROCKSDB}" "$GIT_REPOS/rocksdb" git -C "$GIT_REPOS/rocksdb" checkout 6.15.fb sudo mv "$GIT_REPOS/rocksdb" /usr/src/ else sudo git -C /usr/src/rocksdb checkout 6.15.fb echo "rocksdb already in /usr/src. Not checking out again" fi } function install_fio() { # This version of fio is installed in /usr/src/fio to enable # building the spdk fio plugin. local fio_version="fio-3.28" if [ ! -d /usr/src/fio ]; then if [ ! -d fio ]; then git clone "${GIT_REPO_FIO}" "$GIT_REPOS/fio" sudo mv "$GIT_REPOS/fio" /usr/src/ else sudo mv "$GIT_REPOS/fio" /usr/src/ fi ( git -C /usr/src/fio checkout master \ && git -C /usr/src/fio pull \ && git -C /usr/src/fio checkout $fio_version \ && if [ $OSID == 'freebsd' ]; then gmake -C /usr/src/fio -j${jobs} \ && sudo gmake -C /usr/src/fio install else make -C /usr/src/fio -j${jobs} \ && sudo make -C /usr/src/fio install fi ) else echo "fio already in /usr/src/fio. Not installing" fi } function install_flamegraph() { # Flamegraph is used when printing out timing graphs for the tests. if [ ! -d /usr/local/FlameGraph ]; then git clone "${GIT_REPO_FLAMEGRAPH}" "$GIT_REPOS/FlameGraph" mkdir -p /usr/local sudo mv "$GIT_REPOS/FlameGraph" /usr/local/FlameGraph else echo "flamegraph already installed. Skipping" fi } function _install_qemu() { local repo=$1 local branch=$2 local prefix=${3:-} mkdir -p "$GIT_REPOS/qemu" local repo_dir=$GIT_REPOS/qemu/$branch if [[ -n $prefix ]]; then repo_dir=$GIT_REPOS/qemu/$prefix-$branch fi if [[ ! -d $repo_dir ]]; then git clone "$repo" -b "$branch" "$repo_dir" else echo "qemu already checked out. Skipping" fi declare -a opt_params=("--prefix=/usr/local/qemu/${repo_dir##*/}") declare -a extra_cflags=() opt_params+=("--disable-docs") if ((gcc_version >= 9)); then # GCC 9 fails to compile Qemu due to some old warnings which were not detected by older versions. extra_cflags+=("-Wno-error=stringop-truncation" "-Wno-error=deprecated-declarations") extra_cflags+=("-Wno-error=incompatible-pointer-types" "-Wno-error=format-truncation") extra_cflags+=("-Wno-error=address-of-packed-member") opt_params+=("--disable-glusterfs") fi if ((gcc_version >= 11)); then extra_cflags+=("-Wno-error=array-bounds") fi # Most tsocks proxies rely on a configuration file in /etc/tsocks.conf. # If using tsocks, please make sure to complete this config before trying to build qemu. if [[ $INSTALL_TSOCKS == true && $NO_TSOCKS != true ]]; then if hash tsocks 2> /dev/null; then opt_params+=("--with-git='tsocks git'") fi fi opt_params+=("--extra-cflags=${extra_cflags[*]}") if [[ $prefix == vanilla ]]; then # Latest qemu seems to take sysconfdir from the prefix and instead of checking /etc # it looks under /usr/local/qemu/vanilla*/bin/../etc which is a bit peculiar. Fix it. opt_params+=("--sysconfdir=/etc/") fi # The qemu configure script places several output files in the CWD. (cd "$repo_dir" && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa) make -C "$repo_dir" -j${jobs} sudo make -C "$repo_dir" install } function install_qemu() { # Four versions of QEMU are used in the tests, three are installed # directly from the source. Each QEMU is dedicated for different # use-cases: # - Packed QEMU: version provided by given distro. Used to boot VMs # from within vhost tests. # - SPDK QEMU: A special fork to test OCSSDs. Most jobs on CI use # this version for booting the VMs. # - vfio-user QEMU: A special fork to test libvfio-user components. # - Vanilla QEMU: Used for the upstream PMR support. _install_qemu $GIT_REPO_QEMU_SPDK $SPDK_QEMU_BRANCH _install_qemu $GIT_REPO_QEMU_VFIO $VFIO_QEMU_BRANCH _install_qemu "$GIT_REPO_QEMU" "$VANILLA_QEMU_BRANCH" vanilla } function install_nvmecli() { if [ ! -d "/usr/local/src/nvme-cli" ]; then # Changes required for SPDK are already merged on top of # nvme-cli, however not released yet. # Support for SPDK should be released in nvme-cli >1.11.1 if [[ ! -d $GIT_REPOS/nvme-cli-cuse ]]; then git clone "https://github.com/linux-nvme/nvme-cli.git" "$GIT_REPOS/nvme-cli-cuse" fi git -C "$GIT_REPOS/nvme-cli-cuse" checkout v1.14 make -C "$GIT_REPOS/nvme-cli-cuse" CPPFLAGS="-Wno-error=maybe-uninitialized" sudo mv "$GIT_REPOS/nvme-cli-cuse" /usr/local/src/nvme-cli fi } function install_libiscsi() { # We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need # to later. Cloning from git is just future proofing the machines. if [[ ! -d $GIT_REPOS/libiscsi ]]; then git clone "${GIT_REPO_LIBISCSI}" "$GIT_REPOS/libiscsi" else echo "libiscsi already checked out. Skipping" fi (cd "$GIT_REPOS/libiscsi" && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi) make -C "$GIT_REPOS/libiscsi" -j${jobs} WARN_CFLAGS= sudo make -C "$GIT_REPOS/libiscsi" install } function install_git() { if type -P git; then if ge "$(git --version | awk '{print $3}')" "$GIT_VERSION"; then return 0 fi fi >/dev/null install zlib-devel curl-devel tar -C "$GIT_REPOS" -xzof <(wget -qO- "$GIT_REPO_GIT") (cd "$GIT_REPOS/git-$GIT_VERSION" \ && make configure \ && ./configure --prefix=/usr/local/git \ && sudo make -j${jobs} install) sudo sh -c "echo 'export PATH=/usr/local/git/bin:$PATH' >> /etc/bashrc" export "PATH=/usr/local/git/bin:$PATH" # Be nice for vagrant-proxyconf setup mkdir -p "/usr/local/git/etc" } function install_extra_pkgs() { if [[ $INSTALL_QAT == true ]]; then install libudev-devel || install libudev-dev || : fi if [[ $INSTALL_QEMU == true ]]; then install qemu-system-x86 qemu-img \ || install qemu-system-x86 qemu-utils \ || install qemu \ || : fi } function install_vagrant() { local vagrant_version="2.2.7" local vagrant_installer="vagrant_${vagrant_version}_x86_64.deb" local vagrant_plugins=(vagrant-libvirt vagrant-sshfs vagrant-cachier vagrant-proxyconf) if [[ $OSID != ubuntu ]]; then echo "Currently, Vagrant installation is supported only on ubuntu" return 0 fi # Install vagrant and it's plugins dependencies # function should be defined in pkgdep/$package_manager file install_vagrant_dependencies # Download and install vagrant if hash vagrant &> /dev/null; then echo "Vagrant is already installed" else wget "https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_installer}" sudo dpkg -i "${vagrant_installer}" fi vagrant --version # Install vagrant plugins local vagrant_plugin_list vagrant_plugin_list=$(vagrant plugin list) local plugin for plugin in "${vagrant_plugins[@]}"; do if grep -Fq "$plugin" <<< "$vagrant_plugin_list"; then echo "$plugin already installed" else vagrant plugin install "$plugin" fi done } function install_igb_uio() { git clone "${GIT_REPO_DPDK_KMODS}" "$GIT_REPOS/dpdk-kmods" (cd "$GIT_REPOS/dpdk-kmods/linux/igb_uio" && make -j ${jobs}) sudo mkdir -p "/lib/modules/$(uname -r)/extra/dpdk" sudo cp "$GIT_REPOS/dpdk-kmods/linux/igb_uio/igb_uio.ko" "/lib/modules/$(uname -r)/extra/dpdk" sudo depmod } function install_irdma() { local RDMA_CORE_VERSION=27.0 local RDMA_CORE=https://github.com/linux-rdma/rdma-core/releases/download/v$RDMA_CORE_VERSION/rdma-core-$RDMA_CORE_VERSION.tar.gz if [[ $ID != fedora ]]; then echo "Installation of the irdma can be attempted only on Fedora" return 0 fi # Install extra dependencies needed by the rdma-core-27.0 install ninja-build pandoc perl-generators valgrind-devel python-docutils libnl3 libnl3-devel rm -rf "$GIT_REPOS/irdma-$IRDMA_VERSION" rm -rf "$GIT_REPOS/rdma-core-$RDMA_CORE_VERSION" curl -L -o- "$IRDMA_DRIVER" | tar -C "$GIT_REPOS" -xzf - [[ -e $GIT_REPOS/irdma-$IRDMA_VERSION/build.sh ]] ( cd "$GIT_REPOS/irdma-$IRDMA_VERSION" "$GIT_REPOS/irdma-$IRDMA_VERSION/build.sh" ) # Fetch and build the rdma-core irdma depends on curl -L -o- "$RDMA_CORE" | tar -C "$GIT_REPOS" -xzf - [[ -e $GIT_REPOS/irdma-$IRDMA_VERSION/libirdma-$RDMA_CORE_VERSION.patch ]] patch --dir="$GIT_REPOS/rdma-core-$RDMA_CORE_VERSION" -p2 \ < "$GIT_REPOS/irdma-$IRDMA_VERSION/libirdma-$RDMA_CORE_VERSION.patch" # Note that paths and the name of the package are hardcoded into .spec, hence they need to stay like this. [[ -e $GIT_REPOS/rdma-core-$RDMA_CORE_VERSION/redhat/rdma-core.spec ]] mkdir -p "$HOME/rpmbuild/"{SOURCES,SPECS} cp "$GIT_REPOS/rdma-core-$RDMA_CORE_VERSION/redhat/rdma-core.spec" "$HOME/rpmbuild/SPECS" # Re-package the source tar -czf "$HOME/rpmbuild/SOURCES/rdma-core-$RDMA_CORE_VERSION.tgz" -C "$GIT_REPOS" "rdma-core-$RDMA_CORE_VERSION" # Build the rpms ( cd "$HOME/rpmbuild/SPECS" # Make sure stock ninja-build is used PATH="/usr/bin:$PATH" rpmbuild -ba rdma-core.spec ) # Now, don't install the packages since this will, most likely, conflict with packages already installed # in the system. Instead, simply inform user what the next step is and note what potential issues it may # have with the installation. shopt -s nullglob local rpms=("$HOME/rpmbuild/RPMS/x86_64/"*.rpm) shopt -u nullglob ((${#rpms[@]} > 0)) cat <<-EOF INFO: rdma-core-$RDMA_CORE_VERSION was successfully built, following packages are available for installation: $(printf ' - %s\n' "${rpms[@]##*/}") Note that installing the above packages may raise conflicts with their potentially newer versions already installed on the system. Dependent packages may be uninstalled during the process as well. Please, run the following command to finish the installation: $package_manager install [--allowerasing] $HOME/rpmbuild/RPMS/x86_64/*.rpm EOF } function install_ice() { rm -rf "$GIT_REPOS/ice-$ICE_VERSION" curl -L -o- "$ICE_DRIVER" | tar -C "$GIT_REPOS" -xzf - ( cd "$GIT_REPOS/ice-$ICE_VERSION/src" sudo make -j"$(nproc)" install ) } function install_lcov() { local lcov_version=v1.15 rm -rf /usr/src/lcov sudo git clone "$GIT_REPO_LCOV" --branch "$lcov_version" /usr/src/lcov (cd /usr/src/lcov; sudo make install) } function install_sources() { if [[ $ID == centos ]] && (( VERSION_ID == 7 )); then # install proper version of the git first install_git fi IFS="," read -ra conf_env <<< "$CONF" for conf in "${conf_env[@]}"; do export "INSTALL_${conf^^}=true" done if [[ $OSID == freebsd ]]; then jobs=$(($(sysctl -n hw.ncpu) * 2)) else jobs=$(($(nproc) * 2)) sources+=( install_irdma install_libiscsi install_nvmecli install_qat install_rocksdb install_flamegraph install_qemu install_igb_uio install_ice install_lcov ) install_extra_pkgs fi sources+=(install_fio) sources+=(install_vagrant) sources+=(install_spdk) sudo mkdir -p /usr/{,local}/src sudo mkdir -p "$GIT_REPOS" for source in "${sources[@]}"; do source_conf=${source^^} if [[ ${!source_conf} == true ]]; then "$source" fi done if [[ $INSTALL_REFSPDK == true ]]; then # Serialize builds as refspdk depends on spdk [[ $INSTALL_SPDK != true ]] && install_spdk install_refspdk latest install_refspdk LTS fi } GIT_VERSION=2.25.1 IRDMA_VERSION=1.5.2 ICE_VERSION=1.6.4 SPDK_QEMU_BRANCH=${SPDK_QEMU_BRANCH:-spdk-5.0.0} VFIO_QEMU_BRANCH=${VFIO_QEMU_BRANCH:-vfio-user-v0.9} VANILLA_QEMU_BRANCH=${VANILLA_QEMU_BRANCH:-v6.0.0} : ${GIT_REPO_SPDK=https://github.com/spdk/spdk.git} export GIT_REPO_SPDK : ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git} export GIT_REPO_DPDK : ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb} export GIT_REPO_ROCKSDB : ${GIT_REPO_FIO=https://github.com/axboe/fio.git} export GIT_REPO_FIO : ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git} export GIT_REPO_FLAMEGRAPH : ${GIT_REPO_QEMU=https://github.com/qemu/qemu} export GIT_REPO_QEMU : ${GIT_REPO_QEMU_SPDK=https://github.com/spdk/qemu} export GIT_REPO_QEMU_SPDK : ${GIT_REPO_QEMU_VFIO=https://github.com/oracle/qemu} export GIT_REPO_QEMU_VFIO : ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi} export GIT_REPO_LIBISCSI : ${GIT_REPO_INTEL_IPSEC_MB=https://github.com/spdk/intel-ipsec-mb.git} export GIT_REPO_INTEL_IPSEC_MB : ${DRIVER_LOCATION_QAT=https://01.org/sites/default/files/downloads//qat1.7.l.4.12.0-00011.tar.gz} export DRIVER_LOCATION_QAT : ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz} export GIT_REPO_GIT : ${GIT_REPO_DPDK_KMODS=http://dpdk.org/git/dpdk-kmods} export GIT_REPO_DPDK_KMODS : ${IRDMA_DRIVER=https://downloadmirror.intel.com/29751/eng/irdma-$IRDMA_VERSION.tgz} export IRDMA_DRIVER : ${ICE_DRIVER="https://sourceforge.net/projects/e1000/files/ice stable/$ICE_VERSION/ice-$ICE_VERSION.tar.gz"} export ICE_DRIVER : ${GIT_REPO_LCOV=https://github.com/linux-test-project/lcov} export GIT_REPO_LCOV GIT_REPOS=${GIT_REPOS:-$HOME} gcc_version=$(gcc -dumpversion) gcc_version=${gcc_version%%.*}