Spdk/test/common/config/pkgdep/git
Karol Latecki 00afb74669 test/vm_setup: fix LTS release checkout
We don't typically "describe" our releases and it seems
that last LTS (20.01.2) was described by accident,
so this script worked. Fix it for 21.01 LTS.

Signed-off-by: Karol Latecki <karol.latecki@intel.com>
Change-Id: I2e64380299cd1a4aeaf21f324890d9257efdaea3
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6244
Reviewed-by: Michal Berger <michalx.berger@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
2021-02-05 13:47:12 +00:00

464 lines
15 KiB
Plaintext

function install_spdk() {
mkdir -p "$GIT_REPOS/spdk_repo/output" || echo "Can not create spdk_repo/output directory."
if [[ -d $GIT_REPOS/spdk_repo/spdk ]]; then
echo "spdk source already present, not cloning"
else
git -C "$GIT_REPOS/spdk_repo" clone "${GIT_REPO_SPDK}"
fi
git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.dpdk.url "${GIT_REPO_DPDK}"
git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}"
git -C "$GIT_REPOS/spdk_repo/spdk" submodule update --init --recursive
}
function install_refspdk() {
local release
local output_dir
local config_params
local rootdir
local version
version=$1
# Create a reference SPDK build for ABI tests
git -C "$GIT_REPOS/spdk_repo/spdk" fetch --tags --force
if [[ "$version" == "latest" ]]; then
release=$(git -C "$GIT_REPOS/spdk_repo/spdk" tag | sort --version-sort | grep -v rc | tail -n 1)
output_dir="$GIT_REPOS/spdk_abi_latest"
elif [[ "$version" == "LTS" ]]; then
release=$(git -C "$GIT_REPOS/spdk_repo/spdk" describe --tags --exclude=LTS LTS)
output_dir="$GIT_REPOS/spdk_abi_lts"
fi
rm -rf "$output_dir"
if [[ ! -d $output_dir ]]; then
cp -R "$GIT_REPOS/spdk_repo/spdk" "$output_dir"
fi
lts_2001_fallback=false
if [[ "$release" == v20.01* ]]; then
# We switch to a .x branch in order to slurp all the backports which fix spdk_abi_lts
# build for this particular release. Fetch the branch explicitly as in case of our CI,
# jenkins will set remote.*.fetch to a particular ref the build run against, so plain
# git fetch won't include any branches from the repo. FIXME: This could be removed
# after new LTS is in place.
release=${release%%-*}.x
git -C "$output_dir" fetch origin "+refs/heads/$release:refs/heads/$release"
lts_2001_fallback=true
fi
git -C "$output_dir" checkout "$release"
git -C "$output_dir" submodule update --init
if [[ "$release" == v20.01* ]]; then
makefiles=(
"$output_dir/dpdk/drivers/crypto/aesni_gcm/Makefile"
"$output_dir/dpdk/drivers/crypto/aesni_mb/Makefile"
"$output_dir/dpdk/drivers/crypto/kasumi/Makefile"
"$output_dir/dpdk/drivers/crypto/snow3g/Makefile"
)
# Attempt to replicate dpdk's 2a860943b8 commit which fixes builds under make 4.3
# FIXME: Remove this when LTS changes!
for makefile in "${makefiles[@]}"; do
# This sed call is meant to be compatible with its FreeBSD implementation
echo "$(sed -e 's/\\#/\$H/g' -e '/IMB_HDR =/i\
H := \\#' "$makefile")" > "$makefile"
done
fi
cat > $HOME/autorun-spdk.conf <<- EOF
SPDK_BUILD_SHARED_OBJECT=1
SPDK_TEST_AUTOBUILD=1
SPDK_TEST_UNITTEST=1
SPDK_TEST_BLOCKDEV=1
SPDK_TEST_PMDK=1
SPDK_TEST_ISAL=1
SPDK_TEST_REDUCE=1
SPDK_TEST_CRYPTO=1
SPDK_TEST_FTL=1
SPDK_TEST_OCF=1
SPDK_TEST_RAID5=1
SPDK_TEST_RBD=1
SPDK_RUN_ASAN=1
SPDK_RUN_UBSAN=1
SPDK_TEST_NVME_CUSE=1
SPDK_TEST_BLOBFS=1
SPDK_TEST_URING=1
EOF
mkdir -p $HOME/output
(
rootdir="$output_dir"
source $HOME/autorun-spdk.conf
source $output_dir/test/common/autotest_common.sh
# Prepare separate, fixed, cmdline for the FreeBSD, Issue #1397.
if [[ $OSID == freebsd ]]; then
config_params="--enable-debug --enable-werror"
config_params+=" --without-isal --with-fio=/usr/src/fio"
# TODO: Remove this if-block after 21.01 LTS is released and 20.01 LTS is deprecated.
if ! "$lts_2001_fallback"; then
config_params+=" --with-idxd --disable-unit-tests"
fi
MAKE=gmake
else
# TODO: "get_config_params" was not available in 20.01 LTS release.
# Remove this if-block after 21.01 release.
if "$lts_2001_fallback"; then
config_params="--enable-debug --enable-werror --with-rdma"
config_params+=" --with-fio=/usr/src/fio --with-iscsi-initiator"
config_params+=" --with-nvme-cuse --with-pmdk --with-reduce"
config_params+=" --with-rbd --with-crypto --with-ocf --enable-ubsan"
config_params+=" --enable-asan --with-fuse --with-uring"
else
config_params="$(get_config_params)"
fi
fi
$output_dir/configure $(echo $config_params | sed 's/--enable-coverage//g')
if [[ $OSID != freebsd ]]; then
$MAKE -C $output_dir $MAKEFLAGS include/spdk/config.h
CONFIG_OCF_PATH="$output_dir/ocf" $MAKE -C $output_dir/lib/env_ocf $MAKEFLAGS exportlib O=$output_dir/build/ocf.a
$output_dir/configure $config_params --with-ocf=$output_dir/build/ocf.a --with-shared
fi
$MAKE -C $output_dir $MAKEFLAGS
)
}
function install_qat() {
# Disect the kernel version into maj, min, release and local version
local kernel_maj kernel_min kernel_rel kernel_loc
local kernel_ver
in_syms() {
local syms
if [[ -e /proc/kallsyms ]]; then
syms=/proc/kallsyms
elif [[ -e /boot/System.map-$(< /proc/sys/kernel/osrelease) ]]; then
syms=/boot/System.map-$(< /proc/sys/kernel/osrelease)
else
return 0
fi
grep -q "$1" "$syms"
}
IFS=".-" read -r kernel_{maj,min,rel,loc} < /proc/sys/kernel/osrelease
kernel_ver=$((kernel_maj << 16 | kernel_min << 8 | kernel_rel))
if [[ -e /sys/module/qat_c62x ]]; then
sudo modprobe -r qat_c62x || :
fi
if [[ -d $GIT_REPOS/QAT ]]; then
sudo rm -rf "$GIT_REPOS/QAT"
fi
mkdir "$GIT_REPOS/QAT"
tar -C "$GIT_REPOS/QAT" -xzof - < <(wget -O- "$DRIVER_LOCATION_QAT")
# Patch use of hidden types in kernels >= 5.6.3. See .patch for details
if ((kernel_ver >= 0x050603)); then
# Patch only the driver version that was tested
[[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1
fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-timespec.patch"
# Patch name of the pci_aer function which was renamed in kernels >= 5.7.1. See .patch for details
if ((kernel_ver >= 0x050701)) || ! in_syms pci_cleanup_aer_uncorrect_error_status; then
# Patch only the driver version that was tested
[[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1
fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-pci_aer.patch"
# Patch use of cryptohash.h which was removed in favor of crypto/sha.h in kernels >= 5.8. See .patch for details
if ((kernel_ver >= 0x050800)); then
# Patch only the driver version that was tested
[[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1
fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-cryptohash.patch"
# Object files need to be passed via obj-m otherwise they won't be built with kernels >= 5.10
if ((kernel_ver >= 0x050a01)); then
patch --dir="$GIT_REPOS/QAT" -p1
fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-set-obj-m.patch"
(cd "$GIT_REPOS/QAT" && sudo ./configure --enable-icp-sriov=host && sudo make install)
if ! sudo service qat_service start; then
echo "failed to start the qat service. Something may be wrong with your device or package."
fi
}
function install_rocksdb() {
# Rocksdb is installed for use with the blobfs tests.
if [ ! -d /usr/src/rocksdb ]; then
git clone "${GIT_REPO_ROCKSDB}" "$GIT_REPOS/rocksdb"
git -C "$GIT_REPOS/rocksdb" checkout spdk-v5.6.1
sudo mv "$GIT_REPOS/rocksdb" /usr/src/
else
sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1
echo "rocksdb already in /usr/src. Not checking out again"
fi
}
function install_fio() {
# This version of fio is installed in /usr/src/fio to enable
# building the spdk fio plugin.
local fio_version="fio-3.19"
if [ ! -d /usr/src/fio ]; then
if [ ! -d fio ]; then
git clone "${GIT_REPO_FIO}" "$GIT_REPOS/fio"
sudo mv "$GIT_REPOS/fio" /usr/src/
else
sudo mv "$GIT_REPOS/fio" /usr/src/
fi
(
git -C /usr/src/fio checkout master \
&& git -C /usr/src/fio pull \
&& git -C /usr/src/fio checkout $fio_version \
&& if [ $OSID == 'freebsd' ]; then
gmake -C /usr/src/fio -j${jobs} \
&& sudo gmake -C /usr/src/fio install
else
make -C /usr/src/fio -j${jobs} \
&& sudo make -C /usr/src/fio install
fi
)
else
echo "fio already in /usr/src/fio. Not installing"
fi
}
function install_flamegraph() {
# Flamegraph is used when printing out timing graphs for the tests.
if [ ! -d /usr/local/FlameGraph ]; then
git clone "${GIT_REPO_FLAMEGRAPH}" "$GIT_REPOS/FlameGraph"
mkdir -p /usr/local
sudo mv "$GIT_REPOS/FlameGraph" /usr/local/FlameGraph
else
echo "flamegraph already installed. Skipping"
fi
}
function install_qemu() {
# Two versions of QEMU are used in the tests.
# Stock QEMU is used for vhost. A special fork
# is used to test OCSSDs. Install both.
# Forked QEMU
SPDK_QEMU_BRANCH=spdk-5.0.0
mkdir -p "$GIT_REPOS/qemu"
if [[ ! -d $GIT_REPOS/qemu/$SPDK_QEMU_BRANCH ]]; then
git clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH"
else
echo "qemu already checked out. Skipping"
fi
declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH")
declare -a extra_cflags=()
opt_params+=("--disable-docs")
if ((gcc_version >= 9)); then
# GCC 9 fails to compile Qemu due to some old warnings which were not detected by older versions.
extra_cflags+=("-Wno-error=stringop-truncation" "-Wno-error=deprecated-declarations")
extra_cflags+=("-Wno-error=incompatible-pointer-types" "-Wno-error=format-truncation")
extra_cflags+=("-Wno-error=address-of-packed-member")
opt_params+=("--disable-glusterfs")
fi
# Most tsocks proxies rely on a configuration file in /etc/tsocks.conf.
# If using tsocks, please make sure to complete this config before trying to build qemu.
if [[ $INSTALL_TSOCKS == true && $NO_TSOCKS != true ]]; then
if hash tsocks 2> /dev/null; then
opt_params+=("--with-git='tsocks git'")
fi
fi
opt_params+=("--extra-cflags=${extra_cflags[*]}")
sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules"
sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config"
sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules"
sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config"
# The qemu configure script places several output files in the CWD.
(cd "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa)
make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" -j${jobs}
sudo make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" install
}
function install_nvmecli() {
if [ ! -d "/usr/local/src/nvme-cli" ]; then
# Changes required for SPDK are already merged on top of
# nvme-cli, however not released yet.
# Support for SPDK should be released in nvme-cli >1.11.1
if [[ ! -d $GIT_REPOS/nvme-cli-cuse ]]; then
git clone "https://github.com/linux-nvme/nvme-cli.git" "$GIT_REPOS/nvme-cli-cuse"
fi
git -C "$GIT_REPOS/nvme-cli-cuse" checkout "e770466615096a6d41f038a28819b00bc3078e1d"
make -C "$GIT_REPOS/nvme-cli-cuse" CPPFLAGS="-Wno-error=maybe-uninitialized"
sudo mv "$GIT_REPOS/nvme-cli-cuse" /usr/local/src/nvme-cli
fi
}
function install_libiscsi() {
# We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need
# to later. Cloning from git is just future proofing the machines.
if [[ ! -d $GIT_REPOS/libiscsi ]]; then
git clone "${GIT_REPO_LIBISCSI}" "$GIT_REPOS/libiscsi"
else
echo "libiscsi already checked out. Skipping"
fi
(cd "$GIT_REPOS/libiscsi" && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi)
make -C "$GIT_REPOS/libiscsi" -j${jobs} WARN_CFLAGS=
sudo make -C "$GIT_REPOS/libiscsi" install
}
function install_git() {
if type -P git; then
if ge "$(git --version | awk '{print $3}')" "$GIT_VERSION"; then
return 0
fi
fi >/dev/null
install zlib-devel curl-devel
tar -C "$GIT_REPOS" -xzof <(wget -qO- "$GIT_REPO_GIT")
(cd "$GIT_REPOS/git-$GIT_VERSION" \
&& make configure \
&& ./configure --prefix=/usr/local/git \
&& sudo make -j${jobs} install)
sudo sh -c "echo 'export PATH=/usr/local/git/bin:$PATH' >> /etc/bashrc"
export "PATH=/usr/local/git/bin:$PATH"
# Be nice for vagrant-proxyconf setup
mkdir -p "/usr/local/git/etc"
}
function install_extra_pkgs() {
if [[ $INSTALL_QAT == true ]]; then
install libudev-devel || install libudev-dev || :
fi
if [[ $INSTALL_QEMU == true ]]; then
install qemu-system-x86 qemu-img \
|| install qemu-system-x86 qemu-utils \
|| install qemu \
|| :
fi
}
function install_vagrant() {
local vagrant_version="2.2.7"
local vagrant_installer="vagrant_${vagrant_version}_x86_64.deb"
local vagrant_plugins=(vagrant-libvirt vagrant-sshfs vagrant-cachier vagrant-proxyconf)
if [[ $OSID != ubuntu ]]; then
echo "Currently, Vagrant installation is supported only on ubuntu"
return 0
fi
# Install vagrant and it's plugins dependencies
# function should be defined in pkgdep/$package_manager file
install_vagrant_dependencies
# Download and install vagrant
if hash vagrant &> /dev/null; then
echo "Vagrant is already installed"
else
wget "https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_installer}"
sudo dpkg -i "${vagrant_installer}"
fi
vagrant --version
# Install vagrant plugins
local vagrant_plugin_list
vagrant_plugin_list=$(vagrant plugin list)
local plugin
for plugin in "${vagrant_plugins[@]}"; do
if grep -Fq "$plugin" <<< "$vagrant_plugin_list"; then
echo "$plugin already installed"
else
vagrant plugin install "$plugin"
fi
done
}
function install_igb_uio() {
git clone "${GIT_REPO_DPDK_KMODS}" "$GIT_REPOS/dpdk-kmods"
(cd "$GIT_REPOS/dpdk-kmods/linux/igb_uio" && make -j ${jobs})
sudo mkdir -p "/lib/modules/$(uname -r)/extra/dpdk"
sudo cp "$GIT_REPOS/dpdk-kmods/linux/igb_uio/igb_uio.ko" "/lib/modules/$(uname -r)/extra/dpdk"
sudo depmod
}
GIT_VERSION=2.25.1
: ${GIT_REPO_SPDK=https://github.com/spdk/spdk.git}
export GIT_REPO_SPDK
: ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git}
export GIT_REPO_DPDK
: ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb}
export GIT_REPO_ROCKSDB
: ${GIT_REPO_FIO=https://github.com/axboe/fio.git}
export GIT_REPO_FIO
: ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git}
export GIT_REPO_FLAMEGRAPH
: ${GIT_REPO_QEMU=https://github.com/spdk/qemu}
export GIT_REPO_QEMU
: ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi}
export GIT_REPO_LIBISCSI
: ${GIT_REPO_INTEL_IPSEC_MB=https://github.com/spdk/intel-ipsec-mb.git}
export GIT_REPO_INTEL_IPSEC_MB
: ${DRIVER_LOCATION_QAT=https://01.org/sites/default/files/downloads//qat1.7.l.4.12.0-00011.tar.gz}
export DRIVER_LOCATION_QAT
: ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz}
export GIT_REPO_GIT
: ${GIT_REPO_DPDK_KMODS=http://dpdk.org/git/dpdk-kmods}
export GIT_REPO_DPDK_KMODS
GIT_REPOS=${GIT_REPOS:-$HOME}
gcc_version=$(gcc -dumpversion) gcc_version=${gcc_version%%.*}
if [[ $ID == centos ]] && (( VERSION_ID == 7 )); then
# install proper version of the git first
install_git
fi
IFS="," read -ra conf_env <<< "$CONF"
for conf in "${conf_env[@]}"; do
export "INSTALL_${conf^^}=true"
done
if [[ $OSID == freebsd ]]; then
jobs=$(($(sysctl -n hw.ncpu) * 2))
else
jobs=$(($(nproc) * 2))
sources+=(
install_libiscsi
install_nvmecli
install_qat
install_rocksdb
install_flamegraph
install_qemu
install_igb_uio
)
install_extra_pkgs
fi
sources+=(install_fio)
sources+=(install_vagrant)
sources+=(install_spdk)
sudo mkdir -p /usr/{,local}/src
sudo mkdir -p "$GIT_REPOS"
for source in "${sources[@]}"; do
source_conf=${source^^}
if [[ ${!source_conf} == true ]]; then
"$source"
fi
done
if [[ $INSTALL_REFSPDK == true ]]; then
# Serialize builds as refspdk depends on spdk
[[ $INSTALL_SPDK != true ]] && install_spdk
install_refspdk latest
install_refspdk LTS
fi