vm_setup: Make vm_setup.sh to use specific source for repo installations
Change-Id: I01613720e078a0fa151ed9a4a17e2faf131510bc Signed-off-by: Michal Berger <michalx.berger@intel.com> Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2526 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com> Reviewed-by: Karol Latecki <karol.latecki@intel.com> Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
This commit is contained in:
Michal Berger
Tomasz Zawadzki
parent
585c9298e4
commit
3920b81681
@ -41,300 +41,6 @@ else
|
|||||||
PACKAGEMNG='undefined'
|
PACKAGEMNG='undefined'
|
||||||
fi
|
fi
|
||||||
|
|
||||||
function install_reference_spdk() {
|
|
||||||
local last_release
|
|
||||||
local output_dir
|
|
||||||
local config_params
|
|
||||||
local rootdir
|
|
||||||
|
|
||||||
# Create a reference SPDK build for ABI tests
|
|
||||||
if echo $CONF | grep -q refspdk; then
|
|
||||||
git -C spdk_repo/spdk fetch --tags
|
|
||||||
last_release=$(git -C spdk_repo/spdk tag | sort --version-sort | grep -v rc | tail -n1)
|
|
||||||
git -C spdk_repo/spdk checkout $last_release
|
|
||||||
git -C spdk_repo/spdk submodule update --init
|
|
||||||
output_dir="$HOME/spdk_$(tr . _ < <(tr -d '[:alpha:]' <<< $last_release))"
|
|
||||||
|
|
||||||
cp -r spdk_repo/spdk $output_dir
|
|
||||||
|
|
||||||
cat > $HOME/autorun-spdk.conf << EOF
|
|
||||||
SPDK_BUILD_SHARED_OBJECT=1
|
|
||||||
SPDK_TEST_AUTOBUILD=1
|
|
||||||
SPDK_TEST_UNITTEST=1
|
|
||||||
SPDK_TEST_BLOCKDEV=1
|
|
||||||
SPDK_TEST_PMDK=1
|
|
||||||
SPDK_TEST_ISAL=1
|
|
||||||
SPDK_TEST_REDUCE=1
|
|
||||||
SPDK_TEST_CRYPTO=1
|
|
||||||
SPDK_TEST_FTL=1
|
|
||||||
SPDK_TEST_OCF=1
|
|
||||||
SPDK_TEST_RAID5=1
|
|
||||||
SPDK_TEST_RBD=1
|
|
||||||
SPDK_RUN_ASAN=1
|
|
||||||
SPDK_RUN_UBSAN=1
|
|
||||||
EOF
|
|
||||||
|
|
||||||
mkdir -p $HOME/output
|
|
||||||
rootdir="$output_dir"
|
|
||||||
source $HOME/autorun-spdk.conf
|
|
||||||
source $output_dir/test/common/autotest_common.sh
|
|
||||||
|
|
||||||
# Prepare separate, fixed, cmdline for the FreeBSD, Issue #1397.
|
|
||||||
if [[ $OSID == freebsd ]]; then
|
|
||||||
config_params="--enable-debug --enable-werror"
|
|
||||||
config_params+=" --with-idxd --with-fio=/usr/src/fio"
|
|
||||||
config_params+=" --disable-unit-tests --without-isal"
|
|
||||||
MAKE=gmake
|
|
||||||
else
|
|
||||||
config_params="$(get_config_params)"
|
|
||||||
fi
|
|
||||||
$output_dir/configure $(echo $config_params | sed 's/--enable-coverage//g')
|
|
||||||
if [[ $OSID != freebsd ]]; then
|
|
||||||
$MAKE -C $output_dir $MAKEFLAGS include/spdk/config.h
|
|
||||||
CONFIG_OCF_PATH="$output_dir/ocf" $MAKE -C $output_dir/lib/env_ocf $MAKEFLAGS exportlib O=$output_dir/build/ocf.a
|
|
||||||
$output_dir/configure $config_params --with-ocf=$output_dir/build/ocf.a --with-shared
|
|
||||||
fi
|
|
||||||
$MAKE -C $output_dir $MAKEFLAGS
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_qat() {
|
|
||||||
|
|
||||||
if [ "$PACKAGEMNG" = "dnf" ]; then
|
|
||||||
sudo dnf install -y libudev-devel
|
|
||||||
elif [ "$PACKAGEMNG" = "apt-get" ]; then
|
|
||||||
sudo apt-get install -y libudev-dev
|
|
||||||
fi
|
|
||||||
|
|
||||||
if echo $CONF | grep -q qat; then
|
|
||||||
qat_tarball=$(basename $DRIVER_LOCATION_QAT)
|
|
||||||
kernel_maj=$(uname -r | cut -d'.' -f1)
|
|
||||||
kernel_min=$(uname -r | cut -d'.' -f2)
|
|
||||||
|
|
||||||
sudo modprobe -r qat_c62x
|
|
||||||
if [ -d /QAT ]; then
|
|
||||||
sudo rm -rf /QAT/
|
|
||||||
fi
|
|
||||||
|
|
||||||
sudo mkdir /QAT
|
|
||||||
|
|
||||||
wget $DRIVER_LOCATION_QAT
|
|
||||||
sudo cp $qat_tarball /QAT/
|
|
||||||
(cd /QAT && sudo tar zxof /QAT/$qat_tarball)
|
|
||||||
|
|
||||||
#The driver version 1.7.l.4.3.0-00033 contains a reference to a deprecated function. Remove it so the build won't fail.
|
|
||||||
if [ $kernel_maj -le 4 ]; then
|
|
||||||
if [ $kernel_min -le 17 ]; then
|
|
||||||
sudo sed -i 's/rdtscll(timestamp);/timestamp = rdtsc_ordered();/g' \
|
|
||||||
/QAT/quickassist/utilities/osal/src/linux/kernel_space/OsalServices.c || true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
(cd /QAT && sudo ./configure --enable-icp-sriov=host && sudo make install)
|
|
||||||
|
|
||||||
if sudo service qat_service start; then
|
|
||||||
echo "failed to start the qat service. Something may be wrong with your device or package."
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_rocksdb() {
|
|
||||||
if echo $CONF | grep -q rocksdb; then
|
|
||||||
# Rocksdb is installed for use with the blobfs tests.
|
|
||||||
if [ ! -d /usr/src/rocksdb ]; then
|
|
||||||
git clone "${GIT_REPO_ROCKSDB}"
|
|
||||||
git -C ./rocksdb checkout spdk-v5.6.1
|
|
||||||
sudo mv rocksdb /usr/src/
|
|
||||||
else
|
|
||||||
sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1
|
|
||||||
echo "rocksdb already in /usr/src. Not checking out again"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_fio() {
|
|
||||||
if echo $CONF | grep -q fio; then
|
|
||||||
# This version of fio is installed in /usr/src/fio to enable
|
|
||||||
# building the spdk fio plugin.
|
|
||||||
local fio_version="fio-3.19"
|
|
||||||
|
|
||||||
if [ ! -d /usr/src/fio ]; then
|
|
||||||
if [ ! -d fio ]; then
|
|
||||||
git clone "${GIT_REPO_FIO}"
|
|
||||||
sudo mv fio /usr/src/
|
|
||||||
else
|
|
||||||
sudo mv fio /usr/src/
|
|
||||||
fi
|
|
||||||
(
|
|
||||||
git -C /usr/src/fio checkout master \
|
|
||||||
&& git -C /usr/src/fio pull \
|
|
||||||
&& git -C /usr/src/fio checkout $fio_version \
|
|
||||||
&& if [ $OSID == 'freebsd' ]; then
|
|
||||||
gmake -C /usr/src/fio -j${jobs} \
|
|
||||||
&& sudo gmake -C /usr/src/fio install
|
|
||||||
else
|
|
||||||
make -C /usr/src/fio -j${jobs} \
|
|
||||||
&& sudo make -C /usr/src/fio install
|
|
||||||
fi
|
|
||||||
)
|
|
||||||
else
|
|
||||||
echo "fio already in /usr/src/fio. Not installing"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_flamegraph() {
|
|
||||||
if echo $CONF | grep -q flamegraph; then
|
|
||||||
# Flamegraph is used when printing out timing graphs for the tests.
|
|
||||||
if [ ! -d /usr/local/FlameGraph ]; then
|
|
||||||
git clone "${GIT_REPO_FLAMEGRAPH}"
|
|
||||||
mkdir -p /usr/local
|
|
||||||
sudo mv FlameGraph /usr/local/FlameGraph
|
|
||||||
else
|
|
||||||
echo "flamegraph already installed. Skipping"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_qemu() {
|
|
||||||
if echo $CONF | grep -q qemu; then
|
|
||||||
# Two versions of QEMU are used in the tests.
|
|
||||||
# Stock QEMU is used for vhost. A special fork
|
|
||||||
# is used to test OCSSDs. Install both.
|
|
||||||
|
|
||||||
# Packaged QEMU
|
|
||||||
if [ "$PACKAGEMNG" = "dnf" ]; then
|
|
||||||
sudo dnf install -y qemu-system-x86 qemu-img
|
|
||||||
elif [ "$PACKAGEMNG" = "apt-get" ]; then
|
|
||||||
sudo apt-get install -y qemu-system-x86 qemu-utils
|
|
||||||
elif [ "$PACKAGEMNG" = "pacman" ]; then
|
|
||||||
sudo pacman -Sy --needed --noconfirm qemu
|
|
||||||
elif [[ $PACKAGEMNG == "yum" ]]; then
|
|
||||||
sudo yum install -y qemu-system-x86 qemu-img
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Forked QEMU
|
|
||||||
SPDK_QEMU_BRANCH=spdk-5.0.0
|
|
||||||
mkdir -p qemu
|
|
||||||
if [ ! -d "qemu/$SPDK_QEMU_BRANCH" ]; then
|
|
||||||
git -C ./qemu clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$SPDK_QEMU_BRANCH"
|
|
||||||
else
|
|
||||||
echo "qemu already checked out. Skipping"
|
|
||||||
fi
|
|
||||||
|
|
||||||
declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH")
|
|
||||||
if ((gcc_version >= 9)); then
|
|
||||||
# GCC 9 fails to compile Qemu due to some old warnings which were not detected by older versions.
|
|
||||||
opt_params+=("--extra-cflags=-Wno-error=stringop-truncation -Wno-error=deprecated-declarations -Wno-error=incompatible-pointer-types -Wno-error=format-truncation")
|
|
||||||
opt_params+=("--disable-glusterfs")
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Most tsocks proxies rely on a configuration file in /etc/tsocks.conf.
|
|
||||||
# If using tsocks, please make sure to complete this config before trying to build qemu.
|
|
||||||
if echo $CONF | grep -q tsocks; then
|
|
||||||
if hash tsocks 2> /dev/null; then
|
|
||||||
opt_params+=("--with-git='tsocks git'")
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules
|
|
||||||
sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config
|
|
||||||
sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules
|
|
||||||
sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config
|
|
||||||
# The qemu configure script places several output files in the CWD.
|
|
||||||
(cd qemu/$SPDK_QEMU_BRANCH && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa)
|
|
||||||
|
|
||||||
make -C ./qemu/$SPDK_QEMU_BRANCH -j${jobs}
|
|
||||||
sudo make -C ./qemu/$SPDK_QEMU_BRANCH install
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_vpp() {
|
|
||||||
if echo $CONF | grep -q vpp; then
|
|
||||||
if [ -d /usr/local/src/vpp ]; then
|
|
||||||
echo "vpp already cloned."
|
|
||||||
if [ ! -d /usr/local/src/vpp/build-root ]; then
|
|
||||||
echo "build-root has not been done"
|
|
||||||
echo "remove the $(pwd) and start again"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
git clone "${GIT_REPO_VPP}"
|
|
||||||
git -C ./vpp checkout v19.04.2
|
|
||||||
|
|
||||||
if [ "${OSID}" == 'fedora' ]; then
|
|
||||||
if [ ${OSVERSION} -eq 29 ]; then
|
|
||||||
git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora29-fix.patch
|
|
||||||
fi
|
|
||||||
if [ ${OSVERSION} -eq 30 ]; then
|
|
||||||
git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora30-fix.patch
|
|
||||||
fi
|
|
||||||
if ((OVERSION == 31)); then
|
|
||||||
git -C ./vpp apply "$VM_SETUP_PATH/patch/vpp/fedora31-fix.patch"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# vpp depends on python-ply, however some packages on different Fedoras don't
|
|
||||||
# provide ply.lex. To make sure vpp won't fail, try to reinstall ply via pip.
|
|
||||||
sudo pip3 uninstall -y ply || true
|
|
||||||
sudo pip3 install ply || true
|
|
||||||
|
|
||||||
# Installing required dependencies for building VPP
|
|
||||||
yes | make -C ./vpp install-dep
|
|
||||||
|
|
||||||
make -C ./vpp build -j${jobs}
|
|
||||||
|
|
||||||
sudo mv ./vpp /usr/local/src/vpp-19.04
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_nvmecli() {
|
|
||||||
if echo $CONF | grep -q nvmecli; then
|
|
||||||
SPDK_NVME_CLI_BRANCH=spdk-1.6
|
|
||||||
if [ ! -d nvme-cli ]; then
|
|
||||||
git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH"
|
|
||||||
else
|
|
||||||
echo "nvme-cli already checked out. Skipping"
|
|
||||||
fi
|
|
||||||
if [ ! -d "/usr/local/src/nvme-cli" ]; then
|
|
||||||
# Changes required for SPDK are already merged on top of
|
|
||||||
# nvme-cli, however not released yet.
|
|
||||||
# Support for SPDK should be released in nvme-cli >1.11.1
|
|
||||||
git clone "https://github.com/linux-nvme/nvme-cli.git" "nvme-cli-cuse"
|
|
||||||
git -C ./nvme-cli-cuse checkout "e770466615096a6d41f038a28819b00bc3078e1d"
|
|
||||||
make -C ./nvme-cli-cuse
|
|
||||||
sudo mv ./nvme-cli-cuse /usr/local/src/nvme-cli
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_libiscsi() {
|
|
||||||
if echo $CONF | grep -q libiscsi; then
|
|
||||||
# We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need
|
|
||||||
# to later. Cloning from git is just future proofing the machines.
|
|
||||||
if [ ! -d libiscsi ]; then
|
|
||||||
git clone "${GIT_REPO_LIBISCSI}"
|
|
||||||
else
|
|
||||||
echo "libiscsi already checked out. Skipping"
|
|
||||||
fi
|
|
||||||
(cd libiscsi && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi)
|
|
||||||
make -C ./libiscsi -j${jobs}
|
|
||||||
sudo make -C ./libiscsi install
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function install_git() {
|
|
||||||
sudo yum install -y zlib-devel curl-devel
|
|
||||||
tar -xzof <(wget -qO- "$GIT_REPO_GIT")
|
|
||||||
(cd git-${GIT_VERSION} \
|
|
||||||
&& make configure \
|
|
||||||
&& ./configure --prefix=/usr/local/git \
|
|
||||||
&& sudo make -j${jobs} install)
|
|
||||||
sudo sh -c "echo 'export PATH=/usr/local/git/bin:$PATH' >> /etc/bashrc"
|
|
||||||
exec $SHELL
|
|
||||||
}
|
|
||||||
|
|
||||||
function usage() {
|
function usage() {
|
||||||
echo "This script is intended to automate the environment setup for a linux virtual machine."
|
echo "This script is intended to automate the environment setup for a linux virtual machine."
|
||||||
echo "Please run this script as your regular user. The script will make calls to sudo as needed."
|
echo "Please run this script as your regular user. The script will make calls to sudo as needed."
|
||||||
@ -406,38 +112,6 @@ if [ -n "$CONF_PATH" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
cd ~
|
cd ~
|
||||||
GIT_VERSION=2.25.1
|
|
||||||
: ${GIT_REPO_SPDK=https://github.com/spdk/spdk.git}
|
|
||||||
export GIT_REPO_SPDK
|
|
||||||
: ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git}
|
|
||||||
export GIT_REPO_DPDK
|
|
||||||
: ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb}
|
|
||||||
export GIT_REPO_ROCKSDB
|
|
||||||
: ${GIT_REPO_FIO=http://git.kernel.dk/fio.git}
|
|
||||||
export GIT_REPO_FIO
|
|
||||||
: ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git}
|
|
||||||
export GIT_REPO_FLAMEGRAPH
|
|
||||||
: ${GIT_REPO_QEMU=https://github.com/spdk/qemu}
|
|
||||||
export GIT_REPO_QEMU
|
|
||||||
: ${GIT_REPO_VPP=https://gerrit.fd.io/r/vpp}
|
|
||||||
export GIT_REPO_VPP
|
|
||||||
: ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi}
|
|
||||||
export GIT_REPO_LIBISCSI
|
|
||||||
: ${GIT_REPO_SPDK_NVME_CLI=https://github.com/spdk/nvme-cli}
|
|
||||||
export GIT_REPO_SPDK_NVME_CLI
|
|
||||||
: ${GIT_REPO_INTEL_IPSEC_MB=https://github.com/spdk/intel-ipsec-mb.git}
|
|
||||||
export GIT_REPO_INTEL_IPSEC_MB
|
|
||||||
: ${DRIVER_LOCATION_QAT=https://01.org/sites/default/files/downloads//qat1.7.l.4.9.0-00008.tar.gz}
|
|
||||||
export DRIVER_LOCATION_QAT
|
|
||||||
: ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz}
|
|
||||||
export GIT_REPO_GIT
|
|
||||||
|
|
||||||
if [ $PACKAGEMNG == 'pkg' ]; then
|
|
||||||
jobs=$(($(sysctl -n hw.ncpu) * 2))
|
|
||||||
else
|
|
||||||
jobs=$(($(nproc) * 2))
|
|
||||||
fi
|
|
||||||
|
|
||||||
if $UPGRADE; then
|
if $UPGRADE; then
|
||||||
upgrade
|
upgrade
|
||||||
fi
|
fi
|
||||||
@ -459,22 +133,7 @@ git -C spdk_repo/spdk config submodule.dpdk.url "${GIT_REPO_DPDK}"
|
|||||||
git -C spdk_repo/spdk config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}"
|
git -C spdk_repo/spdk config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}"
|
||||||
git -C spdk_repo/spdk submodule update --init --recursive
|
git -C spdk_repo/spdk submodule update --init --recursive
|
||||||
|
|
||||||
sudo mkdir -p /usr/src
|
source "$vmsetupdir/pkgdep/git"
|
||||||
|
|
||||||
if [ $OSID != 'freebsd' ]; then
|
|
||||||
install_libiscsi &
|
|
||||||
install_vpp &
|
|
||||||
install_nvmecli &
|
|
||||||
install_qat &
|
|
||||||
install_rocksdb &
|
|
||||||
install_flamegraph &
|
|
||||||
install_qemu &
|
|
||||||
fi
|
|
||||||
install_fio &
|
|
||||||
|
|
||||||
wait
|
|
||||||
|
|
||||||
install_reference_spdk
|
|
||||||
|
|
||||||
# create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file.
|
# create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file.
|
||||||
# By setting any one of the values below to 0, you can skip that specific test. If you are
|
# By setting any one of the values below to 0, you can skip that specific test. If you are
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user