From 3920b816817041751ce5a167f283289197ace9e5 Mon Sep 17 00:00:00 2001 From: Michal Berger Date: Mon, 18 May 2020 17:28:25 +0200 Subject: [PATCH] vm_setup: Make vm_setup.sh to use specific source for repo installations Change-Id: I01613720e078a0fa151ed9a4a17e2faf131510bc Signed-off-by: Michal Berger Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2526 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Karol Latecki Reviewed-by: Tomasz Zawadzki --- test/common/config/vm_setup.sh | 343 +-------------------------------- 1 file changed, 1 insertion(+), 342 deletions(-) diff --git a/test/common/config/vm_setup.sh b/test/common/config/vm_setup.sh index 84f09380d..5460553fb 100755 --- a/test/common/config/vm_setup.sh +++ b/test/common/config/vm_setup.sh @@ -41,300 +41,6 @@ else PACKAGEMNG='undefined' fi -function install_reference_spdk() { - local last_release - local output_dir - local config_params - local rootdir - - # Create a reference SPDK build for ABI tests - if echo $CONF | grep -q refspdk; then - git -C spdk_repo/spdk fetch --tags - last_release=$(git -C spdk_repo/spdk tag | sort --version-sort | grep -v rc | tail -n1) - git -C spdk_repo/spdk checkout $last_release - git -C spdk_repo/spdk submodule update --init - output_dir="$HOME/spdk_$(tr . _ < <(tr -d '[:alpha:]' <<< $last_release))" - - cp -r spdk_repo/spdk $output_dir - - cat > $HOME/autorun-spdk.conf << EOF -SPDK_BUILD_SHARED_OBJECT=1 -SPDK_TEST_AUTOBUILD=1 -SPDK_TEST_UNITTEST=1 -SPDK_TEST_BLOCKDEV=1 -SPDK_TEST_PMDK=1 -SPDK_TEST_ISAL=1 -SPDK_TEST_REDUCE=1 -SPDK_TEST_CRYPTO=1 -SPDK_TEST_FTL=1 -SPDK_TEST_OCF=1 -SPDK_TEST_RAID5=1 -SPDK_TEST_RBD=1 -SPDK_RUN_ASAN=1 -SPDK_RUN_UBSAN=1 -EOF - - mkdir -p $HOME/output - rootdir="$output_dir" - source $HOME/autorun-spdk.conf - source $output_dir/test/common/autotest_common.sh - - # Prepare separate, fixed, cmdline for the FreeBSD, Issue #1397. - if [[ $OSID == freebsd ]]; then - config_params="--enable-debug --enable-werror" - config_params+=" --with-idxd --with-fio=/usr/src/fio" - config_params+=" --disable-unit-tests --without-isal" - MAKE=gmake - else - config_params="$(get_config_params)" - fi - $output_dir/configure $(echo $config_params | sed 's/--enable-coverage//g') - if [[ $OSID != freebsd ]]; then - $MAKE -C $output_dir $MAKEFLAGS include/spdk/config.h - CONFIG_OCF_PATH="$output_dir/ocf" $MAKE -C $output_dir/lib/env_ocf $MAKEFLAGS exportlib O=$output_dir/build/ocf.a - $output_dir/configure $config_params --with-ocf=$output_dir/build/ocf.a --with-shared - fi - $MAKE -C $output_dir $MAKEFLAGS - fi -} - -function install_qat() { - - if [ "$PACKAGEMNG" = "dnf" ]; then - sudo dnf install -y libudev-devel - elif [ "$PACKAGEMNG" = "apt-get" ]; then - sudo apt-get install -y libudev-dev - fi - - if echo $CONF | grep -q qat; then - qat_tarball=$(basename $DRIVER_LOCATION_QAT) - kernel_maj=$(uname -r | cut -d'.' -f1) - kernel_min=$(uname -r | cut -d'.' -f2) - - sudo modprobe -r qat_c62x - if [ -d /QAT ]; then - sudo rm -rf /QAT/ - fi - - sudo mkdir /QAT - - wget $DRIVER_LOCATION_QAT - sudo cp $qat_tarball /QAT/ - (cd /QAT && sudo tar zxof /QAT/$qat_tarball) - - #The driver version 1.7.l.4.3.0-00033 contains a reference to a deprecated function. Remove it so the build won't fail. - if [ $kernel_maj -le 4 ]; then - if [ $kernel_min -le 17 ]; then - sudo sed -i 's/rdtscll(timestamp);/timestamp = rdtsc_ordered();/g' \ - /QAT/quickassist/utilities/osal/src/linux/kernel_space/OsalServices.c || true - fi - fi - - (cd /QAT && sudo ./configure --enable-icp-sriov=host && sudo make install) - - if sudo service qat_service start; then - echo "failed to start the qat service. Something may be wrong with your device or package." - fi - fi -} - -function install_rocksdb() { - if echo $CONF | grep -q rocksdb; then - # Rocksdb is installed for use with the blobfs tests. - if [ ! -d /usr/src/rocksdb ]; then - git clone "${GIT_REPO_ROCKSDB}" - git -C ./rocksdb checkout spdk-v5.6.1 - sudo mv rocksdb /usr/src/ - else - sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1 - echo "rocksdb already in /usr/src. Not checking out again" - fi - fi -} - -function install_fio() { - if echo $CONF | grep -q fio; then - # This version of fio is installed in /usr/src/fio to enable - # building the spdk fio plugin. - local fio_version="fio-3.19" - - if [ ! -d /usr/src/fio ]; then - if [ ! -d fio ]; then - git clone "${GIT_REPO_FIO}" - sudo mv fio /usr/src/ - else - sudo mv fio /usr/src/ - fi - ( - git -C /usr/src/fio checkout master \ - && git -C /usr/src/fio pull \ - && git -C /usr/src/fio checkout $fio_version \ - && if [ $OSID == 'freebsd' ]; then - gmake -C /usr/src/fio -j${jobs} \ - && sudo gmake -C /usr/src/fio install - else - make -C /usr/src/fio -j${jobs} \ - && sudo make -C /usr/src/fio install - fi - ) - else - echo "fio already in /usr/src/fio. Not installing" - fi - fi -} - -function install_flamegraph() { - if echo $CONF | grep -q flamegraph; then - # Flamegraph is used when printing out timing graphs for the tests. - if [ ! -d /usr/local/FlameGraph ]; then - git clone "${GIT_REPO_FLAMEGRAPH}" - mkdir -p /usr/local - sudo mv FlameGraph /usr/local/FlameGraph - else - echo "flamegraph already installed. Skipping" - fi - fi -} - -function install_qemu() { - if echo $CONF | grep -q qemu; then - # Two versions of QEMU are used in the tests. - # Stock QEMU is used for vhost. A special fork - # is used to test OCSSDs. Install both. - - # Packaged QEMU - if [ "$PACKAGEMNG" = "dnf" ]; then - sudo dnf install -y qemu-system-x86 qemu-img - elif [ "$PACKAGEMNG" = "apt-get" ]; then - sudo apt-get install -y qemu-system-x86 qemu-utils - elif [ "$PACKAGEMNG" = "pacman" ]; then - sudo pacman -Sy --needed --noconfirm qemu - elif [[ $PACKAGEMNG == "yum" ]]; then - sudo yum install -y qemu-system-x86 qemu-img - fi - - # Forked QEMU - SPDK_QEMU_BRANCH=spdk-5.0.0 - mkdir -p qemu - if [ ! -d "qemu/$SPDK_QEMU_BRANCH" ]; then - git -C ./qemu clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$SPDK_QEMU_BRANCH" - else - echo "qemu already checked out. Skipping" - fi - - declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH") - if ((gcc_version >= 9)); then - # GCC 9 fails to compile Qemu due to some old warnings which were not detected by older versions. - opt_params+=("--extra-cflags=-Wno-error=stringop-truncation -Wno-error=deprecated-declarations -Wno-error=incompatible-pointer-types -Wno-error=format-truncation") - opt_params+=("--disable-glusterfs") - fi - - # Most tsocks proxies rely on a configuration file in /etc/tsocks.conf. - # If using tsocks, please make sure to complete this config before trying to build qemu. - if echo $CONF | grep -q tsocks; then - if hash tsocks 2> /dev/null; then - opt_params+=("--with-git='tsocks git'") - fi - fi - - sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules - sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config - sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules - sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config - # The qemu configure script places several output files in the CWD. - (cd qemu/$SPDK_QEMU_BRANCH && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa) - - make -C ./qemu/$SPDK_QEMU_BRANCH -j${jobs} - sudo make -C ./qemu/$SPDK_QEMU_BRANCH install - fi -} - -function install_vpp() { - if echo $CONF | grep -q vpp; then - if [ -d /usr/local/src/vpp ]; then - echo "vpp already cloned." - if [ ! -d /usr/local/src/vpp/build-root ]; then - echo "build-root has not been done" - echo "remove the $(pwd) and start again" - exit 1 - fi - else - git clone "${GIT_REPO_VPP}" - git -C ./vpp checkout v19.04.2 - - if [ "${OSID}" == 'fedora' ]; then - if [ ${OSVERSION} -eq 29 ]; then - git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora29-fix.patch - fi - if [ ${OSVERSION} -eq 30 ]; then - git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora30-fix.patch - fi - if ((OVERSION == 31)); then - git -C ./vpp apply "$VM_SETUP_PATH/patch/vpp/fedora31-fix.patch" - fi - fi - - # vpp depends on python-ply, however some packages on different Fedoras don't - # provide ply.lex. To make sure vpp won't fail, try to reinstall ply via pip. - sudo pip3 uninstall -y ply || true - sudo pip3 install ply || true - - # Installing required dependencies for building VPP - yes | make -C ./vpp install-dep - - make -C ./vpp build -j${jobs} - - sudo mv ./vpp /usr/local/src/vpp-19.04 - fi - fi -} - -function install_nvmecli() { - if echo $CONF | grep -q nvmecli; then - SPDK_NVME_CLI_BRANCH=spdk-1.6 - if [ ! -d nvme-cli ]; then - git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH" - else - echo "nvme-cli already checked out. Skipping" - fi - if [ ! -d "/usr/local/src/nvme-cli" ]; then - # Changes required for SPDK are already merged on top of - # nvme-cli, however not released yet. - # Support for SPDK should be released in nvme-cli >1.11.1 - git clone "https://github.com/linux-nvme/nvme-cli.git" "nvme-cli-cuse" - git -C ./nvme-cli-cuse checkout "e770466615096a6d41f038a28819b00bc3078e1d" - make -C ./nvme-cli-cuse - sudo mv ./nvme-cli-cuse /usr/local/src/nvme-cli - fi - fi -} - -function install_libiscsi() { - if echo $CONF | grep -q libiscsi; then - # We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need - # to later. Cloning from git is just future proofing the machines. - if [ ! -d libiscsi ]; then - git clone "${GIT_REPO_LIBISCSI}" - else - echo "libiscsi already checked out. Skipping" - fi - (cd libiscsi && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi) - make -C ./libiscsi -j${jobs} - sudo make -C ./libiscsi install - fi -} - -function install_git() { - sudo yum install -y zlib-devel curl-devel - tar -xzof <(wget -qO- "$GIT_REPO_GIT") - (cd git-${GIT_VERSION} \ - && make configure \ - && ./configure --prefix=/usr/local/git \ - && sudo make -j${jobs} install) - sudo sh -c "echo 'export PATH=/usr/local/git/bin:$PATH' >> /etc/bashrc" - exec $SHELL -} - function usage() { echo "This script is intended to automate the environment setup for a linux virtual machine." echo "Please run this script as your regular user. The script will make calls to sudo as needed." @@ -406,38 +112,6 @@ if [ -n "$CONF_PATH" ]; then fi cd ~ -GIT_VERSION=2.25.1 -: ${GIT_REPO_SPDK=https://github.com/spdk/spdk.git} -export GIT_REPO_SPDK -: ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git} -export GIT_REPO_DPDK -: ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb} -export GIT_REPO_ROCKSDB -: ${GIT_REPO_FIO=http://git.kernel.dk/fio.git} -export GIT_REPO_FIO -: ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git} -export GIT_REPO_FLAMEGRAPH -: ${GIT_REPO_QEMU=https://github.com/spdk/qemu} -export GIT_REPO_QEMU -: ${GIT_REPO_VPP=https://gerrit.fd.io/r/vpp} -export GIT_REPO_VPP -: ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi} -export GIT_REPO_LIBISCSI -: ${GIT_REPO_SPDK_NVME_CLI=https://github.com/spdk/nvme-cli} -export GIT_REPO_SPDK_NVME_CLI -: ${GIT_REPO_INTEL_IPSEC_MB=https://github.com/spdk/intel-ipsec-mb.git} -export GIT_REPO_INTEL_IPSEC_MB -: ${DRIVER_LOCATION_QAT=https://01.org/sites/default/files/downloads//qat1.7.l.4.9.0-00008.tar.gz} -export DRIVER_LOCATION_QAT -: ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz} -export GIT_REPO_GIT - -if [ $PACKAGEMNG == 'pkg' ]; then - jobs=$(($(sysctl -n hw.ncpu) * 2)) -else - jobs=$(($(nproc) * 2)) -fi - if $UPGRADE; then upgrade fi @@ -459,22 +133,7 @@ git -C spdk_repo/spdk config submodule.dpdk.url "${GIT_REPO_DPDK}" git -C spdk_repo/spdk config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}" git -C spdk_repo/spdk submodule update --init --recursive -sudo mkdir -p /usr/src - -if [ $OSID != 'freebsd' ]; then - install_libiscsi & - install_vpp & - install_nvmecli & - install_qat & - install_rocksdb & - install_flamegraph & - install_qemu & -fi -install_fio & - -wait - -install_reference_spdk +source "$vmsetupdir/pkgdep/git" # create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file. # By setting any one of the values below to 0, you can skip that specific test. If you are