ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test/common: remove directory switching from vm_setup.sh

Browse Source 
By keeping the main shell in the same directory the whole time, we will
be able to parallelize a lot of the work of cloning and making
dependencies. This should significantly reduce make time to run this
script on a target machine.

Change-Id: I5efbd9c8f50b95411cbe7be69905b08df557344e
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/426980
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Seth Howell 2018-09-26 13:27:23 -07:00 committed by Jim Harris
parent 0c04c729ba
commit 5ae7353072
1 changed files with 51 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

124
test/common/config/vm_setup.sh
View File

@ -69,6 +69,8 @@ if [ ! -f "$CONF_PATH" ]; then
exit 1 exit 1
fi fi
cd ~
source "$CONF_PATH" source "$CONF_PATH"
jobs=$(($(nproc)*2)) jobs=$(($(nproc)*2))
@ -81,19 +83,15 @@ if $INSTALL; then
sudo dnf install -y git sudo dnf install -y git
fi fi
cd ~ mkdir -p spdk_repo/output
mkdir -p spdk_repo
cd spdk_repo if [ -d spdk_repo/spdk ]; then
mkdir -p output
if [ -d spdk ]; then
echo "spdk source already present, not cloning" echo "spdk source already present, not cloning"
else else
git clone "${GIT_REPO_SPDK}" git -C spdk_repo clone "${GIT_REPO_SPDK}"
fi fi
cd spdk git -C spdk_repo/spdk config submodule.dpdk.url "${GIT_REPO_DPDK}"
git config submodule.dpdk.url "${GIT_REPO_DPDK}" git -C spdk_repo/spdk submodule update --init --recursive
git submodule update --init --recursive
if $INSTALL; then if $INSTALL; then
sudo ./scripts/pkgdep.sh sudo ./scripts/pkgdep.sh
@ -138,8 +136,6 @@ if $INSTALL; then
sshfs sshfs
fi fi
cd ~
if echo $CONF | grep -q librxe; then if echo $CONF | grep -q librxe; then
# rxe_cfg is used in the NVMe-oF tests # rxe_cfg is used in the NVMe-oF tests
# The librxe-dev repository provides a command line tool called rxe_cfg which makes it # The librxe-dev repository provides a command line tool called rxe_cfg which makes it
@ -154,11 +150,9 @@ if echo $CONF | grep -q librxe; then
git clone "${GIT_REPO_LIBRXE}" git clone "${GIT_REPO_LIBRXE}"
fi fi
cd librxe-dev ./librxe-dev/configure --libdir=/usr/lib64/ --prefix=
./configure --libdir=/usr/lib64/ --prefix= make -C librxe-dev -j${jobs}
make -j${jobs} sudo make -C librxe-dev install
sudo make install
cd ~
fi fi
fi fi
@ -171,28 +165,23 @@ if echo $CONF | grep -q iscsi; then
OPEN_ISCSI_VER='iscsiadm version 6.2.0.874' OPEN_ISCSI_VER='iscsiadm version 6.2.0.874'
if [ "$CURRENT_VERSION" == "$OPEN_ISCSI_VER" ]; then if [ "$CURRENT_VERSION" == "$OPEN_ISCSI_VER" ]; then
if [ ! -d open-iscsi-install ]; then if [ ! -d open-iscsi-install ]; then
mkdir -p open-iscsi-install mkdir -p open-iscsi-install/patches
cd open-iscsi-install sudo dnf download --downloaddir=./open-iscsi-install --source iscsi-initiator-utils
sudo dnf download --source iscsi-initiator-utils rpm2cpio open-iscsi-install/$(ls ~/open-iscsi-install) | cpio -D open-iscsi-install -idmv
rpm2cpio $(ls) | cpio -idmv mv open-iscsi-install/00* open-iscsi-install/patches/
mkdir -p patches git clone "${GIT_REPO_OPEN_ISCSI}" open-iscsi-install/open-iscsi
mv 00* patches/
git clone "${GIT_REPO_OPEN_ISCSI}"
cd open-iscsi
# the configurations of username and email are needed for applying patches to iscsiadm. # the configurations of username and email are needed for applying patches to iscsiadm.
git config user.name none git -C open-iscsi-install/open-iscsi config user.name none
git config user.email none git -C open-iscsi-install/open-iscsi config user.email none
git checkout 86e8892 git -C open-iscsi-install/open-iscsi checkout 86e8892
for patch in `ls ../patches`; do for patch in `ls open-iscsi-install/patches`; do
git am ../patches/$patch git -C open-iscsi-install/open-iscsi am ../patches/$patch
done done
sed -i '427s/.*/-1);/' usr/session_info.c sed -i '427s/.*/-1);/' open-iscsi-install/open-iscsi/usr/session_info.c
make -j${jobs} make -C open-iscsi-install/open-iscsi -j${jobs}
sudo make install sudo make -C open-iscsi-install/open-iscsi install
cd ~
else else
echo "custom open-iscsi install located, not reinstalling" echo "custom open-iscsi install located, not reinstalling"
fi fi
@ -225,20 +214,17 @@ if echo $CONF | grep -q fio; then
sudo mv fio /usr/src/ sudo mv fio /usr/src/
fi fi
( (
cd /usr/src/fio && git -C /usr/src/fio checkout master &&
git checkout master && git -C /usr/src/fio pull &&
git pull && git -C /usr/src/fio checkout fio-3.3 &&
git checkout fio-3.3 && make -C /usr/src/fio -j${jobs} &&
make -j${jobs} && sudo make -C /usr/src/fio install
sudo make install
) )
else else
echo "fio already in /usr/src/fio. Not installing" echo "fio already in /usr/src/fio. Not installing"
fi fi
fi fi
cd ~
if echo $CONF | grep -q flamegraph; then if echo $CONF | grep -q flamegraph; then
# Flamegraph is used when printing out timing graphs for the tests. # Flamegraph is used when printing out timing graphs for the tests.
if [ ! -d /usr/local/FlameGraph ]; then if [ ! -d /usr/local/FlameGraph ]; then
@ -254,15 +240,12 @@ if echo $CONF | grep -q qemu; then
# Qemu is used in the vhost tests. # Qemu is used in the vhost tests.
SPDK_QEMU_BRANCH=spdk-2.12 SPDK_QEMU_BRANCH=spdk-2.12
mkdir -p qemu mkdir -p qemu
cd qemu if [ ! -d "qemu/$SPDK_QEMU_BRANCH" ]; then
if [ ! -d "$SPDK_QEMU_BRANCH" ]; then git -C ./qemu clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$SPDK_QEMU_BRANCH"
git clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$SPDK_QEMU_BRANCH"
else else
echo "qemu already checked out. Skipping" echo "qemu already checked out. Skipping"
fi fi
cd "$SPDK_QEMU_BRANCH"
declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH") declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH")
# Most tsocks proxies rely on a configuration file in /etc/tsocks.conf. # Most tsocks proxies rely on a configuration file in /etc/tsocks.conf.
@ -273,14 +256,13 @@ if echo $CONF | grep -q qemu; then
fi fi
fi fi
./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa # The qemu configure script places several output files in the CWD.
(cd qemu/$SPDK_QEMU_BRANCH && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa)
make -j${jobs} make -C ./qemu/$SPDK_QEMU_BRANCH -j${jobs}
sudo make install sudo make -C ./qemu/$SPDK_QEMU_BRANCH install
fi fi
cd ~
if echo $CONF | grep -q vpp; then if echo $CONF | grep -q vpp; then
# Vector packet processing (VPP) is installed for use with iSCSI tests. # Vector packet processing (VPP) is installed for use with iSCSI tests.
# At least on fedora 28, the yum setup that vpp uses is deprecated and fails. # At least on fedora 28, the yum setup that vpp uses is deprecated and fails.
@ -291,15 +273,14 @@ if echo $CONF | grep -q vpp; then
echo "%_topdir $HOME/vpp_setup/src/rpm" >> ~/.rpmmacros echo "%_topdir $HOME/vpp_setup/src/rpm" >> ~/.rpmmacros
sudo dnf install perl-generators sudo dnf install perl-generators
mkdir -p ~/vpp_setup/src/rpm mkdir -p ~/vpp_setup/src/rpm
cd ~/vpp_setup/src/rpm mkdir -p vpp_setup/src/rpm/BUILD vpp_setup/src/rpm/RPMS vpp_setup/src/rpm/SOURCES \
mkdir -p BUILD RPMS SOURCES SPECS SRPMS vpp_setup/src/rpm/SPECS vpp_setup/src/rpm/SRPMS
dnf download --source redhat-rpm-config dnf download --downloaddir=./vpp_setup/src/rpm --source redhat-rpm-config
rpm -ivh redhat-rpm-config* rpm -ivh ~/vpp_setup/src/rpm/redhat-rpm-config*
sed -i s/"Requires: (annobin if gcc)"//g SPECS/redhat-rpm-config.spec sed -i s/"Requires: (annobin if gcc)"//g ~/vpp_setup/src/rpm/SPECS/redhat-rpm-config.spec
rpmbuild -ba SPECS/*.spec rpmbuild -ba ~/vpp_setup/src/rpm/SPECS/*.spec
sudo dnf remove -y --noautoremove redhat-rpm-config sudo dnf remove -y --noautoremove redhat-rpm-config
sudo rpm -Uvh RPMS/noarch/* sudo rpm -Uvh ~/vpp_setup/src/rpm/RPMS/noarch/*
cd -
fi fi
if [ -d vpp ]; then if [ -d vpp ]; then
@ -311,29 +292,26 @@ if echo $CONF | grep -q vpp; then
fi fi
else else
git clone "${GIT_REPO_VPP}" git clone "${GIT_REPO_VPP}"
cd vpp git -C ./vpp checkout v18.01.1
git checkout v18.01.1
# VPP 18.01.1 does not support OpenSSL 1.1. # VPP 18.01.1 does not support OpenSSL 1.1.
# For compilation, a compatibility package is used temporarily. # For compilation, a compatibility package is used temporarily.
sudo dnf install -y --allowerasing compat-openssl10-devel sudo dnf install -y --allowerasing compat-openssl10-devel
# Installing required dependencies for building VPP # Installing required dependencies for building VPP
yes | make install-dep yes | make -C ./vpp install-dep
make pkg-rpm -j${jobs} make -C ./vpp pkg-rpm -j${jobs}
# Reinstall latest OpenSSL devel package. # Reinstall latest OpenSSL devel package.
sudo dnf install -y --allowerasing openssl-devel sudo dnf install -y --allowerasing openssl-devel
cd build-root
sudo dnf install -y \ sudo dnf install -y \
./vpp-lib-18.01.1-release.x86_64.rpm \ ./vpp/build_root/vpp-lib-18.01.1-release.x86_64.rpm \
./vpp-devel-18.01.1-release.x86_64.rpm \ ./vpp/build_root/vpp-devel-18.01.1-release.x86_64.rpm \
./vpp-18.01.1-release.x86_64.rpm ./vpp/build_root/vpp-18.01.1-release.x86_64.rpm
# Since hugepage configuration is done via spdk/scripts/setup.sh, # Since hugepage configuration is done via spdk/scripts/setup.sh,
# this default config is not needed. # this default config is not needed.
# #
# NOTE: Parameters kernel.shmmax and vm.max_map_count are set to # NOTE: Parameters kernel.shmmax and vm.max_map_count are set to
# very low count and cause issues with hugepage total sizes above 1GB. # very low count and cause issues with hugepage total sizes above 1GB.
sudo rm -f /etc/sysctl.d/80-vpp.conf sudo rm -f /etc/sysctl.d/80-vpp.conf
cd ~
fi fi
fi fi
@ -353,11 +331,9 @@ if echo $CONF | grep -q libiscsi; then
else else
echo "libiscsi already checked out. Skipping" echo "libiscsi already checked out. Skipping"
fi fi
cd libiscsi ( cd libiscsi && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi)
./autogen.sh make -C ./libiscsi -j${jobs}
./configure --prefix=/usr/local/libiscsi sudo make -C ./libiscsi install
make -j${jobs}
sudo make install
fi fi
# create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file. # create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file.