ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vm_setup: Introduce $GIT_REPOS

Browse Source 
Allow to specify a target place for all the git sources and use it
as a full path (instead of cding directly into $HOME). $HOME is
still kept as a default.

Signed-off-by: Michal Berger <michalx.berger@intel.com>
Change-Id: Idb60ff8f8108368de2ac241971d221142198c89a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2528
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
This commit is contained in:
Michal Berger 2020-06-02 10:48:47 +02:00 committed by Tomasz Zawadzki
parent 40d20acb72
commit 70dfde6f1e
2 changed files with 60 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
test/common/config/pkgdep/git
View File

@ -1,14 +1,14 @@
function install_spdk() { function install_spdk() {
mkdir -p spdk_repo/output || echo "Can not create spdk_repo/output directory." mkdir -p "$GIT_REPOS/spdk_repo/output" || echo "Can not create spdk_repo/output directory."
if [ -d spdk_repo/spdk ]; then if [[ -d $GIT_REPOS/spdk_repo/spdk ]]; then
echo "spdk source already present, not cloning" echo "spdk source already present, not cloning"
else else
git -C spdk_repo clone "${GIT_REPO_SPDK}" git -C "$GIT_REPOS/spdk_repo" clone "${GIT_REPO_SPDK}"
fi fi
git -C spdk_repo/spdk config submodule.dpdk.url "${GIT_REPO_DPDK}" git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.dpdk.url "${GIT_REPO_DPDK}"
git -C spdk_repo/spdk config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}" git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}"
git -C spdk_repo/spdk submodule update --init --recursive git -C "$GIT_REPOS/spdk_repo/spdk" submodule update --init --recursive
} }
function install_refspdk() { function install_refspdk() {
@ -18,12 +18,12 @@ function install_refspdk() {
local rootdir local rootdir
# Create a reference SPDK build for ABI tests # Create a reference SPDK build for ABI tests
git -C spdk_repo/spdk fetch --tags git -C "$GIT_REPOS/spdk_repo/spdk" fetch --tags
last_release=$(git -C spdk_repo/spdk tag | sort --version-sort | grep -v rc | tail -n1) last_release=$(git -C "$GIT_REPOS/spdk_repo/spdk" tag | sort --version-sort | grep -v rc | tail -n1)
output_dir="$HOME/spdk_$(tr . _ < <(tr -d '[:alpha:]' <<< $last_release))" output_dir="$GIT_REPOS/spdk_$(tr . _ < <(tr -d '[:alpha:]' <<< $last_release))"
if [[ ! -d $output_dir ]]; then if [[ ! -d $output_dir ]]; then
cp -r spdk_repo/spdk "$output_dir" cp -r "$GIT_REPOS/spdk_repo/spdk" "$output_dir"
fi fi
git -C "$output_dir" checkout "$last_release" git -C "$output_dir" checkout "$last_release"
@ -76,30 +76,27 @@ function install_qat() {
install libudev-devel || install libudev-dev install libudev-devel || install libudev-dev
qat_tarball=$(basename $DRIVER_LOCATION_QAT)
kernel_maj=$(uname -r | cut -d'.' -f1) kernel_maj=$(uname -r | cut -d'.' -f1)
kernel_min=$(uname -r | cut -d'.' -f2) kernel_min=$(uname -r | cut -d'.' -f2)
sudo modprobe -r qat_c62x sudo modprobe -r qat_c62x
if [ -d /QAT ]; then if [[ -d $GIT_REPOS/QAT ]]; then
sudo rm -rf /QAT/ sudo rm -rf "$GIT_REPOS/QAT"
fi fi
sudo mkdir /QAT sudo mkdir "$GIT_REPOS/QAT"
wget $DRIVER_LOCATION_QAT tar -C "$GIT_REPOS/QAT" -xzof - < <(wget -O- "$DRIVER_LOCATION_QAT")
sudo cp $qat_tarball /QAT/
(cd /QAT && sudo tar zxof /QAT/$qat_tarball)
#The driver version 1.7.l.4.3.0-00033 contains a reference to a deprecated function. Remove it so the build won't fail. #The driver version 1.7.l.4.3.0-00033 contains a reference to a deprecated function. Remove it so the build won't fail.
if [ $kernel_maj -le 4 ]; then if [ $kernel_maj -le 4 ]; then
if [ $kernel_min -le 17 ]; then if [ $kernel_min -le 17 ]; then
sudo sed -i 's/rdtscll(timestamp);/timestamp = rdtsc_ordered();/g' \ sudo sed -i 's/rdtscll(timestamp);/timestamp = rdtsc_ordered();/g' \
/QAT/quickassist/utilities/osal/src/linux/kernel_space/OsalServices.c || true "$GIT_REPOS/QAT/quickassist/utilities/osal/src/linux/kernel_space/OsalServices.c" || true
fi fi
fi fi
(cd /QAT && sudo ./configure --enable-icp-sriov=host && sudo make install) (cd "$GIT_REPOS/QAT" && sudo ./configure --enable-icp-sriov=host && sudo make install)
if sudo service qat_service start; then if sudo service qat_service start; then
echo "failed to start the qat service. Something may be wrong with your device or package." echo "failed to start the qat service. Something may be wrong with your device or package."
@ -109,9 +106,9 @@ function install_qat() {
function install_rocksdb() { function install_rocksdb() {
# Rocksdb is installed for use with the blobfs tests. # Rocksdb is installed for use with the blobfs tests.
if [ ! -d /usr/src/rocksdb ]; then if [ ! -d /usr/src/rocksdb ]; then
git clone "${GIT_REPO_ROCKSDB}" git clone "${GIT_REPO_ROCKSDB}" "$GIT_REPOS/rocksdb"
git -C ./rocksdb checkout spdk-v5.6.1 git -C "$GIT_REPOS/rocksdb" checkout spdk-v5.6.1
sudo mv rocksdb /usr/src/ sudo mv "$GIT_REPOS/rocksdb" /usr/src/
else else
sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1 sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1
echo "rocksdb already in /usr/src. Not checking out again" echo "rocksdb already in /usr/src. Not checking out again"
@ -125,10 +122,10 @@ function install_fio() {
if [ ! -d /usr/src/fio ]; then if [ ! -d /usr/src/fio ]; then
if [ ! -d fio ]; then if [ ! -d fio ]; then
git clone "${GIT_REPO_FIO}" git clone "${GIT_REPO_FIO}" "$GIT_REPOS/fio"
sudo mv fio /usr/src/ sudo mv "$GIT_REPOS/fio" /usr/src/
else else
sudo mv fio /usr/src/ sudo mv "$GIT_REPOS/fio" /usr/src/
fi fi
( (
git -C /usr/src/fio checkout master \ git -C /usr/src/fio checkout master \
@ -150,9 +147,9 @@ function install_fio() {
function install_flamegraph() { function install_flamegraph() {
# Flamegraph is used when printing out timing graphs for the tests. # Flamegraph is used when printing out timing graphs for the tests.
if [ ! -d /usr/local/FlameGraph ]; then if [ ! -d /usr/local/FlameGraph ]; then
git clone "${GIT_REPO_FLAMEGRAPH}" git clone "${GIT_REPO_FLAMEGRAPH}" "$GIT_REPOS/FlameGraph"
mkdir -p /usr/local mkdir -p /usr/local
sudo mv FlameGraph /usr/local/FlameGraph sudo mv "$GIT_REPOS/FlameGraph" /usr/local/FlameGraph
else else
echo "flamegraph already installed. Skipping" echo "flamegraph already installed. Skipping"
fi fi
@ -171,9 +168,9 @@ function install_qemu() {
# Forked QEMU # Forked QEMU
SPDK_QEMU_BRANCH=spdk-5.0.0 SPDK_QEMU_BRANCH=spdk-5.0.0
mkdir -p qemu mkdir -p "$GIT_REPOS/qemu"
if [ ! -d "qemu/$SPDK_QEMU_BRANCH" ]; then if [[ ! -d $GIT_REPOS/qemu/$SPDK_QEMU_BRANCH ]]; then
git -C ./qemu clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$SPDK_QEMU_BRANCH" git clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH"
else else
echo "qemu already checked out. Skipping" echo "qemu already checked out. Skipping"
fi fi
@ -193,15 +190,15 @@ function install_qemu() {
fi fi
fi fi
sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules"
sed -i s@git://git.qemu.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config"
sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.gitmodules sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules"
sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g qemu/$SPDK_QEMU_BRANCH/.git/config sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config"
# The qemu configure script places several output files in the CWD. # The qemu configure script places several output files in the CWD.
(cd qemu/$SPDK_QEMU_BRANCH && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa) (cd "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa)
make -C ./qemu/$SPDK_QEMU_BRANCH -j${jobs} make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" -j${jobs}
sudo make -C ./qemu/$SPDK_QEMU_BRANCH install sudo make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" install
} }
function install_vpp() { function install_vpp() {
@ -213,31 +210,31 @@ function install_vpp() {
exit 1 exit 1
fi fi
else else
git clone "${GIT_REPO_VPP}" git clone "${GIT_REPO_VPP}" "$GIT_REPOS/vpp"
git -C ./vpp checkout v19.04.2 git -C "$GIT_REPOS/vpp" checkout v19.04.2
if [ "${OSID}" == 'fedora' ]; then if [ "${OSID}" == 'fedora' ]; then
if [ ${OSVERSION} -eq 29 ]; then if [ ${OSVERSION} -eq 29 ]; then
git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora29-fix.patch git -C "$GIT_REPOS/vpp" apply ${VM_SETUP_PATH}/patch/vpp/fedora29-fix.patch
fi fi
if [ ${OSVERSION} -eq 30 ]; then if [ ${OSVERSION} -eq 30 ]; then
git -C ./vpp apply ${VM_SETUP_PATH}/patch/vpp/fedora30-fix.patch git -C "$GIT_REPOS/vpp" apply ${VM_SETUP_PATH}/patch/vpp/fedora30-fix.patch
fi fi
fi fi
# Installing required dependencies for building VPP # Installing required dependencies for building VPP
yes | make -C ./vpp install-dep yes | make -C "$GIT_REPOS/vpp" install-dep
make -C ./vpp build -j${jobs} make -C "$GIT_REPOS/vpp" build -j${jobs}
sudo mv ./vpp /usr/local/src/vpp-19.04 sudo mv "$GIT_REPOS/vpp" /usr/local/src/vpp-19.04
fi fi
} }
function install_nvmecli() { function install_nvmecli() {
SPDK_NVME_CLI_BRANCH=spdk-1.6 SPDK_NVME_CLI_BRANCH=spdk-1.6
if [ ! -d nvme-cli ]; then if [[ ! -d $GIT_REPOS/nvme-cli ]]; then
git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH" git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH" "$GIT_REPOS/nvme-cli"
else else
echo "nvme-cli already checked out. Skipping" echo "nvme-cli already checked out. Skipping"
fi fi
@ -245,30 +242,30 @@ function install_nvmecli() {
# Changes required for SPDK are already merged on top of # Changes required for SPDK are already merged on top of
# nvme-cli, however not released yet. # nvme-cli, however not released yet.
# Support for SPDK should be released in nvme-cli >1.11.1 # Support for SPDK should be released in nvme-cli >1.11.1
git clone "https://github.com/linux-nvme/nvme-cli.git" "nvme-cli-cuse" git clone "https://github.com/linux-nvme/nvme-cli.git" "$GIT_REPOS/nvme-cli-cuse"
git -C ./nvme-cli-cuse checkout "e770466615096a6d41f038a28819b00bc3078e1d" git -C "$GIT_REPOS/nvme-cli-cuse checkout" "e770466615096a6d41f038a28819b00bc3078e1d"
make -C ./nvme-cli-cuse make -C "$GIT_REPOS/nvme-cli-cuse"
sudo mv ./nvme-cli-cuse /usr/local/src/nvme-cli sudo mv "$GIT_REPOS/nvme-cli-cuse" /usr/local/src/nvme-cli
fi fi
} }
function install_libiscsi() { function install_libiscsi() {
# We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need # We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need
# to later. Cloning from git is just future proofing the machines. # to later. Cloning from git is just future proofing the machines.
if [ ! -d libiscsi ]; then if [[ ! -d $GIT_REPOS/libiscsi ]]; then
git clone "${GIT_REPO_LIBISCSI}" git clone "${GIT_REPO_LIBISCSI}" "$GIT_REPOS/libiscsi"
else else
echo "libiscsi already checked out. Skipping" echo "libiscsi already checked out. Skipping"
fi fi
(cd libiscsi && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi) (cd "$GIT_REPOS/libiscsi" && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi)
make -C ./libiscsi -j${jobs} make -C "$GIT_REPOS/libiscsi" -j${jobs}
sudo make -C ./libiscsi install sudo make -C "$GIT_REPOS/libiscsi" install
} }
function install_git() { function install_git() {
install zlib-devel curl-devel install zlib-devel curl-devel
tar -xzof <(wget -qO- "$GIT_REPO_GIT") tar -C "$GIT_REPOS" -xzof <(wget -qO- "$GIT_REPO_GIT")
(cd git-${GIT_VERSION} \ (cd "$GIT_REPOS/git-$GIT_VERSION" \
&& make configure \ && make configure \
&& ./configure --prefix=/usr/local/git \ && ./configure --prefix=/usr/local/git \
&& sudo make -j${jobs} install) && sudo make -j${jobs} install)
@ -301,6 +298,7 @@ export GIT_REPO_INTEL_IPSEC_MB
export DRIVER_LOCATION_QAT export DRIVER_LOCATION_QAT
: ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz} : ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz}
export GIT_REPO_GIT export GIT_REPO_GIT
GIT_REPOS=${GIT_REPOS:-$HOME}
if [[ $ID == centos ]] && (( VERSION_ID == 7 )); then if [[ $ID == centos ]] && (( VERSION_ID == 7 )); then
# install proper version of the git first # install proper version of the git first
@ -330,6 +328,7 @@ fi
sources+=(install_fio) sources+=(install_fio)
sudo mkdir -p /usr/{,local}/src sudo mkdir -p /usr/{,local}/src
sudo mkdir -p "$GIT_REPOS"
if [[ $INSTALL_REFSPDK == true ]]; then if [[ $INSTALL_REFSPDK == true ]]; then
# Serialize builds as refspdk depends on spdk # Serialize builds as refspdk depends on spdk

6
test/common/config/vm_setup.sh
View File

@ -51,6 +51,7 @@ function usage() {
echo " -i --install-deps Install $PACKAGEMNG based dependencies" echo " -i --install-deps Install $PACKAGEMNG based dependencies"
echo " -t --test-conf List of test configurations to enable (${CONF})" echo " -t --test-conf List of test configurations to enable (${CONF})"
echo " -c --conf-path Path to configuration file" echo " -c --conf-path Path to configuration file"
echo " -d --dir-git Path to where git sources should be saved"
exit 0 exit 0
} }
@ -75,7 +76,7 @@ else
fi fi
# Parse input arguments # # Parse input arguments #
while getopts 'iuht:c:-:' optchar; do while getopts 'd:iuht:c:-:' optchar; do
case "$optchar" in case "$optchar" in
-) -)
case "$OPTARG" in case "$OPTARG" in
@ -84,6 +85,7 @@ while getopts 'iuht:c:-:' optchar; do
install-deps) INSTALL=true ;; install-deps) INSTALL=true ;;
test-conf=*) CONF="${OPTARG#*=}" ;; test-conf=*) CONF="${OPTARG#*=}" ;;
conf-path=*) CONF_PATH="${OPTARG#*=}" ;; conf-path=*) CONF_PATH="${OPTARG#*=}" ;;
dir-git=*) GIT_REPOS="${OPTARG#*=}" ;;
*) *)
echo "Invalid argument '$OPTARG'" echo "Invalid argument '$OPTARG'"
usage usage
@ -95,6 +97,7 @@ while getopts 'iuht:c:-:' optchar; do
i) INSTALL=true ;; i) INSTALL=true ;;
t) CONF="$OPTARG" ;; t) CONF="$OPTARG" ;;
c) CONF_PATH="$OPTARG" ;; c) CONF_PATH="$OPTARG" ;;
d) GIT_REPOS="$OPTARG" ;;
*) *)
echo "Invalid argument '$OPTARG'" echo "Invalid argument '$OPTARG'"
usage usage
@ -111,7 +114,6 @@ if [ -n "$CONF_PATH" ]; then
fi fi
fi fi
cd ~
if $UPGRADE; then if $UPGRADE; then
upgrade upgrade
fi fi