test: improvements to vm_setup.sh and pkgdep.sh

Change-Id: Ie433edd7c72f84b62c17bacebd31d61ffa6f30ab
Signed-off-by: Ed Rodriguez <ed.rodriguez@netapp.com>
Signed-off-by: John Meneghini <johnm@netapp.com>
Reviewed-on: https://review.gerrithub.io/416052
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Seth Howell <seth.howell5141@gmail.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>

scripts/pkgdep.sh

@@ -9,9 +9,10 @@ if [ -s /etc/redhat-release ]; then
 		# Add EPEL repository for CUnit-devel
 		yum --enablerepo=extras install -y epel-release
 	fi
-	yum install -y gcc gcc-c++ make CUnit-devel libaio-devel openssl-devel \
+	yum install -y gcc gcc-c++ make CUnit-devel libaio-devel \
 		git astyle python-pep8 lcov python clang-analyzer libuuid-devel \
 		sg3_utils libiscsi-devel
+	yum install -y --allowerasing openssl-devel
 	# Additional dependencies for NVMe over Fabrics
 	yum install -y libibverbs-devel librdmacm-devel
 	# Additional dependencies for DPDK

test/common/config/vm_setup.sh

@@ -17,12 +17,12 @@
 #     intel_iommu=on kvm-intel.nested=1
 
 # We have made a lot of progress with removing hardcoded paths from the tests,
-# but it may be easiest if you create your user with the name sys_sgsw for now.
 
 set -e
 
 UPGRADE=false
 INSTALL=false
+CONF="librxe,iscsi,rocksdb,fio,flamegraph,tsocks,qemu,vpp,libiscsi"
 
 function usage()
 {
@@ -30,20 +30,21 @@ function usage()
     echo "Please run this script as your regular user. The script will make calls to sudo as needed."
     echo ""
     echo "./vm_setup.sh"
-    echo "\t-h --help"
+    echo "  -h --help"
-    echo "\t-u --upgrade Run dnf upgrade"
+    echo "  -u --upgrade Run dnf upgrade"
-    echo "\t-i --install-deps Install dnf based dependencies"
+    echo "  -i --install-deps Install dnf based dependencies"
-
+    echo "  -t --test-conf List of test configurations to enable (${CONF})"
     exit 0
 }
 
-while getopts 'iuh-:' optchar; do
+while getopts 'iuht:-:' optchar; do
     case "$optchar" in
         -)
         case "$OPTARG" in
             help) usage;;
             upgrade) UPGRADE=true;;
             install-deps) INSTALL=true;;
+            test-conf=*) CONF="${OPTARG#*=}";;
             *) echo "Invalid argument '$OPTARG'"
             usage;;
         esac
@@ -51,6 +52,7 @@ while getopts 'iuh-:' optchar; do
     h) usage;;
     u) UPGRADE=true;;
     i) INSTALL=true;;
+    t) CONF="$OPTARG";;
     *) echo "Invalid argument '$OPTARG'"
     usage;;
     esac
@@ -82,9 +84,12 @@ git submodule update --init --recursive
 if $INSTALL; then
     sudo ./scripts/pkgdep.sh
 
-    sudo dnf install -y jq
+    if echo $CONF | grep -q tsocks; then
         sudo dnf install -y tsocks
+    fi
+
     sudo dnf install -y valgrind
+    sudo dnf install -y jq
     sudo dnf install -y nvme-cli
     sudo dnf install -y ceph
     sudo dnf install -y gdb
@@ -98,6 +103,7 @@ if $INSTALL; then
     sudo dnf install -y automake
     sudo dnf install -y libtool
     sudo dnf install -y libmount-devel
+    sudo dnf install -y iscsi-initiator-utils
     sudo dnf install -y isns-utils-devel
     sudo dnf install -y pmempool
     sudo dnf install -y perl-open
@@ -115,23 +121,29 @@ fi
 
 cd ~
 
+if echo $CONF | grep -q librxe; then
     # rxe_cfg is used in the NVMe-oF tests
     # The librxe-dev repository provides a command line tool called rxe_cfg which makes it
     # very easy to use Soft-RoCE. The build pool utilizes this command line tool in the absence
     # of any real RDMA NICs to simulate one for the NVMe-oF tests.
-if [ -d librxe-dev ]; then
+    if hash rxe_cfg 2> /dev/null; then
-    echo "librxe-dev source already present, not cloning"
-elif [ hash rxe_cfg ]; then
         echo "rxe_cfg is already installed. skipping"
+    else
+        if [ -d librxe-dev ]; then
+            echo "librxe-dev source already present, not cloning"
         else
             git clone https://github.com/SoftRoCE/librxe-dev.git
+        fi
+
         cd librxe-dev
         ./configure --libdir=/usr/lib64/ --prefix=
         make -j${jobs}
         sudo make install
         cd ~
     fi
+fi
 
+if echo $CONF | grep -q iscsi; then
     # iscsiadm is used in the iscsi_tgt tests
     # The version of iscsiadm that ships with fedora 26 was broken as of November 3 2017.
     # There is already a bug report out about it, and hopefully it is fixed soon, but in the event that
@@ -166,10 +178,12 @@ if [ "$CURRENT_VERSION" == "$OPEN_ISCSI_VER" ]; then
             echo "custom open-iscsi install located, not reinstalling"
         fi
     fi
-
+fi
 
 sudo mkdir -p /usr/src
 
+if echo $CONF | grep -q rocksdb; then
+
     # Rocksdb is installed for use with the blobfs tests.
     if [ ! -d /usr/src/rocksdb ]; then
         git clone https://review.gerrithub.io/spdk/rocksdb
@@ -179,7 +193,9 @@ else
         sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1
         echo "rocksdb already in /usr/src. Not checking out again"
     fi
+fi
 
+if echo $CONF | grep -q fio; then
     # This version of fio is installed in /usr/src/fio to enable
     # building the spdk fio plugin.
     if [ ! -d /usr/src/fio ]; then
@@ -200,8 +216,11 @@ if [ ! -d /usr/src/fio ]; then
     else
         echo "fio already in /usr/src/fio. Not installing"
     fi
+fi
+
 cd ~
 
+if echo $CONF | grep -q flamegraph; then
     # Flamegraph is used when printing out timing graphs for the tests.
     if [ ! -d /usr/local/FlameGraph ]; then
         git clone https://github.com/brendangregg/FlameGraph.git
@@ -210,7 +229,9 @@ if [ ! -d /usr/local/FlameGraph ]; then
     else
         echo "flamegraph already installed. Skipping"
     fi
+fi
 
+if echo $CONF | grep -q qemu; then
     # Qemu is used in the vhost tests.
     SPDK_QEMU_BRANCH=spdk-2.12-pre
     mkdir -p qemu
@@ -220,19 +241,37 @@ if [ ! -d "$SPDK_QEMU_BRANCH" ]; then
     else
         echo "qemu already checked out. Skipping"
     fi
+
     cd "$SPDK_QEMU_BRANCH"
 
+    declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH")
+
     # Most tsocks proxies rely on a configuration file in /etc/tsocks.conf.
     # If using tsocks, please make sure to complete this config before trying to build qemu.
+    if echo $CONF | grep -q tsocks; then
         if hash tsocks 2> /dev/null; then
-    git_param="--with-git='tsocks git'"
+            opt_params+=(--with-git='tsocks git')
         fi
-./configure "$git_param" --prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa
+    fi
+
+    ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa
+
     make -j${jobs}
     sudo make install
+fi
+
 cd ~
 
+if echo $CONF | grep -q vpp; then
     # Vector packet processing (VPP) is installed for use with iSCSI tests.
+    if [ -d vpp ]; then
+        echo "vpp already cloned."
+        if [ ! -d vpp/build-root ]; then
+            echo "build-root has not been done"
+            echo "remove the `pwd` and start again"
+            exit 1
+        fi
+    else
         git clone https://gerrit.fd.io/r/vpp
         cd vpp
         git checkout v18.01.1
@@ -257,7 +296,10 @@ sudo dnf install -y \
         # very low count and cause issues with hugepage total sizes above 1GB.
         sudo rm -f /etc/sysctl.d/80-vpp.conf
         cd ~
+    fi
+fi
 
+if echo $CONF | grep -q libiscsi; then
     # We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need
     # to later. Cloning from git is just future proofing the machines.
     if [ ! -d libiscsi ]; then
@@ -270,7 +312,7 @@ cd libiscsi
     ./configure --prefix=/usr/local/libiscsi
     make -j${jobs}
     sudo make install
-
+fi
 
 # create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file.
 # By setting any one of the values below to 0, you can skip that specific test. If you are