From 9ebbf3acacbe5cd0b55f2177ad330ecbaa56b29b Mon Sep 17 00:00:00 2001 From: Maciej Wawryk Date: Tue, 14 Jan 2020 10:59:31 +0100 Subject: [PATCH] scripts/pkgdep.sh: Add flag for developer tools installation Signed-off-by: Maciej Wawryk Change-Id: I8876fd7c60198c597d931c5f498644c415c2a868 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/596 Tested-by: SPDK CI Jenkins Reviewed-by: Shuhei Matsumoto Reviewed-by: Tomasz Zawadzki Reviewed-by: Aleksey Marchuk --- README.md | 3 + doc/getting_started.md | 11 +- scripts/pkgdep.sh | 156 +++++++++++++++------------ scripts/vagrant/README.md | 2 +- scripts/vagrant/Vagrantfile_vhost_vm | 2 +- scripts/vagrant/update.sh | 8 +- test/common/config/vm_setup.sh | 2 +- 7 files changed, 107 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index 4edf2b7c8..a81560342 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,9 @@ git submodule update --init ## Prerequisites The dependencies can be installed automatically by `scripts/pkgdep.sh`. +The `scripts/pkgdep.sh` script will automatically install the bare minimum +dependencies required to build SPDK. +Use `--help` to see information on installing dependencies for optional components ~~~{.sh} ./scripts/pkgdep.sh diff --git a/doc/getting_started.md b/doc/getting_started.md index 29fbdb551..c4f4d3cf9 100644 --- a/doc/getting_started.md +++ b/doc/getting_started.md @@ -10,13 +10,20 @@ git submodule update --init # Installing Prerequisites {#getting_started_prerequisites} -The `scripts/pkgdep.sh` script will automatically install the full set of -dependencies required to build and develop SPDK. +The `scripts/pkgdep.sh` script will automatically install the bare minimum +dependencies required to build SPDK. +Use `--help` to see information on installing dependencies for optional components. ~~~{.sh} sudo scripts/pkgdep.sh ~~~ +Option --all will install all dependencies needed by SPDK features. + +~~~{.sh} +sudo scripts/pkgdep.sh --all +~~~ + # Building {#getting_started_building} Linux: diff --git a/scripts/pkgdep.sh b/scripts/pkgdep.sh index ebbabbae1..0bf49e8cb 100755 --- a/scripts/pkgdep.sh +++ b/scripts/pkgdep.sh @@ -11,22 +11,34 @@ function usage() echo "" echo "$0" echo " -h --help" + echo " -a --all" + echo " -d --developer-tools Install tools for developers (code styling, code coverage, etc.)" echo "" exit 0 } -INSTALL_CRYPTO=false +function install_all_dependencies () +{ + INSTALL_DEV_TOOLS=true +} -while getopts 'hi-:' optchar; do +INSTALL_CRYPTO=false +INSTALL_DEV_TOOLS=false + +while getopts 'adhi-:' optchar; do case "$optchar" in -) case "$OPTARG" in help) usage;; + all) install_all_dependencies;; + developer-tools) INSTALL_DEV_TOOLS=true;; *) echo "Invalid argument '$OPTARG'" usage;; esac ;; h) usage;; + a) install_all_dependencies;; + d) INSTALL_DEV_TOOLS=true;; *) echo "Invalid argument '$OPTARG'" usage;; esac @@ -39,21 +51,6 @@ rootdir=$(readlink -f $scriptsdir/..) if [ -s /etc/redhat-release ]; then . /etc/os-release - - # Includes Fedora, CentOS 7, RHEL 7 - # Add EPEL repository for CUnit-devel and libunwind-devel - if echo "$ID $VERSION_ID" | grep -E -q 'rhel 7|centos 7'; then - if ! rpm --quiet -q epel-release; then - yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - fi - - if [ $ID = 'rhel' ]; then - subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms" - elif [ $ID = 'centos' ]; then - yum --enablerepo=extras install -y epel-release - fi - fi - # Minimal install yum install -y gcc gcc-c++ make CUnit-devel libaio-devel openssl-devel \ libuuid-devel libiscsi-devel python @@ -65,11 +62,26 @@ if [ -s /etc/redhat-release ]; then yum install -y autoconf automake libtool help2man # Additional dependencies for DPDK yum install -y numactl-devel nasm - # Dependencies for developers - yum install -y git astyle python-pycodestyle lcov \ - sg3_utils pciutils ShellCheck - # Additional (optional) dependencies for showing backtrace in logs - yum install -y libunwind-devel || true + if [[ $INSTALL_DEV_TOOLS == "true" ]]; then + # Tools for developers + # Includes Fedora, CentOS 7, RHEL 7 + # Add EPEL repository for CUnit-devel and libunwind-devel + if echo "$ID $VERSION_ID" | grep -E -q 'rhel 7|centos 7'; then + if ! rpm --quiet -q epel-release; then + yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + fi + + if [[ $ID = 'rhel' ]]; then + subscription-manager repos --enable "rhel-*-optional-rpms" --enable "rhel-*-extras-rpms" + elif [[ $ID = 'centos' ]]; then + yum --enablerepo=extras install -y epel-release + fi + fi + yum install -y git astyle python-pycodestyle lcov \ + sg3_utils pciutils ShellCheck + # Additional (optional) dependencies for showing backtrace in logs + yum install -y libunwind-devel || true + fi # Additional dependencies for NVMe over Fabrics yum install -y libibverbs-devel librdmacm-devel # Additional dependencies for building docs @@ -83,27 +95,30 @@ elif [ -f /etc/debian_version ]; then # Minimal install apt-get install -y gcc g++ make libcunit1-dev libaio-dev libssl-dev \ uuid-dev libiscsi-dev python + # Additional dependencies for SPDK CLI - not available on older Ubuntus + apt-get install -y python3-configshell-fb python3-pexpect || echo \ + "Note: Some SPDK CLI dependencies could not be installed." + # Additional dependencies for DPDK apt-get install -y libnuma-dev nasm # Additional dependencies for ISA-L used in compression apt-get install -y autoconf automake libtool help2man - # Dependencies for developers - apt-get install -y git astyle pep8 lcov clang sg3-utils pciutils shellcheck - # Additional python style checker not available on ubuntu 16.04 or earlier. - apt-get install -y pycodestyle || true - # Additional (optional) dependencies for showing backtrace in logs - apt-get install -y libunwind-dev || true + if [[ $INSTALL_DEV_TOOLS == "true" ]]; then + # Tools for developers + apt-get install -y git astyle pep8 lcov clang sg3-utils pciutils shellcheck + # Additional python style checker not available on ubuntu 16.04 or earlier. + apt-get install -y pycodestyle || true + # Additional (optional) dependencies for showing backtrace in logs + apt-get install -y libunwind-dev || true + # Additional dependecies for nvmf performance test script + apt-get install -y python3-paramiko + fi # Additional dependencies for NVMe over Fabrics apt-get install -y libibverbs-dev librdmacm-dev # Additional dependencies for building docs apt-get install -y doxygen mscgen graphviz - # Additional dependencies for SPDK CLI - not available on older Ubuntus - apt-get install -y python3-configshell-fb python3-pexpect || echo \ - "Note: Some SPDK CLI dependencies could not be installed." # Additional dependencies for FUSE and CUSE apt-get install -y libfuse3-dev - # Additional dependecies for nvmf performance test script - apt-get install -y python3-paramiko elif [ -f /etc/SuSE-release ] || [ -f /etc/SUSE-brand ]; then # Minimal install zypper install -y gcc gcc-c++ make cunit-devel libaio-devel libopenssl-devel \ @@ -112,11 +127,13 @@ elif [ -f /etc/SuSE-release ] || [ -f /etc/SUSE-brand ]; then zypper install -y libnuma-devel nasm # Additional dependencies for ISA-L used in compression zypper install -y autoconf automake libtool help2man - # Dependencies for developers - zypper install -y git-core lcov python-pycodestyle sg3_utils \ - pciutils ShellCheck - # Additional (optional) dependencies for showing backtrace in logs - zypper install libunwind-devel || true + if [[ $INSTALL_DEV_TOOLS == "true" ]]; then + # Tools for developers + zypper install -y git-core lcov python-pycodestyle sg3_utils \ + pciutils ShellCheck + # Additional (optional) dependencies for showing backtrace in logs + zypper install libunwind-devel || true + fi # Additional dependencies for NVMe over Fabrics zypper install -y rdma-core-devel # Additional dependencies for building pmem based backends @@ -130,50 +147,53 @@ elif [ $(uname -s) = "FreeBSD" ] ; then pkg install -y gmake cunit openssl git bash misc/e2fsprogs-libuuid python # Additional dependencies for ISA-L used in compression pkg install -y autoconf automake libtool help2man - # Dependencies for developers - pkg install -y devel/astyle bash py27-pycodestyle \ - misc/e2fsprogs-libuuid sysutils/sg3_utils nasm + if [[ $INSTALL_DEV_TOOLS == "true" ]]; then + # Tools for developers + pkg install -y devel/astyle bash py27-pycodestyle \ + misc/e2fsprogs-libuuid sysutils/sg3_utils nasm + fi # Additional dependencies for building docs pkg install -y doxygen mscgen graphviz elif [ -f /etc/arch-release ]; then # Install main dependencies pacman -Sy --needed --noconfirm gcc make cunit libaio openssl \ libutil-linux libiscsi python + # Additional dependencies for SPDK CLI + pacman -Sy --needed --noconfirm python-pexpect python-pip + pip install configshell_fb # Additional dependencies for DPDK pacman -Sy --needed --noconfirm numactl nasm # Additional dependencies for ISA-L used in compression pacman -Sy --needed --noconfirm autoconf automake libtool help2man - - # Dependencies for developers - pacman -Sy --needed --noconfirm git astyle autopep8 \ - clang sg3_utils pciutils shellcheck - # Additional (optional) dependencies for showing backtrace in logs - pacman -Sy --needed --noconfirm libunwind + if [[ $INSTALL_DEV_TOOLS == "true" ]]; then + # Tools for developers + pacman -Sy --needed --noconfirm git astyle autopep8 \ + clang sg3_utils pciutils shellcheck + # Additional (optional) dependencies for showing backtrace in logs + pacman -Sy --needed --noconfirm libunwind + #fakeroot needed to instal via makepkg + pacman -Sy --needed --noconfirm fakeroot + su - $SUDO_USER -c "pushd /tmp; + git clone https://aur.archlinux.org/perl-perlio-gzip.git; + cd perl-perlio-gzip; + yes y | makepkg -si --needed; + cd ..; rm -rf perl-perlio-gzip + popd" + # sed is to modify sources section in PKGBUILD + # By default it uses git:// which will fail behind proxy, so + # redirect it to http:// source instead + su - $SUDO_USER -c "pushd /tmp; + git clone https://aur.archlinux.org/lcov-git.git; + cd lcov-git; + sed -i 's/git:/git+http:/' PKGBUILD; + makepkg -si --needed --noconfirm; + cd .. && rm -rf lcov-git; + popd" + fi # Additional dependencies for building docs pacman -Sy --needed --noconfirm doxygen graphviz - # Additional dependencies for SPDK CLI - pacman -Sy --needed --noconfirm python-pexpect python-pip - pip install configshell_fb # Additional dependencies for FUSE and CUSE pacman -Sy --needed --noconfirm fuse3 - #fakeroot needed to instal via makepkg - pacman -Sy --needed --noconfirm fakeroot - su - $SUDO_USER -c "pushd /tmp; - git clone https://aur.archlinux.org/perl-perlio-gzip.git; - cd perl-perlio-gzip; - yes y | makepkg -si --needed; - cd ..; rm -rf perl-perlio-gzip - popd" - # sed is to modify sources section in PKGBUILD - # By default it uses git:// which will fail behind proxy, so - # redirect it to http:// source instead - su - $SUDO_USER -c "pushd /tmp; - git clone https://aur.archlinux.org/lcov-git.git; - cd lcov-git; - sed -i 's/git:/git+http:/' PKGBUILD; - makepkg -si --needed --noconfirm; - cd .. && rm -rf lcov-git; - popd" # Additional dependency for building docs pacman -S --noconfirm --needed gd ttf-font su - $SUDO_USER -c "pushd /tmp; diff --git a/scripts/vagrant/README.md b/scripts/vagrant/README.md index 3e74e8027..1a80d78d8 100644 --- a/scripts/vagrant/README.md +++ b/scripts/vagrant/README.md @@ -224,7 +224,7 @@ The following steps are done by the `update.sh` script. It is recommended that y ``` $ sudo pkg upgrade -f - $ sudo spdk_repo/spdk/scripts/pkgdep.sh + $ sudo spdk_repo/spdk/scripts/pkgdep.sh --all $ sudo git clone --depth 10 -b releases/11.1.0 https://github.com/freebsd/freebsd.git /usr/src ``` diff --git a/scripts/vagrant/Vagrantfile_vhost_vm b/scripts/vagrant/Vagrantfile_vhost_vm index 2fd355402..b12e8db34 100644 --- a/scripts/vagrant/Vagrantfile_vhost_vm +++ b/scripts/vagrant/Vagrantfile_vhost_vm @@ -100,7 +100,7 @@ Vagrant.configure(2) do |config| if spdk_dir != "none" config.vm.synced_folder "#{spdk_dir}", "/home/vagrant/spdk_repo/spdk", type: "rsync", rsync__auto: false if install_deps.include? "true" - config.vm.provision "shell", inline: 'sudo /home/vagrant/spdk_repo/spdk/scripts/pkgdep.sh' + config.vm.provision "shell", inline: 'sudo /home/vagrant/spdk_repo/spdk/scripts/pkgdep.sh --all' end end diff --git a/scripts/vagrant/update.sh b/scripts/vagrant/update.sh index e064ced13..596fabe11 100755 --- a/scripts/vagrant/update.sh +++ b/scripts/vagrant/update.sh @@ -33,7 +33,7 @@ SYSTEM=$(uname -s) if [ "$SYSTEM" = "FreeBSD" ]; then # Do initial setup for the system pkg upgrade -f - ${SPDK_DIR}/scripts/pkgdep.sh + ${SPDK_DIR}/scripts/pkgdep.sh --all if [ -d /usr/src/.git ]; then echo echo "/usr/src/ is a git repository" @@ -79,16 +79,16 @@ else # Standard update + upgrade dance apt-get update --assume-yes --no-install-suggests --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" apt-get upgrade --assume-yes --no-install-suggests --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" - ${SPDK_DIR}/scripts/pkgdep.sh + ${SPDK_DIR}/scripts/pkgdep.sh --all elif [ "$DISTRIB_ID" == "CentOS" ]; then # Standard update + upgrade dance yum check-update yum update -y - ${SPDK_DIR}/scripts/pkgdep.sh + ${SPDK_DIR}/scripts/pkgdep.sh --all elif [ "$DISTRIB_ID" == "Fedora" ]; then yum check-update yum update -y - "$SPDK_DIR"/scripts/pkgdep.sh + "$SPDK_DIR"/scripts/pkgdep.sh --all sudo -u vagrant "$SPDK_DIR"/test/common/config/vm_setup.sh -i fi fi diff --git a/test/common/config/vm_setup.sh b/test/common/config/vm_setup.sh index 7b6d2436b..13934deff 100755 --- a/test/common/config/vm_setup.sh +++ b/test/common/config/vm_setup.sh @@ -437,7 +437,7 @@ git -C spdk_repo/spdk config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSE git -C spdk_repo/spdk submodule update --init --recursive if $INSTALL; then - sudo spdk_repo/spdk/scripts/pkgdep.sh + sudo spdk_repo/spdk/scripts/pkgdep.sh --all if [ $PACKAGEMNG == 'dnf' ]; then if echo $CONF | grep -q tsocks; then