Spdk/test/common/config/pkgdep/git

function install_spdk() {
	mkdir -p "$GIT_REPOS/spdk_repo/output" || echo "Can not create spdk_repo/output directory."

	if [[ -d $GIT_REPOS/spdk_repo/spdk ]]; then
		echo "spdk source already present, not cloning"
	else
		git -C "$GIT_REPOS/spdk_repo" clone "${GIT_REPO_SPDK}"
	fi
	git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.dpdk.url "${GIT_REPO_DPDK}"
	git -C "$GIT_REPOS/spdk_repo/spdk" config submodule.intel-ipsec-mb.url "${GIT_REPO_INTEL_IPSEC_MB}"
	git -C "$GIT_REPOS/spdk_repo/spdk" submodule update --init --recursive
}

function install_refspdk() {
	local release
	local output_dir
	local config_params
	local rootdir
	local version

	version=$1

	# Create a reference SPDK build for ABI tests
	git -C "$GIT_REPOS/spdk_repo/spdk" fetch --tags

	if [[ "$version" == "latest" ]]; then
		release=$(git -C "$GIT_REPOS/spdk_repo/spdk" tag | sort --version-sort | grep -v rc | tail -n 1)
		output_dir="$GIT_REPOS/spdk_abi_latest"
	elif [[ "$version" == "LTS" ]]; then
		release="LTS"
		output_dir="$GIT_REPOS/spdk_abi_lts"
	fi

	if [[ ! -d $output_dir ]]; then
		cp -R "$GIT_REPOS/spdk_repo/spdk" "$output_dir"
	fi

	git -C "$output_dir" checkout "$release"
	git -C "$output_dir" submodule update --init
	lts_2001_fallback=false
	if [[ $(git -C "$output_dir" describe --tags) == v20.01* ]]; then
		lts_2001_fallback=true
	fi

	cat > $HOME/autorun-spdk.conf <<- EOF
		SPDK_BUILD_SHARED_OBJECT=1
		SPDK_TEST_AUTOBUILD=1
		SPDK_TEST_UNITTEST=1
		SPDK_TEST_BLOCKDEV=1
		SPDK_TEST_PMDK=1
		SPDK_TEST_ISAL=1
		SPDK_TEST_REDUCE=1
		SPDK_TEST_CRYPTO=1
		SPDK_TEST_FTL=1
		SPDK_TEST_OCF=1
		SPDK_TEST_RAID5=1
		SPDK_TEST_RBD=1
		SPDK_RUN_ASAN=1
		SPDK_RUN_UBSAN=1
		SPDK_TEST_NVME_CUSE=1
		SPDK_TEST_BLOBFS=1
		SPDK_TEST_URING=1
	EOF

	mkdir -p $HOME/output

	(
		rootdir="$output_dir"
		source $HOME/autorun-spdk.conf
		source $output_dir/test/common/autotest_common.sh

		# Prepare separate, fixed, cmdline for the FreeBSD, Issue #1397.
		if [[ $OSID == freebsd ]]; then
			config_params="--enable-debug --enable-werror"
			config_params+=" --without-isal --with-fio=/usr/src/fio"

			# TODO: Remove this if-block after 21.01 LTS is released and 20.01 LTS is deprecated.
			if ! lts_2001_fallback; then
				config_params+=" --with-idxd --disable-unit-tests"
			fi

			MAKE=gmake
		else
			# TODO: "get_config_params" was not available in 20.01 LTS release.
			# Remove this if-block after 21.01 release.
			if lts_2001_fallback; then
				config_params="--enable-debug --enable-werror --with-rdma"
				config_params+=" --with-fio=/usr/src/fio --with-iscsi-initiator"
				config_params+=" --with-nvme-cuse --with-pmdk --with-reduce"
				config_params+=" --with-rbd --with-crypto --with-ocf --enable-ubsan"
				config_params+=" --enable-asan --with-fuse --with-uring"
			else
				config_params="$(get_config_params)"
			fi
		fi
		$output_dir/configure $(echo $config_params | sed 's/--enable-coverage//g')
		if [[ $OSID != freebsd ]]; then
			$MAKE -C $output_dir $MAKEFLAGS include/spdk/config.h
			CONFIG_OCF_PATH="$output_dir/ocf" $MAKE -C $output_dir/lib/env_ocf $MAKEFLAGS exportlib O=$output_dir/build/ocf.a
			$output_dir/configure $config_params --with-ocf=$output_dir/build/ocf.a --with-shared
		fi
		$MAKE -C $output_dir $MAKEFLAGS
	)
}

function install_qat() {
	# Disect the kernel version into maj, min, release and local version
	local kernel_maj kernel_min kernel_rel kernel_loc
	local kernel_ver

	in_syms() {
		local syms
		if [[ -e /proc/kallsyms ]]; then
			syms=/proc/kallsyms
		elif [[ -e /boot/System.map-$(< /proc/sys/kernel/osrelease) ]]; then
			syms=/boot/System.map-$(< /proc/sys/kernel/osrelease)
		else
			return 0
		fi

		grep -q "$1" "$syms"
	}

	IFS=".-" read -r kernel_{maj,min,rel,loc} < /proc/sys/kernel/osrelease
	kernel_ver=$((kernel_maj << 16 | kernel_min << 8 | kernel_rel))

	if [[ -e /sys/module/qat_c62x ]]; then
		sudo modprobe -r qat_c62x || :
	fi
	if [[ -d $GIT_REPOS/QAT ]]; then
		sudo rm -rf "$GIT_REPOS/QAT"
	fi

	mkdir "$GIT_REPOS/QAT"

	tar -C "$GIT_REPOS/QAT" -xzof - < <(wget -O- "$DRIVER_LOCATION_QAT")

	# Patch use of hidden types in kernels >= 5.6.3. See .patch for details
	if ((kernel_ver >= 0x050603)); then
		# Patch only the driver version that was tested
		[[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1
	fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-timespec.patch"

	# Patch name of the pci_aer function which was renamed in kernels >= 5.7.1. See .patch for details
	if ((kernel_ver >= 0x050701)) || ! in_syms pci_cleanup_aer_uncorrect_error_status; then
		# Patch only the driver version that was tested
		[[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1
	fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-pci_aer.patch"

	# Patch use of cryptohash.h which was removed in favor of crypto/sha.h in kernels >= 5.8. See .patch for details
	if ((kernel_ver >= 0x050800)); then
		# Patch only the driver version that was tested
		[[ ${DRIVER_LOCATION_QAT##*/} == qat1.7.l.4.9.0-00008.tar.gz ]] && patch --dir="$GIT_REPOS/QAT" -p1
	fi < "$rootdir/test/common/config/pkgdep/patches/qat/0001-cryptohash.patch"

	(cd "$GIT_REPOS/QAT" && sudo ./configure --enable-icp-sriov=host && sudo make install)

	if ! sudo service qat_service start; then
		echo "failed to start the qat service. Something may be wrong with your device or package."
	fi
}

function install_rocksdb() {
	# Rocksdb is installed for use with the blobfs tests.
	if [ ! -d /usr/src/rocksdb ]; then
		git clone "${GIT_REPO_ROCKSDB}" "$GIT_REPOS/rocksdb"
		git -C "$GIT_REPOS/rocksdb" checkout spdk-v5.6.1
		sudo mv "$GIT_REPOS/rocksdb" /usr/src/
	else
		sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1
		echo "rocksdb already in /usr/src. Not checking out again"
	fi
}

function install_fio() {
	# This version of fio is installed in /usr/src/fio to enable
	# building the spdk fio plugin.
	local fio_version="fio-3.19"

	if [ ! -d /usr/src/fio ]; then
		if [ ! -d fio ]; then
			git clone "${GIT_REPO_FIO}" "$GIT_REPOS/fio"
			sudo mv "$GIT_REPOS/fio" /usr/src/
		else
			sudo mv "$GIT_REPOS/fio" /usr/src/
		fi
		(
			git -C /usr/src/fio checkout master \
				&& git -C /usr/src/fio pull \
				&& git -C /usr/src/fio checkout $fio_version \
				&& if [ $OSID == 'freebsd' ]; then
					gmake -C /usr/src/fio -j${jobs} \
						&& sudo gmake -C /usr/src/fio install
				else
					make -C /usr/src/fio -j${jobs} \
						&& sudo make -C /usr/src/fio install
				fi
		)
	else
		echo "fio already in /usr/src/fio. Not installing"
	fi
}

function install_flamegraph() {
	# Flamegraph is used when printing out timing graphs for the tests.
	if [ ! -d /usr/local/FlameGraph ]; then
		git clone "${GIT_REPO_FLAMEGRAPH}" "$GIT_REPOS/FlameGraph"
		mkdir -p /usr/local
		sudo mv "$GIT_REPOS/FlameGraph" /usr/local/FlameGraph
	else
		echo "flamegraph already installed. Skipping"
	fi
}

function install_qemu() {
	# Two versions of QEMU are used in the tests.
	# Stock QEMU is used for vhost. A special fork
	# is used to test OCSSDs. Install both.

	# Forked QEMU
	SPDK_QEMU_BRANCH=spdk-5.0.0
	mkdir -p "$GIT_REPOS/qemu"
	if [[ ! -d $GIT_REPOS/qemu/$SPDK_QEMU_BRANCH ]]; then
		git clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH"
	else
		echo "qemu already checked out. Skipping"
	fi

	declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH")
	declare -a extra_cflags=()

	opt_params+=("--disable-docs")
	if ((gcc_version >= 9)); then
		# GCC 9 fails to compile Qemu due to some old warnings which were not detected by older versions.
		extra_cflags+=("-Wno-error=stringop-truncation" "-Wno-error=deprecated-declarations")
		extra_cflags+=("-Wno-error=incompatible-pointer-types" "-Wno-error=format-truncation")
		extra_cflags+=("-Wno-error=address-of-packed-member")
		opt_params+=("--disable-glusterfs")
	fi

	# Most tsocks proxies rely on a configuration file in /etc/tsocks.conf.
	# If using tsocks, please make sure to complete this config before trying to build qemu.
	if [[ $INSTALL_TSOCKS == true && $NO_TSOCKS != true ]]; then
		if hash tsocks 2> /dev/null; then
			opt_params+=("--with-git='tsocks git'")
		fi
	fi
	opt_params+=("--extra-cflags=${extra_cflags[*]}")

	sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules"
	sed -i s@git://git.qemu.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config"
	sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.gitmodules"
	sed -i s@git://git.qemu-project.org/@https://github.com/qemu/@g "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH/.git/config"
	# The qemu configure script places several output files in the CWD.
	(cd "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa)

	make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" -j${jobs}
	sudo make -C "$GIT_REPOS/qemu/$SPDK_QEMU_BRANCH" install
}

function install_nvmecli() {
	SPDK_NVME_CLI_BRANCH=spdk-1.6
	if [[ ! -d $GIT_REPOS/nvme-cli ]]; then
		git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH" "$GIT_REPOS/nvme-cli"
	else
		echo "nvme-cli already checked out. Skipping"
	fi
	if [ ! -d "/usr/local/src/nvme-cli" ]; then
		# Changes required for SPDK are already merged on top of
		# nvme-cli, however not released yet.
		# Support for SPDK should be released in nvme-cli >1.11.1
		if [[ ! -d $GIT_REPOS/nvme-cli-cuse ]]; then
			git clone "https://github.com/linux-nvme/nvme-cli.git" "$GIT_REPOS/nvme-cli-cuse"
		fi
		git -C "$GIT_REPOS/nvme-cli-cuse" checkout "e770466615096a6d41f038a28819b00bc3078e1d"
		make -C "$GIT_REPOS/nvme-cli-cuse" CPPFLAGS="-Wno-error=maybe-uninitialized"
		sudo mv "$GIT_REPOS/nvme-cli-cuse" /usr/local/src/nvme-cli
	fi
}

function install_libiscsi() {
	# We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need
	# to later. Cloning from git is just future proofing the machines.
	if [[ ! -d $GIT_REPOS/libiscsi ]]; then
		git clone "${GIT_REPO_LIBISCSI}" "$GIT_REPOS/libiscsi"
	else
		echo "libiscsi already checked out. Skipping"
	fi
	(cd "$GIT_REPOS/libiscsi" && ./autogen.sh && ./configure --prefix=/usr/local/libiscsi)
	make -C "$GIT_REPOS/libiscsi" -j${jobs} WARN_CFLAGS=
	sudo make -C "$GIT_REPOS/libiscsi" install
}

function install_git() {
	if type -P git; then
		if ge "$(git --version | awk '{print $3}')" "$GIT_VERSION"; then
			return 0
		fi
	fi >/dev/null

	install zlib-devel curl-devel
	tar -C "$GIT_REPOS" -xzof <(wget -qO- "$GIT_REPO_GIT")
	(cd "$GIT_REPOS/git-$GIT_VERSION" \
		&& make configure \
		&& ./configure --prefix=/usr/local/git \
		&& sudo make -j${jobs} install)
	sudo sh -c "echo 'export PATH=/usr/local/git/bin:$PATH' >> /etc/bashrc"
	export "PATH=/usr/local/git/bin:$PATH"
	# Be nice for vagrant-proxyconf setup
	mkdir -p "/usr/local/git/etc"
}

function install_extra_pkgs() {
	if [[ $INSTALL_QAT == true ]]; then
		install libudev-devel || install libudev-dev || :
	fi

	if [[ $INSTALL_QEMU == true ]]; then
		install qemu-system-x86 qemu-img \
			|| install qemu-system-x86 qemu-utils \
			|| install qemu \
			|| :
	fi
}

function install_vagrant() {
	local vagrant_version="2.2.7"
	local vagrant_installer="vagrant_${vagrant_version}_x86_64.deb"
	local vagrant_plugins=(vagrant-libvirt vagrant-sshfs vagrant-cachier vagrant-proxyconf)

	if [[ $OSID != ubuntu ]]; then
		echo "Currently, Vagrant installation is supported only on ubuntu"
		return 0
	fi

	# Install vagrant and it's plugins dependencies
	# function should be defined in pkgdep/$package_manager file
	install_vagrant_dependencies

	# Download and install vagrant
	if hash vagrant &> /dev/null; then
		echo "Vagrant is already installed"
	else
		wget "https://releases.hashicorp.com/vagrant/${vagrant_version}/${vagrant_installer}"
		sudo dpkg -i "${vagrant_installer}"
	fi
	vagrant --version

	# Install vagrant plugins
	local vagrant_plugin_list
	vagrant_plugin_list=$(vagrant plugin list)

	local plugin
	for plugin in "${vagrant_plugins[@]}"; do
		if grep -Fq "$plugin" <<< "$vagrant_plugin_list"; then
			echo "$plugin already installed"
		else
			vagrant plugin install "$plugin"
		fi
	done
}

GIT_VERSION=2.25.1
: ${GIT_REPO_SPDK=https://github.com/spdk/spdk.git}
export GIT_REPO_SPDK
: ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git}
export GIT_REPO_DPDK
: ${GIT_REPO_ROCKSDB=https://review.spdk.io/spdk/rocksdb}
export GIT_REPO_ROCKSDB
: ${GIT_REPO_FIO=https://github.com/axboe/fio.git}
export GIT_REPO_FIO
: ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git}
export GIT_REPO_FLAMEGRAPH
: ${GIT_REPO_QEMU=https://github.com/spdk/qemu}
export GIT_REPO_QEMU
: ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi}
export GIT_REPO_LIBISCSI
: ${GIT_REPO_SPDK_NVME_CLI=https://github.com/spdk/nvme-cli}
export GIT_REPO_SPDK_NVME_CLI
: ${GIT_REPO_INTEL_IPSEC_MB=https://github.com/spdk/intel-ipsec-mb.git}
export GIT_REPO_INTEL_IPSEC_MB
: ${DRIVER_LOCATION_QAT=https://01.org/sites/default/files/downloads//qat1.7.l.4.9.0-00008.tar.gz}
export DRIVER_LOCATION_QAT
: ${GIT_REPO_GIT=https://github.com/git/git/archive/v${GIT_VERSION}.tar.gz}
export GIT_REPO_GIT
GIT_REPOS=${GIT_REPOS:-$HOME}

gcc_version=$(gcc -dumpversion) gcc_version=${gcc_version%%.*}
if [[ $ID == centos ]] && (( VERSION_ID == 7 )); then
	# install proper version of the git first
	install_git
fi

IFS="," read -ra conf_env <<< "$CONF"
for conf in "${conf_env[@]}"; do
	export "INSTALL_${conf^^}=true"
done

if [[ $OSID == freebsd ]]; then
	jobs=$(($(sysctl -n hw.ncpu) * 2))
else
	jobs=$(($(nproc) * 2))
	sources+=(
		install_libiscsi
		install_nvmecli
		install_qat
		install_rocksdb
		install_flamegraph
		install_qemu
	)
	install_extra_pkgs
fi
sources+=(install_fio)
sources+=(install_vagrant)
sources+=(install_spdk)

sudo mkdir -p /usr/{,local}/src
sudo mkdir -p "$GIT_REPOS"

for source in "${sources[@]}"; do
	source_conf=${source^^}
	if [[ ${!source_conf} == true ]]; then
		"$source"
	fi
done

if [[ $INSTALL_REFSPDK == true ]]; then
	# Serialize builds as refspdk depends on spdk
	[[ $INSTALL_SPDK != true ]] && install_spdk
	install_refspdk latest
	install_refspdk LTS
fi