#!/usr/bin/env bash
#  SPDX-License-Identifier: BSD-3-Clause
#  Copyright (C) 2019 Intel Corporation
#  All rights reserved.
#
testdir=$(readlink -f $(dirname $0))
rootdir=$(readlink -f $testdir/../..)
rpc_server=/var/tmp/spdk-raid.sock
tmp_file=$SPDK_TEST_STORAGE/raidrandtest

source $rootdir/test/common/autotest_common.sh
source $testdir/nbd_common.sh

rpc_py="$rootdir/scripts/rpc.py -s $rpc_server"

function raid_unmap_data_verify() {
	if hash blkdiscard; then
		local nbd=$1
		local rpc_server=$2
		local blksize
		blksize=$(lsblk -o LOG-SEC $nbd | grep -v LOG-SEC | cut -d ' ' -f 5)
		local rw_blk_num=4096
		local rw_len=$((blksize * rw_blk_num))
		local unmap_blk_offs=(0 1028 321)
		local unmap_blk_nums=(128 2035 456)
		local unmap_off
		local unmap_len

		# data write
		dd if=/dev/urandom of=$tmp_file bs=$blksize count=$rw_blk_num
		dd if=$tmp_file of=$nbd bs=$blksize count=$rw_blk_num oflag=direct
		blockdev --flushbufs $nbd

		# confirm random data is written correctly in raid0 device
		cmp -b -n $rw_len $tmp_file $nbd

		for ((i = 0; i < ${#unmap_blk_offs[@]}; i++)); do
			unmap_off=$((blksize * ${unmap_blk_offs[$i]}))
			unmap_len=$((blksize * ${unmap_blk_nums[$i]}))

			# data unmap on tmp_file
			dd if=/dev/zero of=$tmp_file bs=$blksize seek=${unmap_blk_offs[$i]} count=${unmap_blk_nums[$i]} conv=notrunc

			# data unmap on raid bdev
			blkdiscard -o $unmap_off -l $unmap_len $nbd
			blockdev --flushbufs $nbd

			# data verify after unmap
			cmp -b -n $rw_len $tmp_file $nbd
		done
	fi

	return 0
}

function on_error_exit() {
	if [ -n "$raid_pid" ]; then
		killprocess $raid_pid
	fi

	rm -f $tmp_file
	print_backtrace
	exit 1
}

function configure_raid_bdev() {
	local raid_level=$1
	rm -rf $testdir/rpcs.txt

	cat <<- EOL >> $testdir/rpcs.txt
		bdev_malloc_create 32 512 -b Base_1
		bdev_malloc_create 32 512 -b Base_2
		bdev_raid_create -z 64 -r $raid_level -b "Base_1 Base_2" -n raid
	EOL
	$rpc_py < $testdir/rpcs.txt

	rm -rf $testdir/rpcs.txt
}

function raid_function_test() {
	local raid_level=$1
	if [ $(uname -s) = Linux ] && modprobe -n nbd; then
		local nbd=/dev/nbd0
		local raid_bdev

		modprobe nbd
		$rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid &
		raid_pid=$!
		echo "Process raid pid: $raid_pid"
		waitforlisten $raid_pid $rpc_server

		configure_raid_bdev $raid_level
		raid_bdev=$($rpc_py bdev_raid_get_bdevs online | jq -r '.[0]["name"] | select(.)')
		if [ $raid_bdev = "" ]; then
			echo "No raid0 device in SPDK app"
			return 1
		fi

		nbd_start_disks $rpc_server $raid_bdev $nbd
		count=$(nbd_get_count $rpc_server)
		if [ $count -ne 1 ]; then
			return 1
		fi

		raid_unmap_data_verify $nbd $rpc_server

		nbd_stop_disks $rpc_server $nbd
		count=$(nbd_get_count $rpc_server)
		if [ $count -ne 0 ]; then
			return 1
		fi

		killprocess $raid_pid
	else
		echo "skipping bdev raid tests."
	fi

	return 0
}

function verify_raid_bdev_state() (
	set +x
	local raid_bdev_name=$1
	local expected_state=$2
	local raid_level=$3
	local strip_size=$4
	local num_base_bdevs_operational=$5
	local raid_bdev
	local raid_bdev_info
	local num_base_bdevs
	local num_base_bdevs_discovered
	local tmp

	raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0] | select(.)')
	if [ -z "$raid_bdev" ]; then
		echo "No raid device in SPDK app"
		return 1
	fi

	raid_bdev_info=$($rpc_py bdev_raid_get_bdevs $expected_state | jq -r ".[] | select(.name == \"$raid_bdev_name\")")
	if [ -z "$raid_bdev_info" ]; then
		echo "$raid_bdev_name is not in $expected_state state"
		return 1
	fi

	tmp=$(echo $raid_bdev_info | jq -r '.state')
	if [ "$tmp" != $expected_state ]; then
		echo "incorrect state: $tmp, expected: $expected_state"
		return 1
	fi

	tmp=$(echo $raid_bdev_info | jq -r '.raid_level')
	if [ "$tmp" != $raid_level ]; then
		echo "incorrect level: $tmp, expected: $raid_level"
		return 1
	fi

	tmp=$(echo $raid_bdev_info | jq -r '.strip_size_kb')
	if [ "$tmp" != $strip_size ]; then
		echo "incorrect strip size: $tmp, expected: $strip_size"
		return 1
	fi

	num_base_bdevs=$(echo $raid_bdev_info | jq -r '[.base_bdevs_list[]] | length')
	tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs')
	if [ "$num_base_bdevs" != "$tmp" ]; then
		echo "incorrect num_base_bdevs: $tmp, expected: $num_base_bdevs"
		return 1
	fi

	num_base_bdevs_discovered=$(echo $raid_bdev_info | jq -r '[.base_bdevs_list[] | select(.is_configured)] | length')
	tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs_discovered')
	if [ "$num_base_bdevs_discovered" != "$tmp" ]; then
		echo "incorrect num_base_bdevs_discovered: $tmp, expected: $num_base_bdevs_discovered"
		return 1
	fi

	tmp=$(echo $raid_bdev_info | jq -r '.num_base_bdevs_operational')
	if [ "$num_base_bdevs_operational" != "$tmp" ]; then
		echo "incorrect num_base_bdevs_operational $tmp, expected: $num_base_bdevs_operational"
		return 1
	fi
)

function has_redundancy() {
	case $1 in
		"raid1" | "raid5f") return 0 ;;
		*) return 1 ;;
	esac
}

function raid_state_function_test() {
	local raid_level=$1
	local num_base_bdevs=$2
	local raid_bdev
	local base_bdevs=($(for ((i = 1; i <= num_base_bdevs; i++)); do echo BaseBdev$i; done))
	local raid_bdev_name="Existed_Raid"
	local strip_size
	local strip_size_create_arg

	if [ $raid_level != "raid1" ]; then
		strip_size=64
		strip_size_create_arg="-z $strip_size"
	else
		strip_size=0
	fi

	$rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid &
	raid_pid=$!
	echo "Process raid pid: $raid_pid"
	waitforlisten $raid_pid $rpc_server

	# Step1: create a RAID bdev with no base bdevs
	# Expect state: CONFIGURING
	$rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name
	if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs; then
		return 1
	fi
	$rpc_py bdev_raid_delete $raid_bdev_name

	# Step2: create one base bdev and add to the RAID bdev
	# Expect state: CONFIGURING
	$rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name
	$rpc_py bdev_malloc_create 32 512 -b ${base_bdevs[0]}
	waitforbdev ${base_bdevs[0]}
	if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs; then
		return 1
	fi
	$rpc_py bdev_raid_delete $raid_bdev_name

	# Step3: create remaining base bdevs and add to the RAID bdev
	# Expect state: ONLINE
	$rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs[*]}" -n $raid_bdev_name
	for ((i = 1; i < num_base_bdevs; i++)); do
		if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs; then
			return 1
		fi
		$rpc_py bdev_malloc_create 32 512 -b ${base_bdevs[$i]}
		waitforbdev ${base_bdevs[$i]}
	done
	if ! verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs; then
		return 1
	fi

	# Step4: delete one base bdev from the RAID bdev
	$rpc_py bdev_malloc_delete ${base_bdevs[0]}
	local expected_state
	if ! has_redundancy $raid_level; then
		expected_state="offline"
	else
		expected_state="online"
	fi
	if ! verify_raid_bdev_state $raid_bdev_name $expected_state $raid_level $strip_size $((num_base_bdevs - 1)); then
		return 1
	fi

	# Step5: delete remaining base bdevs from the RAID bdev
	# Expect state: removed from system
	for ((i = 1; i < num_base_bdevs; i++)); do
		raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0]["name"]')
		if [ "$raid_bdev" != $raid_bdev_name ]; then
			echo "$raid_bdev_name removed before all base bdevs were deleted"
			return 1
		fi
		$rpc_py bdev_malloc_delete ${base_bdevs[$i]}
	done
	raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[0]["name"] | select(.)')
	if [ -n "$raid_bdev" ]; then
		echo "$raid_bdev_name is not removed"
		return 1
	fi

	killprocess $raid_pid

	return 0
}

function raid0_resize_test() {
	local blksize=512
	local bdev_size_mb=32
	local new_bdev_size_mb=$((bdev_size_mb * 2))
	local blkcnt
	local raid_size_mb
	local new_raid_size_mb

	$rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid &
	raid_pid=$!
	echo "Process raid pid: $raid_pid"
	waitforlisten $raid_pid $rpc_server

	$rpc_py bdev_null_create Base_1 $bdev_size_mb $blksize
	$rpc_py bdev_null_create Base_2 $bdev_size_mb $blksize

	$rpc_py bdev_raid_create -z 64 -r 0 -b "Base_1 Base_2" -n Raid

	# Resize Base_1 first.
	$rpc_py bdev_null_resize Base_1 $new_bdev_size_mb

	# The size of Raid should not be changed.
	blkcnt=$($rpc_py bdev_get_bdevs -b Raid | jq '.[].num_blocks')
	raid_size_mb=$((blkcnt * blksize / 1048576))
	if [ $raid_size_mb != $((bdev_size_mb * 2)) ]; then
		echo "resize failed"
		return 1
	fi

	# Resize Base_2 next.
	$rpc_py bdev_null_resize Base_2 $new_bdev_size_mb

	# The size of Raid should be updated to the expected value.
	blkcnt=$($rpc_py bdev_get_bdevs -b Raid | jq '.[].num_blocks')
	raid_size_mb=$((blkcnt * blksize / 1048576))
	if [ $raid_size_mb != $((new_bdev_size_mb * 2)) ]; then
		echo "resize failed"
		return 1
	fi

	killprocess $raid_pid

	return 0
}

function raid_superblock_test() {
	local raid_level=$1
	local num_base_bdevs=$2
	local base_bdevs_malloc=()
	local base_bdevs_pt=()
	local base_bdevs_pt_uuid=()
	local raid_bdev_name="raid_bdev1"
	local raid_bdev_uuid
	local raid_bdev
	local strip_size
	local strip_size_create_arg

	if [ $raid_level != "raid1" ]; then
		strip_size=64
		strip_size_create_arg="-z $strip_size"
	else
		strip_size=0
	fi

	$rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -L bdev_raid &
	raid_pid=$!
	echo "Process raid pid: $raid_pid"
	waitforlisten $raid_pid $rpc_server

	# Create base bdevs
	for ((i = 1; i <= num_base_bdevs; i++)); do
		local bdev_malloc="malloc$i"
		local bdev_pt="pt$i"
		local bdev_pt_uuid="00000000-0000-0000-0000-00000000000$i"

		base_bdevs_malloc+=($bdev_malloc)
		base_bdevs_pt+=($bdev_pt)
		base_bdevs_pt_uuid+=($bdev_pt_uuid)

		$rpc_py bdev_malloc_create 32 512 -b $bdev_malloc
		$rpc_py bdev_passthru_create -b $bdev_malloc -p $bdev_pt -u $bdev_pt_uuid
	done

	# Create RAID bdev with superblock
	$rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs_pt[*]}" -n $raid_bdev_name -s
	if ! verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs; then
		return 1
	fi

	# Get RAID bdev's UUID
	raid_bdev_uuid=$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid | select(.)')
	if [ -z "$raid_bdev_uuid" ]; then
		return 1
	fi

	# Stop the RAID bdev
	$rpc_py bdev_raid_delete $raid_bdev_name
	raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]')
	if [ -n "$raid_bdev" ]; then
		return 1
	fi

	# Delete the passthru bdevs
	for i in "${base_bdevs_pt[@]}"; do
		$rpc_py bdev_passthru_delete $i
	done
	if [ "$($rpc_py bdev_get_bdevs | jq -r '[.[] | select(.product_name == "passthru")] | any')" == "true" ]; then
		return 1
	fi

	# Try to create new RAID bdev from malloc bdevs
	# Should not reach online state due to superblock still present on base bdevs
	$rpc_py bdev_raid_create $strip_size_create_arg -r $raid_level -b "${base_bdevs_malloc[*]}" -n $raid_bdev_name
	if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs; then
		return 1
	fi

	# Stop the RAID bdev
	$rpc_py bdev_raid_delete $raid_bdev_name
	raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]')
	if [ -n "$raid_bdev" ]; then
		return 1
	fi

	# Re-add first base bdev
	$rpc_py bdev_passthru_create -b ${base_bdevs_malloc[0]} -p ${base_bdevs_pt[0]} -u ${base_bdevs_pt_uuid[0]}

	# Check if the RAID bdev was assembled from superblock
	if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs; then
		return 1
	fi

	# Re-add remaining base bdevs
	for ((i = 1; i < num_base_bdevs; i++)); do
		$rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]}
	done

	# Check if the RAID bdev is in online state
	if ! verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $num_base_bdevs; then
		return 1
	fi

	# Check if the RAID bdev has the same UUID as when first created
	if [ "$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid')" != "$raid_bdev_uuid" ]; then
		return 1
	fi

	if has_redundancy $raid_level; then
		# Delete one base bdev
		$rpc_py bdev_passthru_delete ${base_bdevs_pt[0]}

		# Check if the RAID bdev is in online state (degraded)
		if ! verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)); then
			return 1
		fi

		# Stop the RAID bdev
		$rpc_py bdev_raid_delete $raid_bdev_name
		raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]')
		if [ -n "$raid_bdev" ]; then
			return 1
		fi

		# Delete remaining base bdevs
		for ((i = 1; i < num_base_bdevs; i++)); do
			$rpc_py bdev_passthru_delete ${base_bdevs_pt[$i]}
		done

		# Re-add base bdevs from the second up to (not including) the last one
		for ((i = 1; i < num_base_bdevs - 1; i++)); do
			$rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]}

			# Check if the RAID bdev is in configuring state
			if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $((num_base_bdevs - 1)); then
				return 1
			fi
		done

		# Re-add the last base bdev
		i=$((num_base_bdevs - 1))
		$rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]}

		# Check if the RAID bdev is in online state (degraded)
		if ! verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)); then
			return 1
		fi

		if [ $num_base_bdevs -gt 2 ]; then
			# Stop the RAID bdev
			$rpc_py bdev_raid_delete $raid_bdev_name
			raid_bdev=$($rpc_py bdev_raid_get_bdevs all | jq -r '.[]')
			if [ -n "$raid_bdev" ]; then
				return 1
			fi

			# Delete remaining base bdevs
			for ((i = 1; i < num_base_bdevs; i++)); do
				$rpc_py bdev_passthru_delete ${base_bdevs_pt[$i]}
			done

			# Re-add first base bdev
			# This is the "failed" device and contains the "old" version of the superblock
			$rpc_py bdev_passthru_create -b ${base_bdevs_malloc[0]} -p ${base_bdevs_pt[0]} -u ${base_bdevs_pt_uuid[0]}

			# Check if the RAID bdev is in configuring state
			if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $num_base_bdevs; then
				return 1
			fi

			# Re-add the last base bdev
			i=$((num_base_bdevs - 1))
			$rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]}

			# Check if the RAID bdev is in configuring state
			# This should use the newer superblock version and have n-1 online base bdevs
			if ! verify_raid_bdev_state $raid_bdev_name "configuring" $raid_level $strip_size $((num_base_bdevs - 1)); then
				return 1
			fi

			# Re-add remaining base bdevs
			for ((i = 1; i < num_base_bdevs - 1; i++)); do
				$rpc_py bdev_passthru_create -b ${base_bdevs_malloc[$i]} -p ${base_bdevs_pt[$i]} -u ${base_bdevs_pt_uuid[$i]}
			done

			# Check if the RAID bdev is in online state (degraded)
			if ! verify_raid_bdev_state $raid_bdev_name "online" $raid_level $strip_size $((num_base_bdevs - 1)); then
				return 1
			fi
		fi

		# Check if the RAID bdev has the same UUID as when first created
		if [ "$($rpc_py bdev_get_bdevs -b $raid_bdev_name | jq -r '.[] | .uuid')" != "$raid_bdev_uuid" ]; then
			return 1
		fi
	fi

	killprocess $raid_pid

	return 0
}

trap 'on_error_exit;' ERR

raid_function_test raid0
raid_function_test concat
raid0_resize_test

for n in {2..4}; do
	for level in raid0 concat raid1; do
		raid_state_function_test $level $n
		raid_superblock_test $level $n
	done
done

if [ "$CONFIG_RAID5F" == y ]; then
	for n in {3..4}; do
		raid_state_function_test raid5f $n
		raid_superblock_test raid5f $n
	done
fi

rm -f $tmp_file