test/vhost: add live migration test case 2

Change-Id: I114c11d9f7c1e9a5a8c0a989541a41b8747cc125
Signed-off-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-on: https://review.gerrithub.io/398753
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>

test/vhost/common/common.sh

@@ -112,6 +112,7 @@ function spdk_vhost_run()
 	local param
 	local vhost_num=0
 	local vhost_conf_path=""
+	local memory=1024
 
 	for param in "$@"; do
 		case $param in
@@ -120,6 +121,7 @@ function spdk_vhost_run()
 				assert_number "$vhost_num"
 				;;
 			--conf-path=*) local vhost_conf_path="${param#*=}" ;;
+			--memory=*) local memory=${param#*=} ;;
 			*)
 				error "Invalid parameter '$param'"
 				return 1
@@ -127,7 +129,6 @@ function spdk_vhost_run()
 		esac
 	done
 
-
 	local vhost_dir="$(get_vhost_dir $vhost_num)"
 	if [[ -z "$vhost_conf_path" ]]; then
 		error "Missing mandatory parameter '--conf-path'"
@@ -163,13 +164,14 @@ function spdk_vhost_run()
 	cp $vhost_conf_template $vhost_conf_file
 	$SPDK_BUILD_DIR/scripts/gen_nvme.sh >> $vhost_conf_file
 
-	local cmd="$vhost_app -m $reactor_mask -p $master_core -c $vhost_conf_file -r $vhost_dir/rpc.sock"
+	local cmd="$vhost_app -m $reactor_mask -p $master_core -c $vhost_conf_file -s $memory -r $vhost_dir/rpc.sock"
 
 	notice "Loging to:   $vhost_log_file"
 	notice "Config file: $vhost_conf_file"
 	notice "Socket:      $vhost_socket"
 	notice "Command:     $cmd"
 
+	timing_enter vhost_start
 	cd $vhost_dir; $cmd &
 	vhost_pid=$!
 	echo $vhost_pid > $vhost_pid_file
@@ -177,12 +179,14 @@ function spdk_vhost_run()
 	notice "waiting for app to run..."
 	waitforlisten "$vhost_pid" "$vhost_dir/rpc.sock"
 	notice "vhost started - pid=$vhost_pid"
+	timing_exit vhost_start
 
 	rm $vhost_conf_file
 }
 
 function spdk_vhost_kill()
 {
+	local rc=0
 	local vhost_num=0
 	if [[ ! -z "$1" ]]; then
 		vhost_num=$1
@@ -196,6 +200,7 @@ function spdk_vhost_kill()
 		return 0
 	fi
 
+	timing_enter vhost_kill
 	local vhost_pid="$(cat $vhost_pid_file)"
 	notice "killing vhost (PID $vhost_pid) app"
 
@@ -213,19 +218,24 @@ function spdk_vhost_kill()
 			error "ERROR: vhost was NOT killed - sending SIGABRT"
 			/bin/kill -ABRT $vhost_pid
 			rm $vhost_pid_file
-			return 1
+			rc=1
+		else
+			#check vhost return code, activate trap on error
+			wait $vhost_pid
 		fi
-
-		#check vhost return code, activate trap on error
-		wait $vhost_pid
 	elif /bin/kill -0 $vhost_pid; then
 		error "vhost NOT killed - you need to kill it manually"
-		return 1
+		rc=1
 	else
 		notice "vhost was no running"
 	fi
 
-	rm $vhost_pid_file
+	timing_exit vhost_kill
+	if [[ $rc == 0 ]]; then
+		rm $vhost_pid_file
+	fi
+
+	return $rc
 }
 
 ###

test/vhost/migration/migration-tc2.job

@@ -0,0 +1,20 @@
+[global]
+blocksize_range=4k-512k
+iodepth=128
+ioengine=libaio
+filename=
+group_reporting
+thread
+numjobs=1
+direct=1
+do_verify=1
+verify=md5
+verify_fatal=1
+verify_dump=1
+verify_backlog=8
+
+[randwrite]
+rw=randwrite
+runtime=15
+time_based
+stonewall

test/vhost/migration/migration-tc2.sh

@@ -0,0 +1,207 @@
+source $SPDK_BUILD_DIR/test/nvmf/common.sh
+
+function migration_tc2_cleanup_nvmf_tgt()
+{
+	local i
+
+	if [[ ! -r "$nvmf_dir/nvmf_tgt.pid" ]]; then
+		warning "Pid file '$nvmf_dir/nvmf_tgt.pid' does not exist. "
+		return
+	fi
+
+	if [[ ! -z "$1" ]]; then
+		trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+		pkill --signal $1 -F $nvmf_dir/nvmf_tgt.pid || true
+		sleep 5
+		if ! pkill -F $nvmf_dir/nvmf_tgt.pid; then
+			fail "failed to kill nvmf_tgt app"
+		fi
+	else
+		pkill --signal SIGTERM -F $nvmf_dir/nvmf_tgt.pid || true
+		for (( i=0; i<20; i++ )); do
+			if ! pkill --signal 0 -F $nvmf_dir/nvmf_tgt.pid; then
+				break
+			fi
+			sleep 0.5
+		done
+
+		if pkill --signal 0 -F $nvmf_dir/nvmf_tgt.pid; then
+			error "nvmf_tgt failed to shutdown"
+		fi
+	fi
+
+	rm $nvmf_dir/nvmf_tgt.pid
+	unset -v nvmf_dir rpc_nvmf
+}
+
+function migration_tc2_cleanup_vhost_config()
+{
+	timing_enter migration_tc2_cleanup_vhost_config
+
+	trap 'migration_tc2_cleanup_nvmf_tgt SIGKILL; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+
+	notice "Shutting down all VMs"
+	vm_shutdown_all
+
+	notice "Removing vhost devices & controllers via RPC ..."
+	# Delete bdev first to remove all LUNs and SCSI targets
+	$rpc_0 delete_bdev Nvme0n1
+	$rpc_0 remove_vhost_controller $incoming_vm_ctrlr
+
+	$rpc_1 delete_bdev Nvme0n1
+	$rpc_1 remove_vhost_controller $target_vm_ctrlr
+
+	notice "killing vhost app"
+	spdk_vhost_kill 0
+	spdk_vhost_kill 1
+
+	unset -v incoming_vm target_vm incoming_vm_ctrlr target_vm_ctrlr
+	unset -v rpc_0 rpc_1
+
+	trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+	migration_tc2_cleanup_nvmf_tgt
+
+	timing_exit migration_tc2_cleanup_vhost_config
+}
+
+function migration_tc2_configure_vhost()
+{
+	timing_enter migration_tc2_configure_vhost
+
+	# Those are global intentionaly - they will be unset in cleanup handler
+	nvmf_dir="$TEST_DIR/nvmf_tgt"
+
+	incoming_vm=1
+	target_vm=2
+	incoming_vm_ctrlr=naa.VhostScsi0.$incoming_vm
+	target_vm_ctrlr=naa.VhostScsi0.$target_vm
+
+	rpc_nvmf="python $SPDK_BUILD_DIR/scripts/rpc.py -s $nvmf_dir/rpc.sock"
+	rpc_0="python $SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 0)/rpc.sock"
+	rpc_1="python $SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 1)/rpc.sock"
+
+	# Default cleanup/error handlers will not shutdown nvmf_tgt app so setup it
+	# here to teardown in cleanup function
+	trap 'migration_tc2_error_cleanup; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+
+	# Run nvmf_tgt and two vhost instances:
+	# nvmf_tgt uses core id 2 (-m 0x4)
+	# First uses core id 0 (vhost_0_reactor_mask=0x1)
+	# Second uses core id 1 (vhost_1_reactor_mask=0x2)
+	# This force to use VM 1 and 2.
+	timing_enter start_nvmf_tgt
+	notice "Running nvmf_tgt..."
+	mkdir -p $nvmf_dir
+	rm -f $nvmf_dir/*
+	cp $SPDK_BUILD_DIR/test/nvmf/nvmf.conf $nvmf_dir/nvmf.conf
+	$SPDK_BUILD_DIR/scripts/gen_nvme.sh >> $nvmf_dir/nvmf.conf
+	$SPDK_BUILD_DIR/app/nvmf_tgt/nvmf_tgt -s 512 -m 0x4 -c $nvmf_dir/nvmf.conf -r $nvmf_dir/rpc.sock &
+	local nvmf_tgt_pid=$!
+	echo $nvmf_tgt_pid > $nvmf_dir/nvmf_tgt.pid
+	waitforlisten "$nvmf_tgt_pid" "$nvmf_dir/rpc.sock"
+	timing_exit start_nvmf_tgt
+
+	spdk_vhost_run --conf-path=$BASE_DIR --memory=512 --vhost-num=0
+	# Those are global intentionaly
+	vhost_1_reactor_mask=0x2
+	vhost_1_master_core=1
+	spdk_vhost_run --conf-path=$BASE_DIR --memory=512 --vhost-num=1
+
+	local rdma_ip_list=$(get_available_rdma_ips)
+	local nvmf_target_ip=$(echo "$rdma_ip_list" | head -n 1)
+
+	if [[ -z "$nvmf_target_ip" ]]; then
+		fail "no NIC for nvmf target"
+	fi
+
+	notice "Configuring nvmf_tgt, vhost devices & controllers via RPC ..."
+
+	# Construct shared bdevs and controllers
+	$rpc_nvmf construct_nvmf_subsystem nqn.2016-06.io.spdk:cnode1 \
+		"trtype:RDMA traddr:$nvmf_target_ip trsvcid:4420" "" -a -s SPDK00000000000001 -n Nvme0n1
+
+	$rpc_0 construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $nvmf_target_ip -s 4420 -n "nqn.2016-06.io.spdk:cnode1"
+	$rpc_0 construct_vhost_scsi_controller $incoming_vm_ctrlr
+	$rpc_0 add_vhost_scsi_lun $incoming_vm_ctrlr 0 Nvme0n1
+
+	$rpc_1 construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $nvmf_target_ip -s 4420 -n "nqn.2016-06.io.spdk:cnode1"
+	$rpc_1 construct_vhost_scsi_controller $target_vm_ctrlr
+	$rpc_1 add_vhost_scsi_lun $target_vm_ctrlr 0 Nvme0n1
+
+	notice "Setting up VMs"
+	vm_setup --os="$os_image" --force=$incoming_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 \
+		--migrate-to=$target_vm  --memory=1024 --vhost-num=0
+	vm_setup --force=$target_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 --incoming=$incoming_vm --memory=1024 \
+		--vhost-num=1
+
+	# Run everything
+	vm_run $incoming_vm $target_vm
+
+	# Wait only for incoming VM, as target is waiting for migration
+	vm_wait_for_boot 600 $incoming_vm
+
+	notice "Configuration done"
+
+	timing_exit migration_tc2_configure_vhost
+}
+
+function migration_tc2_error_cleanup()
+{
+	trap - SIGINT ERR EXIT
+	set -x
+
+	vm_kill_all
+	migration_tc2_cleanup_vhost_config
+	notice "Migration TC2 FAILED"
+}
+
+function migration_tc2()
+{
+	# Use 2 VMs:
+	# incoming VM - the one we want to migrate
+	# targe VM - the one which will accept migration
+	local job_file="$BASE_DIR/migration-tc2.job"
+
+	migration_tc2_configure_vhost
+
+	# Run fio before migration
+	notice "Starting FIO"
+	vm_check_scsi_location $incoming_vm
+	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
+
+	# Wait a while to let the FIO time to issue some IO
+	sleep 5
+
+	# Check if fio is still running before migration
+	if ! is_fio_running $incoming_vm; then
+		vm_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"
+		error "FIO is not running before migration: process crashed or finished too early"
+	fi
+
+	vm_migrate $incoming_vm
+	sleep 3
+
+	# Check if fio is still running after migration
+	if ! is_fio_running $target_vm; then
+		vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
+		error "FIO is not running after migration: process crashed or finished too early"
+	fi
+
+	notice "Waiting for fio to finish"
+	local timeout=40
+	while is_fio_running $target_vm; do
+		sleep 1
+		echo -n "."
+		if (( timeout-- == 0 )); then
+			error "timeout while waiting for FIO!"
+		fi
+	done
+
+	notice "Fio result is:"
+	vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
+
+	migration_tc2_cleanup_vhost_config
+	notice "Migration TC2 SUCCESS"
+}
+
+migration_tc2

test/vhost/migration/migration.sh

@@ -76,6 +76,7 @@ function vm_migrate()
 		fail "source VM $1 or destination VM is not properly configured for live migration"
 	fi
 
+	timing_enter vm_migrate
 	notice "Migrating VM $1 to VM "$(basename $target_vm_dir)
 	echo -e \
 		"migrate_set_speed 1g\n" \
@@ -95,6 +96,7 @@ function vm_migrate()
 	fi
 
 	notice "Migration complete"
+	timing_exit vm_migrate
 }
 
 function is_fio_running()
@@ -120,7 +122,10 @@ for test_case in ${test_cases//,/ }; do
 	notice "==============================="
 	notice "Running Migration test case ${test_case}"
 	notice "==============================="
+
+	timing_enter migration-tc${test_case}
 	source $BASE_DIR/migration-tc${test_case}.sh
+	timing_exit migration-tc${test_case}
 done
 
 notice "Migration Test SUCCESS"

test/vhost/spdk_vhost.sh

@@ -85,7 +85,7 @@ case $1 in
 	-m|--migration)
 		echo 'Running migration suite...'
 		$WORKDIR/migration/migration.sh -x \
-		--fio-bin=$FIO_BIN --os=$VM_IMAGE --test-cases=1
+		--fio-bin=$FIO_BIN --os=$VM_IMAGE --test-cases=1,2
 		;;
 	-i|--integrity)
 		echo 'Running SCSI integrity suite...'