From 01425928ed994e5cbecc0fa7425aa130d2b23597 Mon Sep 17 00:00:00 2001 From: Karol Latecki Date: Mon, 20 Apr 2020 11:26:55 +0200 Subject: [PATCH] test/vhost: fix vhost migration tests Bug introduced in: 9fc706f test/vhost: improve run_fio --local option The original "nohup + no-wait" for spawned local VM fio processes was probably as expected. Turns out we can't "wait" for fio processes in some cases, and migration is one of them. run_fio() must start fio and exit immediately, so that parent script can have a chance to do the migration process while fio is still running. Signed-off-by: Karol Latecki Change-Id: I3c0ab65e96bfa0b020eb42df76b14deda2b43e83 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1937 Community-CI: Mellanox Build Bot Tested-by: SPDK CI Jenkins Reviewed-by: Seth Howell Reviewed-by: Maciej Wawryk Reviewed-by: Darek Stojaczyk Reviewed-by: Tomasz Zawadzki --- test/vhost/common.sh | 7 ++++++- test/vhost/migration/migration-tc1.sh | 10 ++++++---- test/vhost/migration/migration-tc2.sh | 10 ++++++---- test/vhost/migration/migration-tc3a.sh | 5 +++-- test/vhost/migration/migration-tc3b.sh | 4 ++-- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/test/vhost/common.sh b/test/vhost/common.sh index fd026852d..1a7ddd0c0 100644 --- a/test/vhost/common.sh +++ b/test/vhost/common.sh @@ -1070,6 +1070,7 @@ function run_fio() local run_plugin_mode=false local fio_start_cmd local fio_output_format="normal" + local wait_for_fio=true for arg in "$@"; do case "$arg" in @@ -1087,6 +1088,7 @@ function run_fio() run_server_mode=false ;; --json) fio_output_format="json" ;; --hide-results) hide_results=true ;; + --no-wait-for-fio) wait_for_fio=false ;; *) error "Invalid argument '$arg'" return 1 @@ -1134,12 +1136,15 @@ function run_fio() fi notice "Running local fio on VM $vm_num" - vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname" & + vm_exec $vm_num "$vm_fio_bin --output=/root/$log_fname --output-format=$fio_output_format /root/$job_fname & echo \$! > /root/fio.pid" & vm_exec_pids+=("$!") fi done if ! $run_server_mode; then + if ! $wait_for_fio; then + return 0 + fi echo "Waiting for guest fio instances to finish.." wait "${vm_exec_pids[@]}" diff --git a/test/vhost/migration/migration-tc1.sh b/test/vhost/migration/migration-tc1.sh index 9bf54e0d6..1b85774fb 100644 --- a/test/vhost/migration/migration-tc1.sh +++ b/test/vhost/migration/migration-tc1.sh @@ -55,6 +55,8 @@ function migration_tc1() # incoming VM - the one we want to migrate # targe VM - the one which will accept migration local job_file="$testdir/migration-tc1.job" + local log_file + log_file="/root/$(basename ${job_file%%.*}).log" # Run vhost vhost_run 0 @@ -74,14 +76,14 @@ function migration_tc1() notice "Starting FIO" vm_check_scsi_location $incoming_vm - run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)" + run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)" # Wait a while to let the FIO time to issue some IO sleep 5 # Check if fio is still running before migration if ! is_fio_running $incoming_vm; then - vm_exec $incoming_vm "cat /root/$(basename ${job_file}).out" + vm_exec $incoming_vm "cat $log_file" error "FIO is not running before migration: process crashed or finished too early" fi @@ -90,7 +92,7 @@ function migration_tc1() # Check if fio is still running after migration if ! is_fio_running $target_vm; then - vm_exec $target_vm "cat /root/$(basename ${job_file}).out" + vm_exec $target_vm "cat $log_file" error "FIO is not running after migration: process crashed or finished too early" fi @@ -105,7 +107,7 @@ function migration_tc1() done notice "Fio result is:" - vm_exec $target_vm "cat /root/$(basename ${job_file}).out" + vm_exec $target_vm "cat $log_file" notice "Migration DONE" diff --git a/test/vhost/migration/migration-tc2.sh b/test/vhost/migration/migration-tc2.sh index 05f153d4f..994131ff4 100644 --- a/test/vhost/migration/migration-tc2.sh +++ b/test/vhost/migration/migration-tc2.sh @@ -162,20 +162,22 @@ function migration_tc2() # incoming VM - the one we want to migrate # targe VM - the one which will accept migration local job_file="$testdir/migration-tc2.job" + local log_file + log_file="/root/$(basename ${job_file%%.*}).log" migration_tc2_configure_vhost # Run fio before migration notice "Starting FIO" vm_check_scsi_location $incoming_vm - run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)" + run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)" # Wait a while to let the FIO time to issue some IO sleep 5 # Check if fio is still running before migration if ! is_fio_running $incoming_vm; then - vm_exec $incoming_vm "cat /root/$(basename ${job_file}).out" + vm_exec $incoming_vm "cat $log_file" error "FIO is not running before migration: process crashed or finished too early" fi @@ -184,7 +186,7 @@ function migration_tc2() # Check if fio is still running after migration if ! is_fio_running $target_vm; then - vm_exec $target_vm "cat /root/$(basename ${job_file}).out" + vm_exec $target_vm "cat $log_file" error "FIO is not running after migration: process crashed or finished too early" fi @@ -199,7 +201,7 @@ function migration_tc2() done notice "Fio result is:" - vm_exec $target_vm "cat /root/$(basename ${job_file}).out" + vm_exec $target_vm "cat $log_file" migration_tc2_cleanup_vhost_config notice "Migration TC2 SUCCESS" diff --git a/test/vhost/migration/migration-tc3a.sh b/test/vhost/migration/migration-tc3a.sh index 0824383a3..b3c335c59 100644 --- a/test/vhost/migration/migration-tc3a.sh +++ b/test/vhost/migration/migration-tc3a.sh @@ -8,6 +8,7 @@ target_vm_ctrlr=naa.VhostScsi0.$target_vm share_dir=$TEST_DIR/share spdk_repo_share_dir=$TEST_DIR/share_spdk job_file=$testdir/migration-tc3.job +log_file="/root/$(basename ${job_file%%.*}).log" if [ -z "$MGMT_TARGET_IP" ]; then error "No IP address of target is given" @@ -206,11 +207,11 @@ function migration_tc3() notice "Starting fio on local VM" vm_check_scsi_location $incoming_vm - run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)" + run_fio $fio_bin --job-file="$job_file" --no-wait-for-fio --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)" sleep 5 if ! is_fio_running $incoming_vm; then - vh_ssh $incoming_vm "cat /root/$(basename ${job_file}).out" + vm_exec $incoming_vm "cat $log_file" error "Fio not running on local VM before starting migration!" fi diff --git a/test/vhost/migration/migration-tc3b.sh b/test/vhost/migration/migration-tc3b.sh index 3ac21b582..d3d536410 100644 --- a/test/vhost/migration/migration-tc3b.sh +++ b/test/vhost/migration/migration-tc3b.sh @@ -58,7 +58,7 @@ if ! vm_os_booted $target_vm; then fi if ! is_fio_running $target_vm; then - vm_exec $target_vm "cat /root/migration-tc3.job.out" + vm_exec $target_vm "cat /root/migration-tc3.log" error "FIO is not running on remote server after migration!" fi @@ -73,7 +73,7 @@ while is_fio_running $target_vm; do done notice "FIO result after migration:" -vm_exec $target_vm "cat /root/migration-tc3.job.out" +vm_exec $target_vm "cat /root/migration-tc3.log" host_2_cleanup_vhost echo "DONE" > $share_dir/DONE