ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test/vhost: Vhost blk hotremove test plan.

Browse Source 
Change-Id: I2deb9ae125eecf5592d59902cc5a530993f0fa54
Signed-off-by: Pawel Kaminski <pawelx.kaminski@intel.com>
Reviewed-on: https://review.gerrithub.io/402276
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Pawel Kaminski 2018-03-02 10:47:42 -05:00 committed by Jim Harris
parent 8f1f9137b1
commit 6255620460
1 changed files with 65 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
test/vhost/hotplug/blk_hotremove.sh
View File

@ -1,3 +1,18 @@
# Vhost blk hot remove tests
#
# Objective
# The purpose of these tests is to verify that SPDK vhost remains stable during
# hot-remove operations performed on SCSI and BLK controllers devices.
# Hot-remove is a scenario where a NVMe device is removed when already in use.
#
# Test cases description
# 1. FIO I/O traffic is run during hot-remove operations.
# By default FIO uses default_integrity*.job config files located in
# test/vhost/hotplug/fio_jobs directory.
# 2. FIO mode of operation is random write (randwrite) with verification enabled
# which results in also performing read operations.
# 3. In test cases fio status is checked after every run if any errors occurred.
function prepare_fio_cmd_tc1() { function prepare_fio_cmd_tc1() {
print_test_fio_header print_test_fio_header
@ -23,40 +38,56 @@ function remove_vhost_controllers() {
$rpc_py remove_vhost_controller naa.Nvme0n1p3.1 $rpc_py remove_vhost_controller naa.Nvme0n1p3.1
} }
# Vhost blk hot remove test cases
#
# Test Case 1
function blk_hotremove_tc1() { function blk_hotremove_tc1() {
echo "Blk hotremove test case 1" echo "Blk hotremove test case 1"
traddr="" traddr=""
# 1. Run the command to hot remove NVMe disk.
get_traddr "Nvme0" get_traddr "Nvme0"
delete_nvme "Nvme0n1" delete_nvme "Nvme0n1"
# 2. If vhost had crashed then tests would stop running
sleep 1 sleep 1
add_nvme "HotInNvme0" "$traddr" add_nvme "HotInNvme0" "$traddr"
sleep 1 sleep 1
} }
# Test Case 2
function blk_hotremove_tc2() { function blk_hotremove_tc2() {
echo "Blk hotremove test case 2" echo "Blk hotremove test case 2"
# 1. Use rpc command to create blk controllers.
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme0n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme0n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 Nvme1n1p1 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 Nvme1n1p1
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p2 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p2
# 2. Run two VMs and attach every VM to two blk controllers.
vm_run_with_arg "0 1" vm_run_with_arg "0 1"
vms_prepare "0" vms_prepare "0"
traddr="" traddr=""
get_traddr "Nvme0" get_traddr "Nvme0"
prepare_fio_cmd_tc1 "0" prepare_fio_cmd_tc1 "0"
# 3. Run FIO I/O traffic with verification enabled on NVMe disk.
$run_fio & $run_fio &
local last_pid=$! local last_pid=$!
sleep 3 sleep 3
# 4. Run the command to hot remove NVMe disk.
delete_nvme "HotInNvme0n1" delete_nvme "HotInNvme0n1"
local retcode=0 local retcode=0
wait_for_finish $last_pid || retcode=$? wait_for_finish $last_pid || retcode=$?
# 5. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 2: Iteration 1." 1 $retcode check_fio_retcode "Blk hotremove test case 2: Iteration 1." 1 $retcode
# 6. Reboot VM
reboot_all_and_prepare "0" reboot_all_and_prepare "0"
# 7. Run FIO I/O traffic with verification enabled on NVMe disk.
$run_fio & $run_fio &
local retcode=0 local retcode=0
wait_for_finish $! || retcode=$? wait_for_finish $! || retcode=$?
# 8. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 2: Iteration 2." 1 $retcode check_fio_retcode "Blk hotremove test case 2: Iteration 2." 1 $retcode
vm_shutdown_all vm_shutdown_all
remove_vhost_controllers remove_vhost_controllers
@ -64,30 +95,41 @@ function blk_hotremove_tc2() {
sleep 1 sleep 1
} }
# ## Test Case 3
function blk_hotremove_tc3() { function blk_hotremove_tc3() {
echo "Blk hotremove test case 3" echo "Blk hotremove test case 3"
# 1. Use rpc command to create blk controllers.
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme1n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme1n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 HotInNvme1n1p1 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 HotInNvme1n1p1
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p1 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p1
# 2. Run two VMs and attach every VM to two blk controllers.
vm_run_with_arg "0 1" vm_run_with_arg "0 1"
vms_prepare "0 1" vms_prepare "0 1"
traddr="" traddr=""
get_traddr "Nvme0" get_traddr "Nvme0"
prepare_fio_cmd_tc1 "0" prepare_fio_cmd_tc1 "0"
# 3. Run FIO I/O traffic with verification enabled on first NVMe disk.
$run_fio & $run_fio &
local last_pid=$! local last_pid=$!
sleep 3 sleep 3
# 4. Run the command to hot remove of first NVMe disk.
delete_nvme "HotInNvme1n1" delete_nvme "HotInNvme1n1"
local retcode=0 local retcode=0
wait_for_finish $last_pid || retcode=$? wait_for_finish $last_pid || retcode=$?
# 6. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 3: Iteration 1." 1 $retcode check_fio_retcode "Blk hotremove test case 3: Iteration 1." 1 $retcode
# 7. Reboot VM
reboot_all_and_prepare "0" reboot_all_and_prepare "0"
local retcode=0 local retcode=0
# 8. Run FIO I/O traffic with verification enabled on removed NVMe disk.
$run_fio & $run_fio &
wait_for_finish $! || retcode=$? wait_for_finish $! || retcode=$?
# 9. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 3: Iteration 2." 1 $retcode check_fio_retcode "Blk hotremove test case 3: Iteration 2." 1 $retcode
vm_shutdown_all vm_shutdown_all
remove_vhost_controllers remove_vhost_controllers
@ -95,37 +137,49 @@ function blk_hotremove_tc3() {
sleep 1 sleep 1
} }
# Test Case 4
function blk_hotremove_tc4() { function blk_hotremove_tc4() {
echo "Blk hotremove test case 4" echo "Blk hotremove test case 4"
# 1. Use rpc command to create blk controllers.
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme2n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme2n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 HotInNvme2n1p1 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 HotInNvme2n1p1
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p1 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p1
# 2. Run two VM, attached to blk controllers.
vm_run_with_arg "0 1" vm_run_with_arg "0 1"
vms_prepare "0 1" vms_prepare "0 1"
prepare_fio_cmd_tc1 "0" prepare_fio_cmd_tc1 "0"
# 3. Run FIO I/O traffic on first VM with verification enabled on both NVMe disks.
$run_fio & $run_fio &
local last_pid_vm0=$! local last_pid_vm0=$!
prepare_fio_cmd_tc1 "1" prepare_fio_cmd_tc1 "1"
# 4. Run FIO I/O traffic on second VM with verification enabled on both NVMe disks.
$run_fio & $run_fio &
local last_pid_vm1=$! local last_pid_vm1=$!
sleep 3 sleep 3
prepare_fio_cmd_tc1 "0 1" prepare_fio_cmd_tc1 "0 1"
# 5. Run the command to hot remove of first NVMe disk.
delete_nvme "HotInNvme2n1" delete_nvme "HotInNvme2n1"
local retcode_vm0=0 local retcode_vm0=0
local retcode_vm1=0 local retcode_vm1=0
wait_for_finish $last_pid_vm0 || retcode_vm0=$? wait_for_finish $last_pid_vm0 || retcode_vm0=$?
wait_for_finish $last_pid_vm1 || retcode_vm1=$? wait_for_finish $last_pid_vm1 || retcode_vm1=$?
# 6. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 4: Iteration 1." 1 $retcode_vm0 check_fio_retcode "Blk hotremove test case 4: Iteration 1." 1 $retcode_vm0
check_fio_retcode "Blk hotremove test case 4: Iteration 2." 1 $retcode_vm1 check_fio_retcode "Blk hotremove test case 4: Iteration 2." 1 $retcode_vm1
# 7. Reboot all VMs.
reboot_all_and_prepare "0 1" reboot_all_and_prepare "0 1"
# 8. Run FIO I/O traffic with verification enabled on removed NVMe disk.
$run_fio & $run_fio &
local retcode=0 local retcode=0
wait_for_finish $! || retcode=$? wait_for_finish $! || retcode=$?
# 9. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 4: Iteration 3." 1 $retcode check_fio_retcode "Blk hotremove test case 4: Iteration 3." 1 $retcode
vm_shutdown_all vm_shutdown_all
@ -134,28 +188,39 @@ function blk_hotremove_tc4() {
sleep 1 sleep 1
} }
# Test Case 5
function blk_hotremove_tc5() { function blk_hotremove_tc5() {
echo "Blk hotremove test case 5" echo "Blk hotremove test case 5"
# 1. Use rpc command to create blk controllers.
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme3n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p0.0 HotInNvme3n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p1.0 Nvme1n1p0
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 Nvme1n1p1 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p2.1 Nvme1n1p1
$rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p2 $rpc_py construct_vhost_blk_controller naa.Nvme0n1p3.1 Nvme1n1p2
# 2. Run two VM, attached to blk controllers.
vm_run_with_arg "0 1" vm_run_with_arg "0 1"
vms_prepare "0 1" vms_prepare "0 1"
prepare_fio_cmd_tc1 "0" prepare_fio_cmd_tc1 "0"
# 3. Run FIO I/O traffic on first VM with verification enabled on both NVMe disks.
$run_fio & $run_fio &
local last_pid=$! local last_pid=$!
sleep 3 sleep 3
# 4. Run the command to hot remove of first NVMe disk.
delete_nvme "HotInNvme3n1" delete_nvme "HotInNvme3n1"
local retcode=0 local retcode=0
wait_for_finish $last_pid || retcode=$? wait_for_finish $last_pid || retcode=$?
# 5. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 5: Iteration 1." 1 $retcode check_fio_retcode "Blk hotremove test case 5: Iteration 1." 1 $retcode
# 6. Reboot VM.
reboot_all_and_prepare "0" reboot_all_and_prepare "0"
local retcode=0 local retcode=0
# 7. Run FIO I/O traffic with verification enabled on removed NVMe disk.
$run_fio & $run_fio &
wait_for_finish $! || retcode=$? wait_for_finish $! || retcode=$?
# 8. Check that fio job run on hot-removed device stopped.
# Expected: Fio should return error message and return code != 0.
check_fio_retcode "Blk hotremove test case 5: Iteration 2." 1 $retcode check_fio_retcode "Blk hotremove test case 5: Iteration 2." 1 $retcode
vm_shutdown_all vm_shutdown_all
remove_vhost_controllers remove_vhost_controllers