From efb18b9b4606a6e5bebef051e1736fd00c88d658 Mon Sep 17 00:00:00 2001 From: Pawel Wodkowski Date: Wed, 3 Jan 2018 14:24:38 +0100 Subject: [PATCH] vhost: add live migration support This patch adds support for live migration for vhost-scsi and vhost-blk backends. Change-Id: Ibfc8a713dbba14ba8cb38377a71e28fd340b1487 Signed-off-by: Pawel Wodkowski Reviewed-on: https://review.gerrithub.io/394203 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Jim Harris --- lib/vhost/rte_vhost/vhost_user.c | 4 +- lib/vhost/vhost.c | 70 ++++++- lib/vhost/vhost_internal.h | 3 +- test/unit/lib/vhost/vhost.c/vhost_ut.c | 3 + test/vhost/common/common.sh | 109 ++++++++-- test/vhost/common/run_fio.py | 5 +- test/vhost/common/vm_setup.sh | 81 ++++---- test/vhost/initiator/blockdev.sh | 12 +- test/vhost/migration/migration-malloc.job | 25 +++ test/vhost/migration/migration-malloc.sh | 231 ++++++++++++++++++++++ test/vhost/migration/vhost.conf.in | 5 + test/vhost/spdk_vhost.sh | 5 + 12 files changed, 477 insertions(+), 76 deletions(-) create mode 100644 test/vhost/migration/migration-malloc.job create mode 100755 test/vhost/migration/migration-malloc.sh create mode 100644 test/vhost/migration/vhost.conf.in diff --git a/lib/vhost/rte_vhost/vhost_user.c b/lib/vhost/rte_vhost/vhost_user.c index 1f293198c..bc38e3d61 100644 --- a/lib/vhost/rte_vhost/vhost_user.c +++ b/lib/vhost/rte_vhost/vhost_user.c @@ -1076,8 +1076,8 @@ vhost_user_msg_handler(int vid, int fd) return -1; } - RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n", - vhost_message_str[msg.request]); + RTE_LOG(INFO, VHOST_CONFIG, "%s: read message %s\n", + dev->ifname, vhost_message_str[msg.request]); ret = vhost_user_check_and_alloc_queue_pair(dev, &msg); if (ret < 0) { diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index d8bc4ad21..3b2089092 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -91,6 +91,71 @@ void *spdk_vhost_gpa_to_vva(struct spdk_vhost_dev *vdev, uint64_t addr) return (void *)rte_vhost_gpa_to_vva(vdev->mem, addr); } +static void +spdk_vhost_log_req_desc(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *virtqueue, + uint16_t req_id) +{ + struct vring_desc *desc, *desc_table; + uint32_t desc_table_size; + int rc; + + if (spdk_likely(!spdk_vhost_dev_has_feature(vdev, VHOST_F_LOG_ALL))) { + return; + } + + rc = spdk_vhost_vq_get_desc(vdev, virtqueue, req_id, &desc, &desc_table, &desc_table_size); + if (spdk_unlikely(rc != 0)) { + SPDK_ERRLOG("Can't log used ring descriptors!\n"); + return; + } + + do { + if (spdk_vhost_vring_desc_is_wr(desc)) { + /* To be honest, only pages realy touched should be logged, but + * doing so would require tracking those changes in each backed. + * Also backend most likely will touch all/most of those pages so + * for lets assume we touched all pages passed to as writeable buffers. */ + rte_vhost_log_write(vdev->vid, desc->addr, desc->len); + } + spdk_vhost_vring_desc_get_next(&desc, desc_table, desc_table_size); + } while (desc); +} + +static void +spdk_vhost_log_used_vring_elem(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *virtqueue, + uint16_t idx) +{ + uint64_t offset, len; + uint16_t vq_idx; + + if (spdk_likely(!spdk_vhost_dev_has_feature(vdev, VHOST_F_LOG_ALL))) { + return; + } + + offset = offsetof(struct vring_used, ring[idx]); + len = sizeof(virtqueue->vring.used->ring[idx]); + vq_idx = virtqueue - vdev->virtqueue; + + rte_vhost_log_used_vring(vdev->vid, vq_idx, offset, len); +} + +static void +spdk_vhost_log_used_vring_idx(struct spdk_vhost_dev *vdev, struct spdk_vhost_virtqueue *virtqueue) +{ + uint64_t offset, len; + uint16_t vq_idx; + + if (spdk_likely(!spdk_vhost_dev_has_feature(vdev, VHOST_F_LOG_ALL))) { + return; + } + + offset = offsetof(struct vring_used, idx); + len = sizeof(virtqueue->vring.used->idx); + vq_idx = virtqueue - vdev->virtqueue; + + rte_vhost_log_used_vring(vdev->vid, vq_idx, offset, len); +} + /* * Get available requests from avail ring. */ @@ -290,13 +355,17 @@ spdk_vhost_vq_used_ring_enqueue(struct spdk_vhost_dev *vdev, struct spdk_vhost_v "Queue %td - USED RING: last_idx=%"PRIu16" req id=%"PRIu16" len=%"PRIu32"\n", virtqueue - vdev->virtqueue, vring->last_used_idx, id, len); + spdk_vhost_log_req_desc(vdev, virtqueue, id); + vring->last_used_idx++; used->ring[last_idx].id = id; used->ring[last_idx].len = len; + spdk_vhost_log_used_vring_elem(vdev, virtqueue, last_idx); /* Ensure the used ring is updated before we increment used->idx. */ spdk_smp_wmb(); * (volatile uint16_t *) &used->idx = vring->last_used_idx; + spdk_vhost_log_used_vring_idx(vdev, virtqueue); /* Ensure all our used ring changes are visible to the guest at the time * of interrupt. @@ -970,7 +1039,6 @@ start_device(int vid) SPDK_ERRLOG("vhost device %d: Failed to disable guest notification on queue %"PRIu16"\n", vid, i); goto out; } - } vdev->num_queues = num_queues; diff --git a/lib/vhost/vhost_internal.h b/lib/vhost/vhost_internal.h index 66ef7b3b7..453dd385e 100644 --- a/lib/vhost/vhost_internal.h +++ b/lib/vhost/vhost_internal.h @@ -90,8 +90,7 @@ (1ULL << VIRTIO_RING_F_EVENT_IDX) | \ (1ULL << VIRTIO_RING_F_INDIRECT_DESC)) -#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VHOST_F_LOG_ALL) | \ - (1ULL << VIRTIO_RING_F_EVENT_IDX) | \ +#define SPDK_VHOST_DISABLED_FEATURES ((1ULL << VIRTIO_RING_F_EVENT_IDX) | \ (1ULL << VIRTIO_RING_F_INDIRECT_DESC)) enum spdk_vhost_dev_type { diff --git a/test/unit/lib/vhost/vhost.c/vhost_ut.c b/test/unit/lib/vhost/vhost.c/vhost_ut.c index 1774c5c80..fa025d1bc 100644 --- a/test/unit/lib/vhost/vhost.c/vhost_ut.c +++ b/test/unit/lib/vhost/vhost.c/vhost_ut.c @@ -81,6 +81,9 @@ DEFINE_STUB(rte_vhost_driver_callback_register, int, DEFINE_STUB(rte_vhost_driver_disable_features, int, (const char *path, uint64_t features), 0); DEFINE_STUB(rte_vhost_driver_set_features, int, (const char *path, uint64_t features), 0); DEFINE_STUB(rte_vhost_driver_register, int, (const char *path, uint64_t flags), 0); +DEFINE_STUB_V(rte_vhost_log_used_vring, (int vid, uint16_t vring_idx, uint64_t offset, + uint64_t len)); +DEFINE_STUB_V(rte_vhost_log_write, (int vid, uint64_t addr, uint64_t len)); DEFINE_STUB(spdk_vhost_scsi_controller_construct, int, (void), 0); DEFINE_STUB(spdk_vhost_blk_controller_construct, int, (void), 0); DEFINE_STUB(rte_vhost_set_vhost_vring_last_idx, int, diff --git a/test/vhost/common/common.sh b/test/vhost/common/common.sh index 34edc75cc..5c4054dfa 100644 --- a/test/vhost/common/common.sh +++ b/test/vhost/common/common.sh @@ -388,7 +388,7 @@ function vm_shutdown_all() while [[ $timeo -gt 0 ]]; do all_vms_down=1 for vm in $VM_BASE_DIR/[0-9]*; do - if /bin/kill -0 "$(cat $vm/qemu.pid)"; then + if [[ -r $vm/qemu.pid ]] && pkill -0 -F "$vm/qemu.pid"; then all_vms_down=0 break fi @@ -412,13 +412,16 @@ function vm_shutdown_all() function vm_setup() { local shell_restore_x="$( [[ "$-" =~ x ]] && echo 'set -x' )" - local OPTIND optchar a + local OPTIND optchar vm_num local os="" + local os_mode="" local qemu_args="" local disk_type=NOT_DEFINED local disks="" local raw_cache="" + local vm_incoming="" + local vm_migrate_to="" local force_vm="" local guest_memory=1024 local queue_number="" @@ -435,6 +438,8 @@ function vm_setup() force=*) local force_vm=${OPTARG#*=} ;; memory=*) local guest_memory=${OPTARG#*=} ;; queue_num=*) local queue_number=${OPTARG#*=} ;; + incoming=*) local vm_incoming="${OPTARG#*=}" ;; + migrate-to=*) local vm_migrate_to="${OPTARG#*=}" ;; *) error "unknown argument $OPTARG" return 1 @@ -454,8 +459,6 @@ function vm_setup() vm_num_is_valid $vm_num || return 1 local vm_dir="$VM_BASE_DIR/$vm_num" [[ -d $vm_dir ]] && warning "removing existing VM in '$vm_dir'" - # FIXME: why this is just echo??? - echo "rm -rf $vm_dir" else local vm_dir="" @@ -474,11 +477,43 @@ function vm_setup() return 1 fi + if [[ ! -z "$vm_migrate_to" && ! -z "$vm_incoming" ]]; then + error "'--incoming' and '--migrate-to' cannot be used together" + return 1 + elif [[ ! -z "$vm_incoming" ]]; then + if [[ ! -z "$os_mode" || ! -z "$os_img" ]]; then + error "'--incoming' can't be used together with '--os' nor '--os-mode'" + return 1 + fi + + os_mode="original" + os="$VM_BASE_DIR/$vm_incoming/os.qcow2" + elif [[ ! -z "$vm_migrate_to" ]]; then + [[ "$os_mode" != "backing" ]] && warning "Using 'backing' mode for OS since '--migrate-to' is used" + os_mode=backing + fi + notice "Creating new VM in $vm_dir" mkdir -p $vm_dir - if [[ ! -r $os ]]; then - error "file not found: $os" - return 1 + + if [[ "$os_mode" == "backing" ]]; then + notice "Creating backing file for OS image file: $os" + if ! $INSTALL_DIR/bin/qemu-img create -f qcow2 -b $os $vm_dir/os.qcow2; then + error "Failed to create OS backing file in '$vm_dir/os.qcow2' using '$os'" + return 1 + fi + + local os=$vm_dir/os.qcow2 + elif [[ "$os_mode" == "original" ]]; then + warning "Using original OS image file: $os" + elif [[ "$os_mode" != "snapshot" ]]; then + if [[ -z "$os_mode" ]]; then + notice "No '--os-mode' parameter provided - using 'snapshot'" + os_mode="snapshot" + else + error "Invalid '--os-mode=$os_mode'" + return 1 + fi fi # WARNING: @@ -501,8 +536,8 @@ function vm_setup() local ssh_socket=$(( vm_socket_offset + 0 )) local fio_socket=$(( vm_socket_offset + 1 )) - # vm_socket_offset + 2 - can be reused - # vm_socket_offset + 3 - can be reused + local monitor_port=$(( vm_socket_offset + 2 )) + local migration_port=$(( vm_socket_offset + 3 )) local gdbserver_socket=$(( vm_socket_offset + 4 )) local vnc_socket=$(( 100 + vm_num )) local qemu_pid_file="$vm_dir/qemu.pid" @@ -520,18 +555,19 @@ function vm_setup() $shell_restore_x - #-cpu host local node_num=${!qemu_numa_node_param} notice "NUMA NODE: $node_num" - cmd+="-m $guest_memory --enable-kvm -cpu host -smp $cpu_num -vga std -vnc :$vnc_socket -daemonize -snapshot ${eol}" + cmd+="-m $guest_memory --enable-kvm -cpu host -smp $cpu_num -vga std -vnc :$vnc_socket -daemonize ${eol}" cmd+="-object memory-backend-file,id=mem,size=${guest_memory}M,mem-path=/dev/hugepages,share=on,prealloc=yes,host-nodes=$node_num,policy=bind ${eol}" + [[ $os_mode == snapshot ]] && cmd+="-snapshot ${eol}" + [[ ! -z "$vm_incoming" ]] && cmd+=" -incoming tcp:0:$migration_port ${eol}" + cmd+="-monitor telnet:127.0.0.1:$monitor_port,server,nowait ${eol}" cmd+="-numa node,memdev=mem ${eol}" cmd+="-pidfile $qemu_pid_file ${eol}" cmd+="-serial file:$vm_dir/serial.log ${eol}" cmd+="-D $vm_dir/qemu.log ${eol}" cmd+="-net user,hostfwd=tcp::$ssh_socket-:22,hostfwd=tcp::$fio_socket-:8765 ${eol}" cmd+="-net nic ${eol}" - cmd+="-drive file=$os,if=none,id=os_disk ${eol}" cmd+="-device ide-hd,drive=os_disk,bootindex=0 ${eol}" @@ -608,7 +644,7 @@ function vm_setup() # remove last $eol cmd="${cmd%\\\\\\n }" - notice "Saving to $vm_dir/run.sh:" + notice "Saving to $vm_dir/run.sh" ( echo '#!/bin/bash' echo 'if [[ $EUID -ne 0 ]]; then ' @@ -638,8 +674,16 @@ function vm_setup() # Save generated sockets redirection echo $ssh_socket > $vm_dir/ssh_socket echo $fio_socket > $vm_dir/fio_socket + echo $monitor_port > $vm_dir/monitor_port + + rm -f $vm_dir/migration_port + [[ -z $vm_incoming ]] || echo $migration_port > $vm_dir/migration_port + echo $gdbserver_socket > $vm_dir/gdbserver_socket echo $vnc_socket >> $vm_dir/vnc_socket + + [[ -z $vm_incoming ]] || ln -fs $VM_BASE_DIR/$vm_incoming $vm_dir/vm_incoming + [[ -z $vm_migrate_to ]] || ln -fs $VM_BASE_DIR/$vm_migrate_to $vm_dir/vm_migrate_to } function vm_run() @@ -840,16 +884,18 @@ function run_fio() local out="" local fio_disks="" local vm + local run_server_mode=true for arg in $@; do case "$arg" in --job-file=*) local job_file="${arg#*=}" ;; - --fio-bin=*) local fio_bin="--fio-bin=${arg#*=}" ;; + --fio-bin=*) local fio_bin="${arg#*=}" ;; --vm=*) vms+=( "${arg#*=}" ) ;; --out=*) - local out="$arg" - mkdir -p ${out#*=} - ;; + local out="${arg#*=}" + mkdir -p $out + ;; + --local) run_server_mode=false ;; *) error "Invalid argument '$arg'" return 1 @@ -857,8 +903,17 @@ function run_fio() esac done - local job_fname=$(basename "$job_file") + if [[ ! -z "$fio_bin" && ! -r "$fio_bin" ]]; then + error "FIO binary '$fio_bin' does not exist" + return 1 + fi + if [[ ! -r "$job_file" ]]; then + error "Fio job '$job_file' does not exist" + return 1 + fi + + local job_fname=$(basename "$job_file") # prepare job file for each VM for vm in ${vms[@]}; do local vm_num=${vm%%:*} @@ -868,9 +923,25 @@ function run_fio() fio_disks+="127.0.0.1:$(vm_fio_socket $vm_num):$vmdisks," vm_ssh $vm_num cat /root/$job_fname + if ! $run_server_mode; then + if [[ ! -z "$fio_bin" ]]; then + cat $fio_bin | vm_ssh $vm_num 'cat > /root/fio; chmod +x /root/fio' + fi + + notice "Running local fio on VM $vm_num" + vm_ssh $vm_num "nohup /root/fio /root/$job_fname 1>/root/$job_fname.out 2>/root/$job_fname.out /root/fio.pid" + fi done - python $SPDK_BUILD_DIR/test/vhost/common/run_fio.py --job-file=/root/$job_fname $fio_bin $out ${fio_disks%,} + if ! $run_server_mode; then + # Give FIO time to run + sleep 0.5 + return 0 + fi + + python $SPDK_BUILD_DIR/test/vhost/common/run_fio.py --job-file=/root/$job_fname \ + $([[ ! -z "$fio_bin" ]] && echo "--fio-bin=$fio_bin") \ + --out=$out ${fio_disks%,} } # Shutdown or kill any running VM and SPDK APP. diff --git a/test/vhost/common/run_fio.py b/test/vhost/common/run_fio.py index 959369a8e..502409d75 100755 --- a/test/vhost/common/run_fio.py +++ b/test/vhost/common/run_fio.py @@ -20,7 +20,7 @@ def show_help(): Options: -h, --help Show this message. -j, --job-file Paths to file with FIO job configuration on remote host. - -f, --fio-bin Location of FIO binary on remote host (Default "fio") + -f, --fio-bin Location of FIO binary on local host (Default "fio") -o, --out Directory used to save generated job files and files with test results -p, --perf-vmex Enable aggregating statistic for VMEXITS for VMs @@ -112,6 +112,7 @@ def main(): vms = [] fio_cfg = None + out_dir = None perf_vmex = False try: @@ -143,7 +144,7 @@ def main(): print("ERROR! No FIO job provided!") sys.exit(1) - if not os.path.exists(out_dir): + if out_dir is None or not os.path.exists(out_dir): print("ERROR! Folder {out_dir} does not exist ".format(out_dir=out_dir)) sys.exit(1) diff --git a/test/vhost/common/vm_setup.sh b/test/vhost/common/vm_setup.sh index 3b26fe1bf..8dffb2340 100755 --- a/test/vhost/common/vm_setup.sh +++ b/test/vhost/common/vm_setup.sh @@ -11,70 +11,63 @@ function usage() echo "Usage: $(basename $1) [OPTIONS] VM_NUM" echo echo "-h, --help print help and exit" - echo "-f VM_NUM Force VM_NUM reconfiguration if already exist" echo " --work-dir=WORK_DIR Where to find build file. Must exit. (default: $TEST_DIR)" - echo " --test-type=TYPE Perform specified test:" + echo " --force=VM_NUM Force VM_NUM reconfiguration if already exist" + echo " --disk-type=TYPE Perform specified test:" echo " virtio - test host virtio-scsi-pci using file as disk image" echo " kernel_vhost - use kernel driver vhost-scsi" echo " spdk_vhost_scsi - use spdk vhost scsi" echo " spdk_vhost_blk - use spdk vhost block" - echo " ---cache=CACHE Use CACHE for virtio test: " + echo " --raw-cache=CACHE Use CACHE for virtio test: " echo " writethrough, writeback, none, unsafe or directsyns" - echo " Default is writethrough" echo " --disk=PATH Disk to use in test. test specific meaning:" echo " virtio - disk path (file or block device ex: /dev/nvme0n1)" echo " kernel_vhost - the WWN number to be used" - echo " spdk_vhost - the socket path. Default is WORK_DIR/vhost/usvhost" + echo " spdk_vhost_[scsi|blk] - the socket path." echo " --os=OS_QCOW2 Custom OS qcow2 image file" echo " --os-mode=MODE MODE how to use provided image: default: backing" echo " backing - create new image but use provided backing file" echo " copy - copy provided image and use a copy" echo " orginal - use file directly. Will modify the provided file" + echo " --incoming=VM_NUM Use VM_NUM as source migration VM." + echo " --migrate-to=VM_NUM Use VM_NUM as target migration VM." echo "-x Turn on script debug (set -x)" + echo "-v Be more verbose" exit 0 } -disk="" -raw_cache="" -img_mode="" -os="" -while getopts 'xf:h-:' optchar; do - case "$optchar" in - -) - case "$OPTARG" in - help) usage $0 ;; - work-dir=*) TEST_DIR="${OPTARG#*=}" ;; - raw-cache=*) raw_cache="--raw-cache=${OPTARG#*=}" ;; - test-type=*) test_type="${OPTARG#*=}" ;; - spdk-vhost-mode=*) spdk_vhost_mode="${OPTARG#*=}" ;; - disk=*) disk="${OPTARG#*=}" ;; - os=*) os="${OPTARG#*=}" - if [[ ! -r "$os" ]]; then - echo "ERROR: can't read '$os'" - usage $0 - fi - os="$(readlink -f $os)" - ;; - os-mode=*) os_mode="--os-mode=${OPTARG#*=}" ;; - *) usage $0 "Invalid argument '$OPTARG'" ;; - esac - ;; - h) usage $0 ;; - x) set -x ;; - f) force_vm_num="--force=${OPTARG#*=}" ;; - *) usage $0 "Invalid argument '$OPTARG'" ;; + +setup_params=() +for param in "$@"; do + case "$param" in + --help|-h) usage $0 ;; + --work-dir=*) + TEST_DIR="${param#*=}" + continue + ;; + --raw-cache=*) ;; + --disk-type=*) ;; + --disks=*) ;; + --os=*) ;; + --os-mode=*) ;; + --force=*) ;; + --incoming=*) ;; + --migrate-to=*) ;; + -x) + set -x + continue + ;; + -v) + SPDK_VHOST_VERBOSE=true + continue + ;; + *) usage $0 "Invalid argument '$param'" ;; esac + + setup_params+=( "$param" ) done . $COMMON_DIR/common.sh -[[ -z "$os" ]] && os="$TEST_DIR/debian.qcow2" -[[ $test_type =~ "spdk_vhost" ]] && [[ -z "$disk" ]] && disk="$SPDK_VHOST_SCSI_TEST_DIR/usvhost" -if [[ $test_type == "kernel_vhost" ]] && [[ -z "$disk" ]]; then - fail "for $test_type '--disk=WWN' is mandatory" -fi +vm_setup ${setup_params[@]} -vm_setup \ - --os=$os \ - --disk-type=$test_type \ - --disks=$disk \ - $wwn $raw_cache $force_vm_num $os_mode +trap -- ERR diff --git a/test/vhost/initiator/blockdev.sh b/test/vhost/initiator/blockdev.sh index 2b8f7b95e..55ad91120 100755 --- a/test/vhost/initiator/blockdev.sh +++ b/test/vhost/initiator/blockdev.sh @@ -107,10 +107,10 @@ vm_no="0" vm_setup --disk-type=spdk_vhost_scsi --force=$vm_no --os=$os_image --disks="Nvme0n1:Malloc0:Malloc1" --queue_num=18 --memory=6144 vm_run $vm_no vm_wait_for_boot 600 $vm_no -vm_scp $vm_num -r $ROOT_DIR "127.0.0.1:/root/spdk" -vm_ssh $vm_num " cd spdk ; make clean ; ./configure --with-fio=/root/fio_src ; make -j2" -vm_ssh $vm_num "/root/spdk/scripts/setup.sh" -vbdevs=$(vm_ssh $vm_num ". /root/spdk/scripts/autotest_common.sh && discover_bdevs /root/spdk \ +vm_scp $vm_no -r $ROOT_DIR "127.0.0.1:/root/spdk" +vm_ssh $vm_no " cd spdk ; make clean ; ./configure --with-fio=/root/fio_src ; make -j2" +vm_ssh $vm_no "/root/spdk/scripts/setup.sh" +vbdevs=$(vm_ssh $vm_no ". /root/spdk/scripts/autotest_common.sh && discover_bdevs /root/spdk \ /root/spdk/test/vhost/initiator/bdev_pci.conf") virtio_bdevs=$(jq -r '[.[].name] | join(":")' <<< $vbdevs) virtio_with_unmap=$(jq -r '[.[] | select(.supported_io_types.unmap==true).name] @@ -118,14 +118,14 @@ virtio_with_unmap=$(jq -r '[.[] | select(.supported_io_types.unmap==true).name] timing_exit setup_vm timing_enter run_spdk_fio_pci -vm_ssh $vm_num "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \ +vm_ssh $vm_no "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \ /root/spdk/test/vhost/initiator/bdev.fio --filename=$virtio_bdevs --section=job_randwrite \ --section=job_randrw --section=job_write --section=job_rw \ --spdk_conf=/root/spdk/test/vhost/initiator/bdev_pci.conf --spdk_mem=1024" timing_exit run_spdk_fio_pci timing_enter run_spdk_fio_pci_unmap -vm_ssh $vm_num "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \ +vm_ssh $vm_no "LD_PRELOAD=/root/spdk/examples/bdev/fio_plugin/fio_plugin /root/fio_src/fio --ioengine=spdk_bdev \ /root/spdk/test/vhost/initiator/bdev.fio --filename=$virtio_with_unmap \ --spdk_conf=/root/spdk/test/vhost/initiator/bdev_pci.conf --spdk_mem=1024" timing_exit run_spdk_fio_pci_unmap diff --git a/test/vhost/migration/migration-malloc.job b/test/vhost/migration/migration-malloc.job new file mode 100644 index 000000000..5383b243f --- /dev/null +++ b/test/vhost/migration/migration-malloc.job @@ -0,0 +1,25 @@ +[global] +blocksize_range=4k-512k +#bs=512k +iodepth=128 +ioengine=libaio +filename= +group_reporting +thread +numjobs=1 +direct=1 +do_verify=1 +verify=md5 +verify_fatal=1 +verify_dump=1 +size=100% + +[write] +rw=write +stonewall + +[randread] +rw=randread +runtime=10 +time_based +stonewall diff --git a/test/vhost/migration/migration-malloc.sh b/test/vhost/migration/migration-malloc.sh new file mode 100755 index 000000000..bc55a3944 --- /dev/null +++ b/test/vhost/migration/migration-malloc.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash + +set -e + +vms=() +declare -A vms_os +declare -A vms_raw_disks +declare -A vms_ctrlrs +declare -A vms_ctrlrs_disks + +# By default use Guest fio +fio_bin="" + +function usage() +{ + [[ ! -z $2 ]] && ( echo "$2"; echo ""; ) + echo "Shortcut script for doing automated test of live migration." + echo "Usage: $(basename $1) [OPTIONS]" + echo + echo " --work-dir=WORK_DIR Where to find build file. Must exist. [default: $TEST_DIR]" + echo " --os ARGS VM configuration. This parameter might be used more than once:" + echo " --fio-bin=FIO Use specific fio binary (will be uploaded to VM)" + echo " --fio-job= Fio config to use for test." + echo " num=NUM - VM number" + echo " os=OS - VM os disk path" + echo " bdevs=DISKS - VM test disks/devices path separated by ':'" + echo " incoming - set this VM to wait for incoming migration" + echo " If test-type=spdk_vhost_blk then each disk size is 20G e.g." + echo " --vm num=X,os=os.qcow,bdevs=Malloc0:Nvme0n1:Malloc1" + echo "-x set -x for script debug" +} + +for param in "$@"; do + case "$param" in + --help|-h) + usage $0 + exit 0 + ;; + --work-dir=*) TEST_DIR="${param#*=}" ;; + --os=*) os_image="${param#*=}" ;; + --fio-bin=*) fio_bin="${param}" ;; + -x) set -x ;; + -v) SPDK_VHOST_VERBOSE=true ;; + *) + usage $0 "Invalid argument '$param'" + exit 1;; + esac +done + +. $(readlink -e "$(dirname $0)/../common/common.sh") || exit 1 + +job_file="$BASE_DIR/migration-malloc.job" + +trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT + +function vm_monitor_send() +{ + local vm_num=$1 + local cmd_result_file="$2" + local vm_dir="$VM_BASE_DIR/$1" + local vm_monitor_port=$(cat $vm_dir/monitor_port) + + [[ ! -z "$vm_monitor_port" ]] || fail "No monitor port!" + + shift 2 + nc 127.0.0.1 $vm_monitor_port "$@" > $cmd_result_file +} + +# Migrate VM $1 +function vm_migrate() +{ + local from_vm_dir="$VM_BASE_DIR/$1" + local target_vm_dir="$(readlink -e $from_vm_dir/vm_migrate_to)" + local target_vm="$(basename $target_vm_dir)" + local target_vm_migration_port="$(cat $target_vm_dir/migration_port)" + + # Sanity check if target VM (QEMU) is configured to accept source VM (QEMU) migration + if [[ "$(readlink -e ${target_vm_dir}/vm_incoming)" != "$(readlink -e ${from_vm_dir})" ]]; then + fail "source VM $1 or destination VM is not properly configured for live migration" + fi + + notice "Migrating VM $1 to VM "$(basename $target_vm_dir) + echo -e \ + "migrate_set_speed 1g\n" \ + "migrate tcp:127.0.0.1:$target_vm_migration_port\n" \ + "info migrate\n" \ + "quit" | vm_monitor_send $1 "$from_vm_dir/migration_result" + + # Post migration checks: + if ! grep "Migration status: completed" $from_vm_dir/migration_result -q; then + cat $from_vm_dir/migration_result + fail "Migration failed:\n" + fi + + if ! vm_os_booted $target_vm; then + fail "VM$target_vm is not running" + cat $target_vm $target_vm_dir/cont_result + fi + + notice "Migration complete" +} + +# FIXME: this shoul'd not be needed +vm_kill_all + +rpc="python $SPDK_BUILD_DIR/scripts/rpc.py " + +# Use 2 VMs: +# incoming VM - the one we want to migrate +# targe VM - the one which will accept migration +incoming_vm=0 +target_vm=1 +incoming_vm_ctrlr=naa.Malloc0.$incoming_vm +target_vm_ctrlr=naa.Malloc0.$target_vm + +vm_setup --os="$os_image" --force=$incoming_vm --disk-type=spdk_vhost_scsi --disks=Malloc0 --migrate-to=$target_vm +vm_setup --force=$target_vm --disk-type=spdk_vhost_scsi --disks=Malloc0 --incoming=$incoming_vm + +spdk_vhost_run $BASE_DIR + +notice "===============" +notice "" +notice "Setting up VMs" +notice "" + +declare -A vm_ctrlr + +function clean_vhost_config() +{ + notice "Removing vhost devices & controllers via RPC ..." + # Delete bdev first to remove all LUNs and SCSI targets + $rpc delete_bdev Malloc0 + + # Delete controllers + $rpc remove_vhost_controller $incoming_vm_ctrlr + $rpc remove_vhost_controller $target_vm_ctrlr +} + +function error_migration_clean() +{ + trap - SIGINT ERR EXIT + set -x + + vm_kill_all + clean_vhost_config +} + +function is_fio_running() +{ + local shell_restore_x="$( [[ "$-" =~ x ]] && echo 'set -x' )" + set +x + + if vm_ssh $1 'kill -0 $(cat /root/fio.pid)'; then + local ret=0 + else + local ret=1 + fi + + $shell_restore_x + return $ret +} + +trap 'error_migration_clean; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT + +# Construct shared Malloc Bdev +$rpc construct_malloc_bdev -b Malloc0 128 4096 + +# And two controllers - one for each VM. Both are using the same Malloc Bdev as LUN 0 +$rpc construct_vhost_scsi_controller $incoming_vm_ctrlr +$rpc add_vhost_scsi_lun $incoming_vm_ctrlr 0 Malloc0 + +$rpc construct_vhost_scsi_controller $target_vm_ctrlr +$rpc add_vhost_scsi_lun $target_vm_ctrlr 0 Malloc0 + +# Run everything +vm_run $incoming_vm $target_vm + +# Wait only for incoming VM, as target is waiting for migration +vm_wait_for_boot 600 $incoming_vm + +# Run fio before migration +notice "Starting FIO" + +vm_check_scsi_location $incoming_vm +run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)" + +# Wait a while to let the FIO time to issue some IO +sleep 5 + +# Check if fio is still running before migration +if ! is_fio_running $incoming_vm; then + vm_ssh $incoming_vm "cat /root/$(basename ${job_file}).out" + error "FIO is not running before migration: process crashed or finished too early" +fi + +vm_migrate $incoming_vm +sleep 3 + +# Check if fio is still running after migration +if ! is_fio_running $target_vm; then + vm_ssh $target_vm "cat /root/$(basename ${job_file}).out" + error "FIO is not running after migration: process crashed or finished too early" +fi + +notice "Waiting for fio to finish" +timeout=40 +while is_fio_running $target_vm; do + sleep 1 + echo -n "." + if (( timeout-- == 0 )); then + error "timeout while waiting for FIO!" + fi +done + +notice "Fio result is:" +vm_ssh $target_vm "cat /root/$(basename ${job_file}).out" + +notice "Migration DONE" + + +notice "Shutting down all VMs" +vm_shutdown_all +clean_vhost_config + +notice "killing vhost app" +spdk_vhost_kill + +notice "Migration Test SUCCESS" +notice "===============" + +trap - SIGINT ERR EXIT diff --git a/test/vhost/migration/vhost.conf.in b/test/vhost/migration/vhost.conf.in new file mode 100644 index 000000000..967a08766 --- /dev/null +++ b/test/vhost/migration/vhost.conf.in @@ -0,0 +1,5 @@ +[Global] + NoPci Yes + +[Ioat] + Disable Yes diff --git a/test/vhost/spdk_vhost.sh b/test/vhost/spdk_vhost.sh index da747297b..f4267e88f 100755 --- a/test/vhost/spdk_vhost.sh +++ b/test/vhost/spdk_vhost.sh @@ -74,6 +74,11 @@ case $1 in --test-type=spdk_vhost_blk \ --fio-job=$WORKDIR/common/fio_jobs/default_performance.job ;; + -m|--migration) + echo 'Running migration suite...' + ./migration/migration-malloc.sh -x \ + --fio-bin=$FIO_BIN --os=$VM_IMAGE + ;; -i|--integrity) echo 'Running SCSI integrity suite...' ./fiotest/autotest.sh -x --fio-bin=$FIO_BIN \