diff --git a/test/vhost/fiotest/autotest.sh b/test/vhost/fiotest/autotest.sh index 45fdac33b..a62315a66 100755 --- a/test/vhost/fiotest/autotest.sh +++ b/test/vhost/fiotest/autotest.sh @@ -218,7 +218,7 @@ for vm_num in $used_vms; do run_fio+="127.0.0.1:$(cat $vm_dir/fio_socket):" for disk in $SCSI_DISK; do - run_fio+="$disk:" + run_fio+="/dev/$disk:" done run_fio="${run_fio::-1}" run_fio+="," diff --git a/test/vhost/fiotest/run_fio.py b/test/vhost/fiotest/run_fio.py index 1a235570e..d70397c79 100755 --- a/test/vhost/fiotest/run_fio.py +++ b/test/vhost/fiotest/run_fio.py @@ -4,55 +4,25 @@ import os import sys import getopt import subprocess -import itertools -import datetime import signal import re fio_bin = "fio" -perf_vmex = False - -fio_template = """ -[global] -ioengine=%(ioengine)s -size=%(size)s -filename=%(filename)s -numjobs=%(numjobs)s -bs=%(blocksize)s -iodepth=%(iodepth)s -direct=%(direct)s -rw=%(testtype)s -group_reporting -thread -%(verify)s - -[nvme-host] -""" -def show_help(fio_args_dict): +def show_help(): print("""Usage: python run_fio.py [options] [args] Args: - [VMs] (ex. vm1_IP:vm1_port,vm2_IP:vm2_port,etc...) - [fio filename arg], ex. /dev/sda) + [VMs] (ex. vm1_IP:vm1_port:vm1_disk1:vm_disk2,vm2_IP:vm2_port:vm2_disk1,etc...) Options: -h, --help Show this message. -j, --job-files Paths to files with custom FIO jobs configuration. - -F, --fio-bin Location of FIO binary (Default "fio") - -s, --size Size of IO for job. Will be distributed among - number of numjobs (Default: %(size)s) - -t, --testtype Type of FIO test (Default: %(testtype)s) - -b, --blocksize Blocksize for FIO test (Default: %(blocksize)s) - -i, --iodepth IO depth for FIO test (Default: %(iodepth)s) - -I, --ioengine Type of FIO ioengine to use (Default: %(ioengine)s) - -n, --numjobs Number of threads for job (Default: %(numjobs)s) - -D, --direct Use non-buffered IO? (Default: %(direct)s) - -v, --verify Verify after writing to file (Default: %(verify)s) + -f, --fio-bin Location of FIO binary (Default "fio") -o, --out Directory used to save generated job files and files with test results (Default: same dir where this script is located) - -p, --perf-vmex Enable aggregating statistic for VMEXITS - """ % fio_args_dict) + -p, --perf-vmex Enable aggregating statistic for VMEXITS for VMs + """) def exec_cmd(cmd, blocking): @@ -71,242 +41,140 @@ def save_file(path, mode, contents): fh.close() -def prep_fio_cfg_file(out_dir, fio_cfg, vm_nb): - job_file = os.path.join(out_dir, "fio_job_vm{0}".format(vm_nb)) - print "file {0} written".format(job_file) - save_file(job_file, "w", fio_cfg) - return job_file - - -def calc_size(size, numjobs): - return str(int(filter(lambda x: x.isdigit(), size)) / int(numjobs)) + \ - filter(lambda x: x.isalpha(), size) - - -def cfg_product(fio_args_dict): - return (dict(zip(fio_args_dict, x)) for - x in itertools.product(*fio_args_dict.itervalues())) - - -def run_fio(vms, fio_cfg_file, out_path): - - global perf_vmex - # Prepare command template for FIO - fio_cmd = fio_bin - fio_cmd = " ".join([fio_cmd, "--eta=never"]) - print fio_cfg_file - fio_cfg_name = (os.path.basename(fio_cfg_file)).split(".")[0] - for i, vm in enumerate(vms): - print("Starting thread {0} for VM: {1}".format(i, vm)) +def run_fio(vms, fio_cfg_fname, out_path, perf_vmex=False): + global fio_bin + fio_cfg_prefix = fio_cfg_fname.split(".")[0] + # Build command for FIO + fio_cmd = " ".join([fio_bin, "--eta=never"]) + for vm in vms: # vm[0] = IP address, vm[1] = Port number fio_cmd = " ".join([fio_cmd, - "--client={0},{1}".format(vm[0], vm[1])]) - fio_cmd = " ".join([fio_cmd, - "--remote-config /root/fio.job{0}".format(i)]) - - print fio_cmd + "--client={vm_ip},{vm_port}".format(vm_ip=vm[0], vm_port=vm[1]), + "--remote-config /root/{cfg}".format(cfg=fio_cfg_fname)]) + print(fio_cmd) if perf_vmex: + perf_dir = os.path.join(out_path, "perf_stats") + try: + os.mkdir(perf_dir) + except OSError: + pass + # Start gathering perf statistics for host and VM guests - perf_rec_file = os.path.join(out_path, "perf.data.kvm") + perf_rec_file = os.path.join(perf_dir, "perf.data.kvm") perf_run_cmd = "perf kvm --host --guest " + \ "-o {0} stat record -a".format(perf_rec_file) - print perf_run_cmd + print(perf_run_cmd) perf_p = exec_cmd(perf_run_cmd, blocking=False) # Run FIO test on VMs rc, out = exec_cmd(fio_cmd, blocking=True) # if for some reason output contains lines with "eta" - remove them - out = re.sub(r'.+\[eta\s+\d{2}m\:\d{2}s\]', '', out) + out = re.sub(r'.+\[eta\s+\d{2}m:\d{2}s\]', '', out) if rc != 0: - print(rc, out) - return rc + print("ERROR! While executing FIO jobs - RC: {rc}, Err message: {out}".format(rc=rc, out=out)) + sys.exit(rc) else: - print out - save_file(os.path.join(out_path, "".join([fio_cfg_name, ".log"])), "w", out) - # out = out[out.find("Disk"):] - # out = out[out.find(":")+2:] - # JSON format nos supported on Debian IMG for now, not parsing - # data = json.loads(out) - # pprint(data) - pass + print(out) + save_file(os.path.join(out_path, ".".join([fio_cfg_prefix, "log"])), "w", out) if perf_vmex: # Stop gathering perf statistics and prepare some result files perf_p.send_signal(signal.SIGINT) perf_p.wait() - perf_stat_cmd = "perf kvm --host " + \ - "-i {0} stat report".format(perf_rec_file) + perf_stat_cmd = "perf kvm --host -i {perf_rec} stat report --event vmexit"\ + .format(perf_rec=perf_rec_file) - print(" ".join([perf_stat_cmd, "--event vmexit"])) rc, out = exec_cmd(" ".join([perf_stat_cmd, "--event vmexit"]), blocking=True) - print("VMexit host stats:") - print("{0}".format(out)) - save_file(os.path.join(out_path, "vmexit_stats"), - "w", "{0}".format(out)) + print("{perf_out}".format(perf_out=out)) + save_file(os.path.join(perf_dir, "vmexit_stats_" + fio_cfg_prefix), + "w", "{perf_out}".format(perf_out=out)) + try: + os.remove(perf_rec_file) + except OSError: + pass def main(): + global fio_bin abspath = os.path.abspath(__file__) dname = os.path.dirname(abspath) os.chdir(os.path.join(dname, "../../..")) - global fio_bin - global perf_vmex - job_file_opt = False vms = [] - split_disks = [] - filenames = "" + fio_cfgs = [] + perf_vmex = False out_dir = os.path.join(os.getcwd(), "fio_results") - fio_cfg_files = [] - rc = 0 - - fio_args_def = { - 'size': ["10G"], - 'numjobs': ["1"], - 'testtype': ["randread"], - 'blocksize': ["4k"], - 'iodepth': ["128"], - 'ioengine': ["libaio"], - 'direct': ["1"], - 'verify': [""] - } - - fio_args = fio_args_def.copy() try: - opts, args = getopt.getopt(sys.argv[1:], "hj:t:b:i:D:n:F:I:s:v:o:S:p", - ["help", "job-file=", "testtype=", - "blocksize=", "iodepth=", "direct=", - "numjobs=", "fio-bin=", "ioengine=", - "size=", "verify=", "out=", - "split-disks=", "perf"]) - except: - show_help(fio_args_def) + opts, args = getopt.getopt(sys.argv[1:], "hj:f:o:p", + ["help", "job-file=", "fio-bin=", + "out=", "perf-vmex"]) + except getopt.GetoptError: + show_help() sys.exit(1) for o, a in opts: - print o, a if o in ("-j", "--job-file"): - fio_cfg_files = a.split(",") - job_file_opt = True - fio_args = fio_args_def.copy() + fio_cfgs = a.split(",") elif o in ("-h", "--help"): - show_help(fio_args_def) + show_help() sys.exit(1) elif o in ("-p", "--perf-vmex"): perf_vmex = True elif o in ("-o", "--out"): out_dir = os.path.join(a, "fio_results") - elif o in ("-F", "--fio-bin"): + elif o in ("-f", "--fio-bin"): fio_bin = a - elif o in ("-S", "--split-disks"): - split_disks = [x.split("-") for x in a.split(",")] - split_disks = [[int(x) - 1 for x in y] for y in split_disks] - print split_disks - elif o in ("-s", "--size"): - fio_args["size"] = a.split(",") - elif o in ("-t", "--testtype"): - fio_args["testtype"] = a.split(",") - elif o in ("-b", "--blocksize"): - fio_args["blocksize"] = a.split(",") - elif o in ("-i", "--iodepth"): - fio_args["iodepth"] = a.split(",") - elif o in ("-D", "--direct"): - fio_args["direct"] = a.split(",") - elif o in ("-n", "--numjobs"): - fio_args["numjobs"] = a.split(",") - elif o in ("-I", "--ioengine"): - fio_args["ioengine"] = a.split(",") - elif o in ("-v", "--verify"): - fio_args["verify"] = a.split(",") - fio_args["verify"] = ["" if x in "0" else - "verify=crc32" for x in fio_args["verify"]] + + if len(fio_cfgs) < 1: + print("ERROR! No FIO jobs provided!") + sys.exit(1) if len(args) < 1: - show_help(fio_args_def) + show_help() sys.exit(1) else: - # Get IP, Port tuples from args and filename for fio config - vms = [tuple(x.split(":")) for x in args[0].split(",")] - filenames = [["/dev/" + y for y in x[2:]] for x in vms] - vms = [x[0:2] for x in vms] + # Get IP, port and fio 'filename' information from positional args + for arg in args[0].split(","): + _ = arg.split(":") + ip, port, filenames = _[0], _[1], ":".join(_[2:]) + vms.append((ip, port, filenames)) if not os.path.exists(out_dir): os.mkdir(out_dir) - if job_file_opt is True: - for fio_cfg in fio_cfg_files: - print("Running job file: {0}".format(fio_cfg)) + for fio_cfg in fio_cfgs: + fio_cfg_fname = os.path.basename(fio_cfg) + print("Running job file: {0}".format(fio_cfg_fname)) - for i, vm in enumerate(zip(vms, filenames)): - fnames = vm[1] - if split_disks: - if len(split_disks[i]) < 2: - filename = fnames[split_disks[i][0]:split_disks[i][0] + 1] - filename = ":".join(filename) - else: - filename = fnames[split_disks[i][0]:split_disks[i][1] + 1] - filename = ":".join(filename) - else: - filename = ":".join(fnames) + for i, vm in enumerate(vms): + # VM - tuple of IP / Port / Filename for VM to run test + print("Preparing VM {0} - {1} for FIO job".format(i, vm[0])) - a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " + - "{0} sh -c 'rm fio.job{1}'" - .format(i, i), blocking=True) + exec_cmd("./test/vhost/fiotest/vm_ssh.sh {vm_num} sh -c 'rm {cfg}'" + .format(vm_num=i, cfg=fio_cfg_fname), blocking=True) - for cfg in fio_cfg.split("\n"): - with open(cfg, "r") as fh: - lines = fh.readlines() - for line in lines: - if "filename" in line: - line = "filename=" + filename - a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " + - "{0} sh -c 'echo {1} >> fio.job{2}'" - .format(i, line.strip(), i), blocking=True) - fh.close() - rc = run_fio(vms, fio_cfg, out_dir) - else: - for cfg in cfg_product(fio_args): - # Update fio "size" parameter so that total work done by - # all numjobs is equal to assigned size and not size*numjobs - cfg["size"] = calc_size(cfg["size"], cfg["numjobs"]) + # Copy FIO config to VM + with open(fio_cfg, "r") as fio_cfg_fh: + for line in fio_cfg_fh.readlines(): + if "filename" in line: + line = "filename=" + vm[2] + out = exec_cmd("./test/vhost/fiotest/vm_ssh.sh {vm_num} sh -c 'echo {line} >> {cfg}'" + .format(vm_num=i, line=line.strip(), cfg=fio_cfg_fname), blocking=True) + if out[0] != 0: + print("ERROR! While copying FIO job config file to VM {vm_num} - {vm_ip}" + .format(vm_num=1, vm_ip=vm[0])) + sys.exit(1) - # Prepare this test run FIO job file - for i, vm in enumerate(zip(vms, filenames)): - fnames = vm[1] - if split_disks: - if len(split_disks[i]) < 2: - filename = fnames[split_disks[i][0]:split_disks[i][0] + 1] - filename = ":".join(filename) - else: - filename = fnames[split_disks[i][0]:split_disks[i][1] + 1] - filename = ":".join(filename) - else: - filename = ":".join(fnames) - - cfg.update({"filename": filename}) - fio_cfg = fio_template % cfg - fio_cfg_files.append(prep_fio_cfg_file(out_dir, - fio_cfg, i)) - a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " + - "{0} sh -c 'rm fio.job{1}'" - .format(i, i), blocking=True) - for line in fio_cfg.split("\n"): - a = exec_cmd("./test/vhost/fiotest/vm_ssh.sh " + - "{0} sh -c 'echo {1} >> fio.job{2}'" - .format(i, line.strip(), i), blocking=True) - - rc = run_fio(vms, cfg, out_dir) - - return rc + run_fio(vms, fio_cfg_fname, out_dir, perf_vmex) if __name__ == "__main__": sys.exit(main())