diff --git a/scripts/perf/nvmf/common.py b/scripts/perf/nvmf/common.py index a233df9e3..bbe528829 100644 --- a/scripts/perf/nvmf/common.py +++ b/scripts/perf/nvmf/common.py @@ -1,4 +1,10 @@ +import os +import re +import json +import logging from subprocess import check_output +from collections import OrderedDict +from json.decoder import JSONDecodeError def get_nvme_devices_count(): @@ -23,3 +29,173 @@ def get_nvme_devices(): output = [x for x in output.split("\n") if "nvme" in x] print("Done getting kernel NVMe names") return output + + +def read_json_stats(file): + with open(file, "r") as json_data: + data = json.load(json_data) + job_pos = 0 # job_post = 0 because using aggregated results + + # Check if latency is in nano or microseconds to choose correct dict key + def get_lat_unit(key_prefix, dict_section): + # key prefix - lat, clat or slat. + # dict section - portion of json containing latency bucket in question + # Return dict key to access the bucket and unit as string + for k, _ in dict_section.items(): + if k.startswith(key_prefix): + return k, k.split("_")[1] + + def get_clat_percentiles(clat_dict_leaf): + if "percentile" in clat_dict_leaf: + p99_lat = float(clat_dict_leaf["percentile"]["99.000000"]) + p99_9_lat = float(clat_dict_leaf["percentile"]["99.900000"]) + p99_99_lat = float(clat_dict_leaf["percentile"]["99.990000"]) + p99_999_lat = float(clat_dict_leaf["percentile"]["99.999000"]) + + return [p99_lat, p99_9_lat, p99_99_lat, p99_999_lat] + else: + # Latest fio versions do not provide "percentile" results if no + # measurements were done, so just return zeroes + return [0, 0, 0, 0] + + read_iops = float(data["jobs"][job_pos]["read"]["iops"]) + read_bw = float(data["jobs"][job_pos]["read"]["bw"]) + lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["read"]) + read_avg_lat = float(data["jobs"][job_pos]["read"][lat_key]["mean"]) + read_min_lat = float(data["jobs"][job_pos]["read"][lat_key]["min"]) + read_max_lat = float(data["jobs"][job_pos]["read"][lat_key]["max"]) + clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["read"]) + read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat = get_clat_percentiles( + data["jobs"][job_pos]["read"][clat_key]) + + if "ns" in lat_unit: + read_avg_lat, read_min_lat, read_max_lat = [x / 1000 for x in [read_avg_lat, read_min_lat, read_max_lat]] + if "ns" in clat_unit: + read_p99_lat = read_p99_lat / 1000 + read_p99_9_lat = read_p99_9_lat / 1000 + read_p99_99_lat = read_p99_99_lat / 1000 + read_p99_999_lat = read_p99_999_lat / 1000 + + write_iops = float(data["jobs"][job_pos]["write"]["iops"]) + write_bw = float(data["jobs"][job_pos]["write"]["bw"]) + lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["write"]) + write_avg_lat = float(data["jobs"][job_pos]["write"][lat_key]["mean"]) + write_min_lat = float(data["jobs"][job_pos]["write"][lat_key]["min"]) + write_max_lat = float(data["jobs"][job_pos]["write"][lat_key]["max"]) + clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["write"]) + write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat = get_clat_percentiles( + data["jobs"][job_pos]["write"][clat_key]) + + if "ns" in lat_unit: + write_avg_lat, write_min_lat, write_max_lat = [x / 1000 for x in [write_avg_lat, write_min_lat, write_max_lat]] + if "ns" in clat_unit: + write_p99_lat = write_p99_lat / 1000 + write_p99_9_lat = write_p99_9_lat / 1000 + write_p99_99_lat = write_p99_99_lat / 1000 + write_p99_999_lat = write_p99_999_lat / 1000 + + return [read_iops, read_bw, read_avg_lat, read_min_lat, read_max_lat, + read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat, + write_iops, write_bw, write_avg_lat, write_min_lat, write_max_lat, + write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat] + + +def parse_results(results_dir, csv_file): + files = os.listdir(results_dir) + fio_files = filter(lambda x: ".fio" in x, files) + json_files = [x for x in files if ".json" in x] + + headers = ["read_iops", "read_bw", "read_avg_lat_us", "read_min_lat_us", "read_max_lat_us", + "read_p99_lat_us", "read_p99.9_lat_us", "read_p99.99_lat_us", "read_p99.999_lat_us", + "write_iops", "write_bw", "write_avg_lat_us", "write_min_lat_us", "write_max_lat_us", + "write_p99_lat_us", "write_p99.9_lat_us", "write_p99.99_lat_us", "write_p99.999_lat_us"] + + aggr_headers = ["iops", "bw", "avg_lat_us", "min_lat_us", "max_lat_us", + "p99_lat_us", "p99.9_lat_us", "p99.99_lat_us", "p99.999_lat_us"] + + header_line = ",".join(["Name", *headers]) + aggr_header_line = ",".join(["Name", *aggr_headers]) + + # Create empty results file + with open(os.path.join(results_dir, csv_file), "w") as fh: + fh.write(aggr_header_line + "\n") + rows = set() + + for fio_config in fio_files: + logging.info("Getting FIO stats for %s" % fio_config) + job_name, _ = os.path.splitext(fio_config) + + # Look in the filename for rwmixread value. Function arguments do + # not have that information. + # TODO: Improve this function by directly using workload params instead + # of regexing through filenames. + if "read" in job_name: + rw_mixread = 1 + elif "write" in job_name: + rw_mixread = 0 + else: + rw_mixread = float(re.search(r"m_(\d+)", job_name).group(1)) / 100 + + # If "_CPU" exists in name - ignore it + # Initiators for the same job could have different num_cores parameter + job_name = re.sub(r"_\d+CPU", "", job_name) + job_result_files = [x for x in json_files if x.startswith(job_name)] + logging.info("Matching result files for current fio config:") + for j in job_result_files: + logging.info("\t %s" % j) + + # There may have been more than 1 initiator used in test, need to check that + # Result files are created so that string after last "_" separator is server name + inits_names = set([os.path.splitext(x)[0].split("_")[-1] for x in job_result_files]) + inits_avg_results = [] + for i in inits_names: + logging.info("\tGetting stats for initiator %s" % i) + # There may have been more than 1 test run for this job, calculate average results for initiator + i_results = [x for x in job_result_files if i in x] + i_results_filename = re.sub(r"run_\d+_", "", i_results[0].replace("json", "csv")) + + separate_stats = [] + for r in i_results: + try: + stats = read_json_stats(os.path.join(results_dir, r)) + separate_stats.append(stats) + logging.info(stats) + except JSONDecodeError: + logging.error("ERROR: Failed to parse %s results! Results might be incomplete!" % r) + + init_results = [sum(x) for x in zip(*separate_stats)] + init_results = [x / len(separate_stats) for x in init_results] + inits_avg_results.append(init_results) + + logging.info("\tAverage results for initiator %s" % i) + logging.info(init_results) + with open(os.path.join(results_dir, i_results_filename), "w") as fh: + fh.write(header_line + "\n") + fh.write(",".join([job_name, *["{0:.3f}".format(x) for x in init_results]]) + "\n") + + # Sum results of all initiators running this FIO job. + # Latency results are an average of latencies from accros all initiators. + inits_avg_results = [sum(x) for x in zip(*inits_avg_results)] + inits_avg_results = OrderedDict(zip(headers, inits_avg_results)) + for key in inits_avg_results: + if "lat" in key: + inits_avg_results[key] /= len(inits_names) + + # Aggregate separate read/write values into common labels + # Take rw_mixread into consideration for mixed read/write workloads. + aggregate_results = OrderedDict() + for h in aggr_headers: + read_stat, write_stat = [float(value) for key, value in inits_avg_results.items() if h in key] + if "lat" in h: + _ = rw_mixread * read_stat + (1 - rw_mixread) * write_stat + else: + _ = read_stat + write_stat + aggregate_results[h] = "{0:.3f}".format(_) + + rows.add(",".join([job_name, *aggregate_results.values()])) + + # Save results to file + for row in rows: + with open(os.path.join(results_dir, csv_file), "a") as fh: + fh.write(row + "\n") + logging.info("You can find the test results in the file %s" % os.path.join(results_dir, csv_file)) diff --git a/scripts/perf/nvmf/run_nvmf.py b/scripts/perf/nvmf/run_nvmf.py index c4f02d553..9521b1f1f 100755 --- a/scripts/perf/nvmf/run_nvmf.py +++ b/scripts/perf/nvmf/run_nvmf.py @@ -4,7 +4,6 @@ # All rights reserved. # -from json.decoder import JSONDecodeError import os import re import sys @@ -18,7 +17,6 @@ import itertools import configparser import time import uuid -from collections import OrderedDict import paramiko import pandas as pd @@ -476,175 +474,6 @@ class Target(Server): ip_bdev_map.append((ip, c)) return ip_bdev_map - @staticmethod - def read_json_stats(file): - with open(file, "r") as json_data: - data = json.load(json_data) - job_pos = 0 # job_post = 0 because using aggregated results - - # Check if latency is in nano or microseconds to choose correct dict key - def get_lat_unit(key_prefix, dict_section): - # key prefix - lat, clat or slat. - # dict section - portion of json containing latency bucket in question - # Return dict key to access the bucket and unit as string - for k, _ in dict_section.items(): - if k.startswith(key_prefix): - return k, k.split("_")[1] - - def get_clat_percentiles(clat_dict_leaf): - if "percentile" in clat_dict_leaf: - p99_lat = float(clat_dict_leaf["percentile"]["99.000000"]) - p99_9_lat = float(clat_dict_leaf["percentile"]["99.900000"]) - p99_99_lat = float(clat_dict_leaf["percentile"]["99.990000"]) - p99_999_lat = float(clat_dict_leaf["percentile"]["99.999000"]) - - return [p99_lat, p99_9_lat, p99_99_lat, p99_999_lat] - else: - # Latest fio versions do not provide "percentile" results if no - # measurements were done, so just return zeroes - return [0, 0, 0, 0] - - read_iops = float(data["jobs"][job_pos]["read"]["iops"]) - read_bw = float(data["jobs"][job_pos]["read"]["bw"]) - lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["read"]) - read_avg_lat = float(data["jobs"][job_pos]["read"][lat_key]["mean"]) - read_min_lat = float(data["jobs"][job_pos]["read"][lat_key]["min"]) - read_max_lat = float(data["jobs"][job_pos]["read"][lat_key]["max"]) - clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["read"]) - read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat = get_clat_percentiles( - data["jobs"][job_pos]["read"][clat_key]) - - if "ns" in lat_unit: - read_avg_lat, read_min_lat, read_max_lat = [x / 1000 for x in [read_avg_lat, read_min_lat, read_max_lat]] - if "ns" in clat_unit: - read_p99_lat = read_p99_lat / 1000 - read_p99_9_lat = read_p99_9_lat / 1000 - read_p99_99_lat = read_p99_99_lat / 1000 - read_p99_999_lat = read_p99_999_lat / 1000 - - write_iops = float(data["jobs"][job_pos]["write"]["iops"]) - write_bw = float(data["jobs"][job_pos]["write"]["bw"]) - lat_key, lat_unit = get_lat_unit("lat", data["jobs"][job_pos]["write"]) - write_avg_lat = float(data["jobs"][job_pos]["write"][lat_key]["mean"]) - write_min_lat = float(data["jobs"][job_pos]["write"][lat_key]["min"]) - write_max_lat = float(data["jobs"][job_pos]["write"][lat_key]["max"]) - clat_key, clat_unit = get_lat_unit("clat", data["jobs"][job_pos]["write"]) - write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat = get_clat_percentiles( - data["jobs"][job_pos]["write"][clat_key]) - - if "ns" in lat_unit: - write_avg_lat, write_min_lat, write_max_lat = [x / 1000 for x in [write_avg_lat, write_min_lat, write_max_lat]] - if "ns" in clat_unit: - write_p99_lat = write_p99_lat / 1000 - write_p99_9_lat = write_p99_9_lat / 1000 - write_p99_99_lat = write_p99_99_lat / 1000 - write_p99_999_lat = write_p99_999_lat / 1000 - - return [read_iops, read_bw, read_avg_lat, read_min_lat, read_max_lat, - read_p99_lat, read_p99_9_lat, read_p99_99_lat, read_p99_999_lat, - write_iops, write_bw, write_avg_lat, write_min_lat, write_max_lat, - write_p99_lat, write_p99_9_lat, write_p99_99_lat, write_p99_999_lat] - - def parse_results(self, results_dir, csv_file): - files = os.listdir(results_dir) - fio_files = filter(lambda x: ".fio" in x, files) - json_files = [x for x in files if ".json" in x] - - headers = ["read_iops", "read_bw", "read_avg_lat_us", "read_min_lat_us", "read_max_lat_us", - "read_p99_lat_us", "read_p99.9_lat_us", "read_p99.99_lat_us", "read_p99.999_lat_us", - "write_iops", "write_bw", "write_avg_lat_us", "write_min_lat_us", "write_max_lat_us", - "write_p99_lat_us", "write_p99.9_lat_us", "write_p99.99_lat_us", "write_p99.999_lat_us"] - - aggr_headers = ["iops", "bw", "avg_lat_us", "min_lat_us", "max_lat_us", - "p99_lat_us", "p99.9_lat_us", "p99.99_lat_us", "p99.999_lat_us"] - - header_line = ",".join(["Name", *headers]) - aggr_header_line = ",".join(["Name", *aggr_headers]) - - # Create empty results file - with open(os.path.join(results_dir, csv_file), "w") as fh: - fh.write(aggr_header_line + "\n") - rows = set() - - for fio_config in fio_files: - self.log.info("Getting FIO stats for %s" % fio_config) - job_name, _ = os.path.splitext(fio_config) - - # Look in the filename for rwmixread value. Function arguments do - # not have that information. - # TODO: Improve this function by directly using workload params instead - # of regexing through filenames. - if "read" in job_name: - rw_mixread = 1 - elif "write" in job_name: - rw_mixread = 0 - else: - rw_mixread = float(re.search(r"m_(\d+)", job_name).group(1)) / 100 - - # If "_CPU" exists in name - ignore it - # Initiators for the same job could have different num_cores parameter - job_name = re.sub(r"_\d+CPU", "", job_name) - job_result_files = [x for x in json_files if x.startswith(job_name)] - self.log.info("Matching result files for current fio config:") - for j in job_result_files: - self.log.info("\t %s" % j) - - # There may have been more than 1 initiator used in test, need to check that - # Result files are created so that string after last "_" separator is server name - inits_names = set([os.path.splitext(x)[0].split("_")[-1] for x in job_result_files]) - inits_avg_results = [] - for i in inits_names: - self.log.info("\tGetting stats for initiator %s" % i) - # There may have been more than 1 test run for this job, calculate average results for initiator - i_results = [x for x in job_result_files if i in x] - i_results_filename = re.sub(r"run_\d+_", "", i_results[0].replace("json", "csv")) - - separate_stats = [] - for r in i_results: - try: - stats = self.read_json_stats(os.path.join(results_dir, r)) - separate_stats.append(stats) - self.log.info(stats) - except JSONDecodeError: - self.log.error("ERROR: Failed to parse %s results! Results might be incomplete!" % r) - - init_results = [sum(x) for x in zip(*separate_stats)] - init_results = [x / len(separate_stats) for x in init_results] - inits_avg_results.append(init_results) - - self.log.info("\tAverage results for initiator %s" % i) - self.log.info(init_results) - with open(os.path.join(results_dir, i_results_filename), "w") as fh: - fh.write(header_line + "\n") - fh.write(",".join([job_name, *["{0:.3f}".format(x) for x in init_results]]) + "\n") - - # Sum results of all initiators running this FIO job. - # Latency results are an average of latencies from accros all initiators. - inits_avg_results = [sum(x) for x in zip(*inits_avg_results)] - inits_avg_results = OrderedDict(zip(headers, inits_avg_results)) - for key in inits_avg_results: - if "lat" in key: - inits_avg_results[key] /= len(inits_names) - - # Aggregate separate read/write values into common labels - # Take rw_mixread into consideration for mixed read/write workloads. - aggregate_results = OrderedDict() - for h in aggr_headers: - read_stat, write_stat = [float(value) for key, value in inits_avg_results.items() if h in key] - if "lat" in h: - _ = rw_mixread * read_stat + (1 - rw_mixread) * write_stat - else: - _ = read_stat + write_stat - aggregate_results[h] = "{0:.3f}".format(_) - - rows.add(",".join([job_name, *aggregate_results.values()])) - - # Save results to file - for row in rows: - with open(os.path.join(results_dir, csv_file), "a") as fh: - fh.write(row + "\n") - self.log.info("You can find the test results in the file %s" % os.path.join(results_dir, csv_file)) - def measure_sar(self, results_dir, sar_file_prefix): cpu_number = os.cpu_count() sar_idle_sum = 0 @@ -1717,7 +1546,7 @@ if __name__ == "__main__": i.restore_sysctl() if i.enable_adq: i.reload_driver("ice") - target_obj.parse_results(args.results, args.csv_filename) + parse_results(args.results, args.csv_filename) finally: for i in initiators: try: