huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-25 12:02:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Fix to allow report for a full failed test

Browse Source
This commit is contained in:
Hugo Larcher 2024-07-17 16:44:52 +02:00
parent 4642fd27ad
commit 46775b1c03
No known key found for this signature in database
GPG Key ID: 3DAF63124699CA2B
1 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
load_tests/parse_load_test.py
View File

@ -1,5 +1,6 @@
import json
import os
import re
from enum import Enum
import pandas as pd
from loguru import logger
@ -15,6 +16,20 @@ class TestType(Enum):
CONSTANT_ARRIVAL_RATE = "constant_arrival_rate"
def get_nested(obj, path, default=None):
for key in path.split("."):
if obj is None:
return default
if re.match(r"\d+", key):
key = int(key)
if key >= len(obj):
return default
obj = obj[key]
else:
obj = obj.get(key, default)
return obj
def parse_json_files(directory: str, test_type: TestType) -> pd.DataFrame:
metrics_to_keep = {'inter_token_latency': {'y': 'Time (ms)'}, 'end_to_end_latency': {'y': 'Time (ms)'},
'time_to_first_token': {'y': 'Time (ms)'}, 'tokens_throughput': {'y': 'Tokens/s'},
@ -38,10 +53,9 @@ def parse_json_files(directory: str, test_type: TestType) -> pd.DataFrame:
}
entry['input_type'] = data['k6_config']['input_type']
entry['test_duration'] = data['state']['testRunDurationMs'] / 1000.
entry['requests_ok'] = data['root_group']['checks'][0]['passes']
entry['requests_fail'] = data['root_group']['checks'][0]['fails']
entry['dropped_iterations'] = data['metrics']['dropped_iterations']['values'][
'count'] if 'dropped_iterations' in data['metrics'] else 0
entry['requests_ok'] = get_nested(data, 'root_group.checks.0.passes', 0)
entry['requests_fail'] = get_nested(data, 'root_group.checks.0.fails', 0)
entry['dropped_iterations'] = get_nested(data, 'metrics.dropped_iterations.values.count', 0)
# add up requests_fail and dropped_iterations to get total dropped requests
entry['dropped_requests'] = entry['requests_fail'] + entry['dropped_iterations']
entry['error_rate'] = entry['dropped_requests'] / (
@ -60,7 +74,7 @@ def parse_json_files(directory: str, test_type: TestType) -> pd.DataFrame:
return df
def plot_metrics(model_name:str, df: pd.DataFrame, test_type: TestType, save_name: str):
def plot_metrics(model_name: str, df: pd.DataFrame, test_type: TestType, save_name: str):
vus_param = ''
if test_type == TestType.CONSTANT_VUS:
vus_param = 'vus'