huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-26 20:42:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Fix to allow report for a full failed test

Browse Source
This commit is contained in:
Hugo Larcher 2024-07-17 16:44:52 +02:00
parent 4642fd27ad
commit 46775b1c03
No known key found for this signature in database
GPG Key ID: 3DAF63124699CA2B
1 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
load_tests/parse_load_test.py
View File

@ -1,5 +1,6 @@
import json import json
import os import os
import re
from enum import Enum from enum import Enum
import pandas as pd import pandas as pd
from loguru import logger from loguru import logger
@ -15,6 +16,20 @@ class TestType(Enum):
CONSTANT_ARRIVAL_RATE = "constant_arrival_rate" CONSTANT_ARRIVAL_RATE = "constant_arrival_rate"
def get_nested(obj, path, default=None):
for key in path.split("."):
if obj is None:
return default
if re.match(r"\d+", key):
key = int(key)
if key >= len(obj):
return default
obj = obj[key]
else:
obj = obj.get(key, default)
return obj
def parse_json_files(directory: str, test_type: TestType) -> pd.DataFrame: def parse_json_files(directory: str, test_type: TestType) -> pd.DataFrame:
metrics_to_keep = {'inter_token_latency': {'y': 'Time (ms)'}, 'end_to_end_latency': {'y': 'Time (ms)'}, metrics_to_keep = {'inter_token_latency': {'y': 'Time (ms)'}, 'end_to_end_latency': {'y': 'Time (ms)'},
'time_to_first_token': {'y': 'Time (ms)'}, 'tokens_throughput': {'y': 'Tokens/s'}, 'time_to_first_token': {'y': 'Time (ms)'}, 'tokens_throughput': {'y': 'Tokens/s'},
@ -38,10 +53,9 @@ def parse_json_files(directory: str, test_type: TestType) -> pd.DataFrame:
} }
entry['input_type'] = data['k6_config']['input_type'] entry['input_type'] = data['k6_config']['input_type']
entry['test_duration'] = data['state']['testRunDurationMs'] / 1000. entry['test_duration'] = data['state']['testRunDurationMs'] / 1000.
entry['requests_ok'] = data['root_group']['checks'][0]['passes'] entry['requests_ok'] = get_nested(data, 'root_group.checks.0.passes', 0)
entry['requests_fail'] = data['root_group']['checks'][0]['fails'] entry['requests_fail'] = get_nested(data, 'root_group.checks.0.fails', 0)
entry['dropped_iterations'] = data['metrics']['dropped_iterations']['values'][ entry['dropped_iterations'] = get_nested(data, 'metrics.dropped_iterations.values.count', 0)
'count'] if 'dropped_iterations' in data['metrics'] else 0
# add up requests_fail and dropped_iterations to get total dropped requests # add up requests_fail and dropped_iterations to get total dropped requests
entry['dropped_requests'] = entry['requests_fail'] + entry['dropped_iterations'] entry['dropped_requests'] = entry['requests_fail'] + entry['dropped_iterations']
entry['error_rate'] = entry['dropped_requests'] / ( entry['error_rate'] = entry['dropped_requests'] / (
@ -60,7 +74,7 @@ def parse_json_files(directory: str, test_type: TestType) -> pd.DataFrame:
return df return df
def plot_metrics(model_name:str, df: pd.DataFrame, test_type: TestType, save_name: str): def plot_metrics(model_name: str, df: pd.DataFrame, test_type: TestType, save_name: str):
vus_param = '' vus_param = ''
if test_type == TestType.CONSTANT_VUS: if test_type == TestType.CONSTANT_VUS:
vus_param = 'vus' vus_param = 'vus'