huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fmt

Browse Source
This commit is contained in:
OlivierDehaene 2024-11-22 17:39:20 +01:00
parent 690702b1ce
commit 7213d30141
No known key found for this signature in database
GPG Key ID: BB104D67809DA93C
2 changed files with 130 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

211
load_tests/benchmarks.py
View File

@ -24,10 +24,12 @@ class InferenceEngineRunner:
class TGIDockerRunner(InferenceEngineRunner): class TGIDockerRunner(InferenceEngineRunner):
def __init__(self, def __init__(
model: str, self,
image: str = "ghcr.io/huggingface/text-generation-inference:latest", model: str,
volumes=None): image: str = "ghcr.io/huggingface/text-generation-inference:latest",
volumes=None,
):
super().__init__(model) super().__init__(model)
if volumes is None: if volumes is None:
volumes = [] volumes = []
@ -43,13 +45,15 @@ class TGIDockerRunner(InferenceEngineRunner):
volumes = {} volumes = {}
for v in self.volumes: for v in self.volumes:
volumes[v[0]] = {"bind": v[1], "mode": "rw"} volumes[v[0]] = {"bind": v[1], "mode": "rw"}
self.container = run_docker(self.image, params, self.container = run_docker(
"Connected", self.image,
"ERROR", params,
volumes=volumes, "Connected",
gpus=gpus, "ERROR",
ports={"8080/tcp": 8080} volumes=volumes,
) gpus=gpus,
ports={"8080/tcp": 8080},
)
def stop(self): def stop(self):
if self.container: if self.container:
@ -57,9 +61,11 @@ class TGIDockerRunner(InferenceEngineRunner):
class BenchmarkRunner: class BenchmarkRunner:
def __init__(self, def __init__(
image: str = "ghcr.io/huggingface/text-generation-inference-benchmark:latest", self,
volumes: List[Tuple[str, str]] = None): image: str = "ghcr.io/huggingface/text-generation-inference-benchmark:latest",
volumes: List[Tuple[str, str]] = None,
):
if volumes is None: if volumes is None:
volumes = [] volumes = []
self.container = None self.container = None
@ -70,26 +76,41 @@ class BenchmarkRunner:
params = "text-generation-inference-benchmark" params = "text-generation-inference-benchmark"
for p in parameters: for p in parameters:
params += f" --{p[0]} {str(p[1])}" if p[1] is not None else f" --{p[0]}" params += f" --{p[0]} {str(p[1])}" if p[1] is not None else f" --{p[0]}"
logger.info(f"Running text-generation-inference-benchmarks with parameters: {params}") logger.info(
f"Running text-generation-inference-benchmarks with parameters: {params}"
)
volumes = {} volumes = {}
for v in self.volumes: for v in self.volumes:
volumes[v[0]] = {"bind": v[1], "mode": "rw"} volumes[v[0]] = {"bind": v[1], "mode": "rw"}
self.container = run_docker(self.image, params, self.container = run_docker(
"Benchmark finished", self.image,
"Fatal:", params,
volumes=volumes, "Benchmark finished",
extra_env={"RUST_LOG": "text_generation_inference_benchmark=info", "Fatal:",
"RUST_BACKTRACE": "full"}, volumes=volumes,
network_mode=network_mode) extra_env={
"RUST_LOG": "text_generation_inference_benchmark=info",
"RUST_BACKTRACE": "full",
},
network_mode=network_mode,
)
def stop(self): def stop(self):
if self.container: if self.container:
self.container.stop() self.container.stop()
def run_docker(image: str, args: str, success_sentinel: str, def run_docker(
error_sentinel: str, ports: Dict[str, int] = None, volumes=None, network_mode: str = "bridge", image: str,
gpus: int = 0, extra_env: Dict[str, str] = None) -> Container: args: str,
success_sentinel: str,
error_sentinel: str,
ports: Dict[str, int] = None,
volumes=None,
network_mode: str = "bridge",
gpus: int = 0,
extra_env: Dict[str, str] = None,
) -> Container:
if ports is None: if ports is None:
ports = {} ports = {}
if volumes is None: if volumes is None:
@ -98,21 +119,24 @@ def run_docker(image: str, args: str, success_sentinel: str,
extra_env = {} extra_env = {}
client = docker.from_env(timeout=300) client = docker.from_env(timeout=300)
# retrieve the GPU devices from CUDA_VISIBLE_DEVICES # retrieve the GPU devices from CUDA_VISIBLE_DEVICES
devices = [f"{i}" for i in devices = [f"{i}" for i in range(get_num_gpus())][:gpus]
range(get_num_gpus())][:gpus]
environment = {"HF_TOKEN": os.environ.get("HF_TOKEN")} environment = {"HF_TOKEN": os.environ.get("HF_TOKEN")}
environment.update(extra_env) environment.update(extra_env)
container = client.containers.run(image, args, container = client.containers.run(
detach=True, image,
device_requests=[ args,
docker.types.DeviceRequest(device_ids=devices, detach=True,
capabilities=[['gpu']]) device_requests=(
] if gpus > 0 else None, [docker.types.DeviceRequest(device_ids=devices, capabilities=[["gpu"]])]
volumes=volumes, if gpus > 0
shm_size="1g", else None
ports=ports, ),
network_mode=network_mode, volumes=volumes,
environment=environment, ) shm_size="1g",
ports=ports,
network_mode=network_mode,
environment=environment,
)
for line in container.logs(stream=True): for line in container.logs(stream=True):
print(line.decode("utf-8"), end="") print(line.decode("utf-8"), end="")
if success_sentinel.encode("utf-8") in line: if success_sentinel.encode("utf-8") in line:
@ -126,14 +150,14 @@ def run_docker(image: str, args: str, success_sentinel: str,
def get_gpu_names() -> str: def get_gpu_names() -> str:
gpus = GPUtil.getGPUs() gpus = GPUtil.getGPUs()
if len(gpus) == 0: if len(gpus) == 0:
return '' return ""
return f'{len(gpus)}x{gpus[0].name if gpus else "No GPU available"}' return f'{len(gpus)}x{gpus[0].name if gpus else "No GPU available"}'
def get_gpu_name() -> str: def get_gpu_name() -> str:
gpus = GPUtil.getGPUs() gpus = GPUtil.getGPUs()
if len(gpus) == 0: if len(gpus) == 0:
return '' return ""
return gpus[0].name return gpus[0].name
@ -147,29 +171,29 @@ def build_df(model: str, data_files: dict[str, str]) -> pd.DataFrame:
created_at = now.isoformat() # '2024-10-02T11:53:17.026215+00:00' created_at = now.isoformat() # '2024-10-02T11:53:17.026215+00:00'
# Load the results # Load the results
for key, filename in data_files.items(): for key, filename in data_files.items():
with open(filename, 'r') as f: with open(filename, "r") as f:
data = json.load(f) data = json.load(f)
for result in data['results']: for result in data["results"]:
entry = result entry = result
[config] = pd.json_normalize(result['config']).to_dict(orient='records') [config] = pd.json_normalize(result["config"]).to_dict(orient="records")
entry.update(config) entry.update(config)
entry['engine'] = data['config']['meta']['engine'] entry["engine"] = data["config"]["meta"]["engine"]
entry['tp'] = data['config']['meta']['tp'] entry["tp"] = data["config"]["meta"]["tp"]
entry['version'] = data['config']['meta']['version'] entry["version"] = data["config"]["meta"]["version"]
entry['model'] = model entry["model"] = model
entry['created_at'] = created_at entry["created_at"] = created_at
del entry['config'] del entry["config"]
df = pd.concat([df, pd.DataFrame(entry, index=[0])]) df = pd.concat([df, pd.DataFrame(entry, index=[0])])
return df return df
def main(sha, results_file): def main(sha, results_file):
results_dir = 'results' results_dir = "results"
# get absolute path # get absolute path
results_dir = os.path.join(os.path.dirname(__file__), results_dir) results_dir = os.path.join(os.path.dirname(__file__), results_dir)
logger.info('Starting benchmark') logger.info("Starting benchmark")
models = [ models = [
('meta-llama/Llama-3.1-8B-Instruct', 1), ("meta-llama/Llama-3.1-8B-Instruct", 1),
# ('meta-llama/Llama-3.1-70B-Instruct', 4), # ('meta-llama/Llama-3.1-70B-Instruct', 4),
# ('mistralai/Mixtral-8x7B-Instruct-v0.1', 2), # ('mistralai/Mixtral-8x7B-Instruct-v0.1', 2),
] ]
@ -177,31 +201,42 @@ def main(sha, results_file):
for model in models: for model in models:
tgi_runner = TGIDockerRunner(model[0]) tgi_runner = TGIDockerRunner(model[0])
# create results directory # create results directory
model_dir = os.path.join(results_dir, f'{model[0].replace("/", "_").replace(".", "_")}') model_dir = os.path.join(
results_dir, f'{model[0].replace("/", "_").replace(".", "_")}'
)
os.makedirs(model_dir, exist_ok=True) os.makedirs(model_dir, exist_ok=True)
runner = BenchmarkRunner( runner = BenchmarkRunner(
volumes=[(model_dir, '/opt/text-generation-inference-benchmark/results')] volumes=[(model_dir, "/opt/text-generation-inference-benchmark/results")]
) )
try: try:
tgi_runner.run([('max-concurrent-requests', 512)], gpus=model[1]) tgi_runner.run([("max-concurrent-requests", 512)], gpus=model[1])
logger.info(f'TGI started for model {model[0]}') logger.info(f"TGI started for model {model[0]}")
parameters = [ parameters = [
('tokenizer-name', model[0]), ("tokenizer-name", model[0]),
('max-vus', 800), ("max-vus", 800),
('url', 'http://localhost:8080'), ("url", "http://localhost:8080"),
('duration', '120s'), ("duration", "120s"),
('warmup', '30s'), ("warmup", "30s"),
('benchmark-kind', 'rate'), ("benchmark-kind", "rate"),
('prompt-options', 'num_tokens=200,max_tokens=220,min_tokens=180,variance=10'), (
('decode-options', 'num_tokens=200,max_tokens=220,min_tokens=180,variance=10'), "prompt-options",
('extra-meta', f'"engine=TGI,tp={model[1]},version={sha},gpu={get_gpu_name()}"'), "num_tokens=200,max_tokens=220,min_tokens=180,variance=10",
('no-console', None) ),
(
"decode-options",
"num_tokens=200,max_tokens=220,min_tokens=180,variance=10",
),
(
"extra-meta",
f'"engine=TGI,tp={model[1]},version={sha},gpu={get_gpu_name()}"',
),
("no-console", None),
] ]
rates = [('rates', f'{r / 10.}') for r in list(range(8, 248, 8))] rates = [("rates", f"{r / 10.}") for r in list(range(8, 248, 8))]
parameters.extend(rates) parameters.extend(rates)
runner.run(parameters, f'container:{tgi_runner.container.id}') runner.run(parameters, f"container:{tgi_runner.container.id}")
except Exception as e: except Exception as e:
logger.error(f'Error running benchmark for model {model[0]}: {e}') logger.error(f"Error running benchmark for model {model[0]}: {e}")
# print the stack trace # print the stack trace
print(traceback.format_exc()) print(traceback.format_exc())
success = False success = False
@ -209,33 +244,45 @@ def main(sha, results_file):
tgi_runner.stop() tgi_runner.stop()
runner.stop() runner.stop()
if not success: if not success:
logger.error('Some benchmarks failed') logger.error("Some benchmarks failed")
exit(1) exit(1)
df = pd.DataFrame() df = pd.DataFrame()
# list recursively directories # list recursively directories
directories = [f'{results_dir}/{d}' for d in os.listdir(results_dir) if os.path.isdir(f'{results_dir}/{d}')] directories = [
logger.info(f'Found result directories: {directories}') f"{results_dir}/{d}"
for d in os.listdir(results_dir)
if os.path.isdir(f"{results_dir}/{d}")
]
logger.info(f"Found result directories: {directories}")
for directory in directories: for directory in directories:
data_files = {} data_files = {}
for filename in os.listdir(directory): for filename in os.listdir(directory):
if filename.endswith('.json'): if filename.endswith(".json"):
data_files[filename.split('.')[-2]] = f'{directory}/{filename}' data_files[filename.split(".")[-2]] = f"{directory}/{filename}"
logger.info(f'Processing directory {directory}') logger.info(f"Processing directory {directory}")
df = pd.concat([df, build_df(directory.split('/')[-1], data_files)]) df = pd.concat([df, build_df(directory.split("/")[-1], data_files)])
df['device'] = get_gpu_name() df["device"] = get_gpu_name()
df['error_rate'] = df['failed_requests'] / (df['failed_requests'] + df['successful_requests']) * 100.0 df["error_rate"] = (
df["failed_requests"]
/ (df["failed_requests"] + df["successful_requests"])
* 100.0
)
df.to_parquet(results_file) df.to_parquet(results_file)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--sha", help="SHA of the commit to add to the results", required=True) parser.add_argument(
parser.add_argument("--results-file", "--sha", help="SHA of the commit to add to the results", required=True
help="The file where to store the results, can be a local file or a s3 path") )
parser.add_argument(
"--results-file",
help="The file where to store the results, can be a local file or a s3 path",
)
args = parser.parse_args() args = parser.parse_args()
if args.results_file is None: if args.results_file is None:
results_file = f'{args.sha}.parquet' results_file = f"{args.sha}.parquet"
else: else:
results_file = args.results_file results_file = args.results_file