huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 12:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Early exit on server too.

Browse Source
This commit is contained in:
Nicolas Patry 2024-08-09 12:47:39 +02:00
parent 6bcad66c6e
commit d84b98b40f
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
2 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -18,3 +18,4 @@ server/exllama_kernels/exllama_kernels/exllama_ext_hip.cpp
data/
load_tests/*.json
server/fbgemmm

2
server/text_generation_server/models/globals.py
View File

@ -6,6 +6,8 @@ from typing import Dict, Optional
from text_generation_server.utils.log import log_master
ATTENTION = os.getenv("ATTENTION", "paged")
_expected = {"paged", "flashdecoding", "flashinfer"}
assert ATTENTION in _expected, f"Attention is not valid {ATTENTION}, expected {_expected}"
log_master(logger.info, f"Using Attention = {ATTENTION}")
MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None