huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-06-19 15:52:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pre commit

Browse Source
This commit is contained in:
Nicolas Patry 2025-04-24 15:51:01 +02:00
parent d7a609d4ad
commit 8c782858bb
No known key found for this signature in database
GPG Key ID: 87B37D879D09DEB4
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/text_generation_server/models/globals.py
View File

@ -15,9 +15,9 @@ PREFIX_CACHING = os.environ["PREFIX_CACHING"].lower() in {
PREFILL_CHUNKING = os.getenv("PREFILL_CHUNKING", "1").lower() in {"1", "true"} PREFILL_CHUNKING = os.getenv("PREFILL_CHUNKING", "1").lower() in {"1", "true"}
log_master(logger.info, f"Using prefix caching = {PREFIX_CACHING}") log_master(logger.info, f"Using prefix caching = {PREFIX_CACHING}")
_expected = {"paged", "flashdecoding", "flashdecoding-ipex", "flashinfer"} _expected = {"paged", "flashdecoding", "flashdecoding-ipex", "flashinfer"}
assert ATTENTION in _expected, ( assert (
f"Attention is not valid {ATTENTION}, expected {_expected}" ATTENTION in _expected
) ), f"Attention is not valid {ATTENTION}, expected {_expected}"
log_master(logger.info, f"Using Attention = {ATTENTION}") log_master(logger.info, f"Using Attention = {ATTENTION}")
if PREFIX_CACHING and ATTENTION not in { if PREFIX_CACHING and ATTENTION not in {