huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-19 22:02:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update server/text_generation_server/models/flash_causal_lm.py

Browse Source 
Co-authored-by: Daniël de Kok <me@github.danieldk.eu>
This commit is contained in:
Wang, Yi 2025-01-14 08:58:48 +08:00 committed by GitHub
parent 5ad8c9a40b
commit 22ed5703de
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/text_generation_server/models/flash_causal_lm.py
View File

@ -1595,11 +1595,7 @@ class FlashCausalLM(Model):
if max_total_tokens is None: if max_total_tokens is None:
if get_support_chunking(): if get_support_chunking():
model_max_length = self.tokenizer.model_max_length model_max_length = self.tokenizer.model_max_length
max_position_embeddings = ( max_position_embeddings = getattr(self.config, "max_position_embeddings", model_max_length)
self.config.max_position_embeddings
if hasattr(self.config, "max_position_embeddings")
else model_max_length
)
max_total_tokens = min( max_total_tokens = min(
num_blocks * BLOCK_SIZE, model_max_length, max_position_embeddings num_blocks * BLOCK_SIZE, model_max_length, max_position_embeddings
) )