huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 00:12:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fmt

Browse Source 
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi A 2025-01-13 17:23:28 -08:00
parent 22ed5703de
commit 48067e4a0d
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/text_generation_server/models/flash_causal_lm.py
View File

@ -1595,7 +1595,9 @@ class FlashCausalLM(Model):
if max_total_tokens is None: if max_total_tokens is None:
if get_support_chunking(): if get_support_chunking():
model_max_length = self.tokenizer.model_max_length model_max_length = self.tokenizer.model_max_length
max_position_embeddings = getattr(self.config, "max_position_embeddings", model_max_length) max_position_embeddings = getattr(
self.config, "max_position_embeddings", model_max_length
)
max_total_tokens = min( max_total_tokens = min(
num_blocks * BLOCK_SIZE, model_max_length, max_position_embeddings num_blocks * BLOCK_SIZE, model_max_length, max_position_embeddings
) )