huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-25 03:52:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Baichuan2-13B does not have max_position_embeddings in config

Browse Source 
see https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/main/config.json

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi A 2025-01-12 22:47:23 -08:00
parent 83624a07be
commit 5ad8c9a40b
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/text_generation_server/models/flash_causal_lm.py
View File

@ -1595,7 +1595,11 @@ class FlashCausalLM(Model):
if max_total_tokens is None: if max_total_tokens is None:
if get_support_chunking(): if get_support_chunking():
model_max_length = self.tokenizer.model_max_length model_max_length = self.tokenizer.model_max_length
max_position_embeddings = self.config.max_position_embeddings max_position_embeddings = (
self.config.max_position_embeddings
if hasattr(self.config, "max_position_embeddings")
else model_max_length
)
max_total_tokens = min( max_total_tokens = min(
num_blocks * BLOCK_SIZE, model_max_length, max_position_embeddings num_blocks * BLOCK_SIZE, model_max_length, max_position_embeddings
) )