From 535149d872db15b3aeea93c3413893b7cf77e696 Mon Sep 17 00:00:00 2001 From: Dmitry Rogozhkin Date: Sun, 1 Dec 2024 21:26:37 -0800 Subject: [PATCH] fix: only use eos_token_id as pad_token_id if int (#2774) LLama 3 has a list of values as eos_token_id: "['<|end_of_text|>', '<|eom_id|>', '<|eot_id|>']" This breaks tokenizer since it expects single value. This commit uses tokenizer.eos_token_id instead in such a case. Fixes: #2440 Signed-off-by: Dmitry Rogozhkin --- server/text_generation_server/models/causal_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py index bd8176be..82407de3 100644 --- a/server/text_generation_server/models/causal_lm.py +++ b/server/text_generation_server/models/causal_lm.py @@ -630,7 +630,7 @@ class CausalLM(Model): if tokenizer.pad_token_id is None: if model.config.pad_token_id is not None: tokenizer.pad_token_id = model.config.pad_token_id - elif model.config.eos_token_id is not None: + elif model.config.eos_token_id is not None and isinstance(model.config.eos_token_id, int): tokenizer.pad_token_id = model.config.eos_token_id elif tokenizer.eos_token_id is not None: tokenizer.pad_token_id = tokenizer.eos_token_id