From d1639a58276053e1b6c2eb59369a26400c4889e3 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 1 May 2024 18:20:44 +0200 Subject: [PATCH] "Fixing" double BOS for mistral too. --- server/text_generation_server/models/flash_mistral.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py index 6959e2ec..85e93543 100644 --- a/server/text_generation_server/models/flash_mistral.py +++ b/server/text_generation_server/models/flash_mistral.py @@ -121,6 +121,11 @@ class FlashMistralBatch(FlashCausalLMBatch): requests_idx_mapping[r.id] = i tokenized_input = tokenized_input[-r.truncate :] + if ( + tokenized_input[0] == tokenizer.bos_token_id + and tokenized_input[1] == tokenizer.bos_token_id + ): + tokenized_input = tokenized_input[1:] input_length = len(tokenized_input) input_lengths.append(input_length)