From ffea15d6b6c59d338e5a5b08de99dd8c2bfd4418 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 25 Apr 2024 16:42:35 +0000 Subject: [PATCH] Ignore missing generation config. --- .../text_generation_server/models/flash_llama.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py index f638a95e..f3578f88 100644 --- a/server/text_generation_server/models/flash_llama.py +++ b/server/text_generation_server/models/flash_llama.py @@ -52,12 +52,15 @@ class FlashLlama(FlashCausalLM): truncation_side="left", trust_remote_code=trust_remote_code, ) - generation_config = GenerationConfig.from_pretrained( - model_id, revision=revision, trust_remote_code=trust_remote_code - ) - if isinstance(generation_config.eos_token_id, (list, set)): - # TODO Huge hack - tokenizer._eos_token_ids = set(generation_config.eos_token_id) + try: + generation_config = GenerationConfig.from_pretrained( + model_id, revision=revision, trust_remote_code=trust_remote_code + ) + if isinstance(generation_config.eos_token_id, (list, set)): + # TODO Huge hack + tokenizer._eos_token_ids = set(generation_config.eos_token_id) + except Exception: + pass config = AutoConfig.from_pretrained( model_id, revision=revision, trust_remote_code=trust_remote_code