huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ignore missing generation config.

Browse Source
This commit is contained in:
Nicolas Patry 2024-04-25 16:42:35 +00:00
parent 1f1885d911
commit ffea15d6b6
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/models/flash_llama.py
View File

@ -52,12 +52,15 @@ class FlashLlama(FlashCausalLM):
truncation_side="left", truncation_side="left",
trust_remote_code=trust_remote_code, trust_remote_code=trust_remote_code,
) )
try:
generation_config = GenerationConfig.from_pretrained( generation_config = GenerationConfig.from_pretrained(
model_id, revision=revision, trust_remote_code=trust_remote_code model_id, revision=revision, trust_remote_code=trust_remote_code
) )
if isinstance(generation_config.eos_token_id, (list, set)): if isinstance(generation_config.eos_token_id, (list, set)):
# TODO Huge hack # TODO Huge hack
tokenizer._eos_token_ids = set(generation_config.eos_token_id) tokenizer._eos_token_ids = set(generation_config.eos_token_id)
except Exception:
pass
config = AutoConfig.from_pretrained( config = AutoConfig.from_pretrained(
model_id, revision=revision, trust_remote_code=trust_remote_code model_id, revision=revision, trust_remote_code=trust_remote_code