From 9be1db3101a9c02ecbffb56f4f97ac49d5441dbc Mon Sep 17 00:00:00 2001 From: drbh Date: Tue, 23 Apr 2024 10:26:54 -0400 Subject: [PATCH] feat: allow null eos and bos tokens in config (#1791) This PR resolves an issue loading in tokenizer_configs where the eos or bos token is null as in: [Qwen/Qwen1.5-72B-Chat](https://huggingface.co/Qwen/Qwen1.5-72B-Chat/blob/main/tokenizer_config.json) resolves: https://github.com/huggingface/text-generation-inference/issues/1545 and related to https://github.com/QwenLM/Qwen1.5/issues/162 --- router/src/lib.rs | 1 + router/src/main.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index 9e847fe2..5e207a03 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -116,6 +116,7 @@ mod token_serde { )) } } + Value::Null => Ok(None), _ => Err(de::Error::custom("invalid token format")), } } diff --git a/router/src/main.rs b/router/src/main.rs index b77117a1..c7e3f90b 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -254,7 +254,10 @@ async fn main() -> Result<(), RouterError> { // Load tokenizer config if found locally, or check if we can get it from the API if needed let tokenizer_config = if let Some(path) = tokenizer_config_path { - tracing::info!("Using local tokenizer config from user specified path"); + tracing::info!( + "Using local tokenizer config from user specified path {}", + path + ); HubTokenizerConfig::from_file(&std::path::PathBuf::from(path)) } else if local_model { tracing::info!("Using local tokenizer config");