From 87c4828c4e21ebaec3ae4cc0dbf22d06dda6f4b4 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 22 Apr 2024 13:58:46 -0400
Subject: [PATCH] feat: allow null eos and bos tokens in config

---
 router/src/lib.rs  | 1 +
 router/src/main.rs | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/router/src/lib.rs b/router/src/lib.rs
index 53982c36..da58413f 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -116,6 +116,7 @@ mod token_serde {
                     ))
                 }
             }
+            Value::Null => Ok(None),
             _ => Err(de::Error::custom("invalid token format")),
         }
     }
diff --git a/router/src/main.rs b/router/src/main.rs
index b77117a1..c7e3f90b 100644
--- a/router/src/main.rs
+++ b/router/src/main.rs
@@ -254,7 +254,10 @@ async fn main() -> Result<(), RouterError> {
 
     // Load tokenizer config if found locally, or check if we can get it from the API if needed
     let tokenizer_config = if let Some(path) = tokenizer_config_path {
-        tracing::info!("Using local tokenizer config from user specified path");
+        tracing::info!(
+            "Using local tokenizer config from user specified path {}",
+            path
+        );
         HubTokenizerConfig::from_file(&std::path::PathBuf::from(path))
     } else if local_model {
         tracing::info!("Using local tokenizer config");