From 41fbf5c2541b17de2b3e2108890e58ca37a253d8 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Fri, 26 Jan 2024 04:41:39 -0500
Subject: [PATCH] fix: show warning with tokenizer config parsing error (#1488)

This tiny PR just prints the parsing error when a tokenizer config fails
to load.

This is helpful when a chat_template wont load due to formatting issues
https://github.com/huggingface/text-generation-inference/pull/1427#issuecomment-1909226388
---
 router/src/main.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/router/src/main.rs b/router/src/main.rs
index 5f52b801..6526b859 100644
--- a/router/src/main.rs
+++ b/router/src/main.rs
@@ -473,7 +473,12 @@ pub async fn get_tokenizer_config(api_repo: &ApiRepo) -> Option<HubTokenizerConf
     let reader = BufReader::new(file);
 
     // Read the JSON contents of the file as an instance of 'HubTokenizerConfig'.
-    let tokenizer_config: HubTokenizerConfig = serde_json::from_reader(reader).ok()?;
+    let tokenizer_config: HubTokenizerConfig = serde_json::from_reader(reader)
+        .map_err(|e| {
+            tracing::warn!("Unable to parse tokenizer config: {}", e);
+            e
+        })
+        .ok()?;
 
     Some(tokenizer_config)
 }