From c326ffdac0c0b70748da2ac677f3602e30986dbb Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Thu, 27 Jun 2024 11:33:29 -0400
Subject: [PATCH] fix: adjust HubTokenizerConfig after rebase

---
 router/src/lib.rs    | 24 +++++++++++++++++-------
 router/src/server.rs |  2 +-
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/router/src/lib.rs b/router/src/lib.rs
index a4bd3229..d5e551a0 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -59,10 +59,8 @@ use std::path::Path;
 pub struct HubTokenizerConfig {
     pub chat_template: Option<ChatTemplateVersions>,
     pub completion_template: Option<String>,
-    #[serde(deserialize_with = "token_serde::deserialize")]
-    pub bos_token: Option<String>,
-    #[serde(deserialize_with = "token_serde::deserialize")]
-    pub eos_token: Option<String>,
+    pub bos_token: Option<TokenizerConfigToken>,
+    pub eos_token: Option<TokenizerConfigToken>,
     pub tokenizer_class: Option<String>,
     pub add_bos_token: Option<bool>,
     pub add_eos_token: Option<bool>,
@@ -83,15 +81,27 @@ pub enum TokenizerConfigToken {
     Object { content: String },
 }
 
-impl From<TokenizerConfigToken> for String {
-    fn from(token: TokenizerConfigToken) -> Self {
-        match token {
+impl TokenizerConfigToken {
+    pub fn as_str(&self) -> &str {
+        match self {
             TokenizerConfigToken::String(s) => s,
             TokenizerConfigToken::Object { content } => content,
         }
     }
 }
 
+impl From<TokenizerConfigToken> for String {
+    fn from(token: TokenizerConfigToken) -> Self {
+        token.as_str().to_string()
+    }
+}
+
+impl From<String> for TokenizerConfigToken {
+    fn from(s: String) -> Self {
+        TokenizerConfigToken::String(s)
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(tag = "processor_class")]
 pub enum HubPreprocessorConfig {
diff --git a/router/src/server.rs b/router/src/server.rs
index e11403eb..d24774f9 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -23,7 +23,7 @@ use crate::{
     CompletionRequest, CompletionType, DeltaToolCall, Function, Tool, VertexRequest,
     VertexResponse,
 };
-use crate::{FunctionDefinition, ToolCall, ToolType};
+use crate::{FunctionDefinition, HubPreprocessorConfig, ToolCall, ToolType};
 use async_stream::__private::AsyncStream;
 use axum::extract::Extension;
 use axum::http::{HeaderMap, Method, StatusCode};