diff --git a/router/src/lib.rs b/router/src/lib.rs index 63728a28..3c15daae 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -669,7 +669,7 @@ pub(crate) struct ErrorResponse { #[cfg(test)] mod tests { use super::*; - use std::path::Path; + use tokenizers::Tokenizer; pub(crate) async fn get_tokenizer() -> Tokenizer { @@ -679,14 +679,6 @@ mod tests { Tokenizer::from_file(filename).unwrap() } - #[test] - fn test_hub_tokenizer_config() { - let config = HubTokenizerConfig::from_file(Path::new("tokenizer.json")); - assert_eq!(config.chat_template, Some("test".to_string())); - assert_eq!(config.bos_token, Some("test".to_string())); - assert_eq!(config.eos_token, Some("test".to_string())); - } - #[test] fn test_hub_nested_tokens_tokenizer_config() { // this is a subset of the tokenizer.json file