From 048bc5b4b79f56b3a77eeffa518a4e215ec159fe Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Tue, 23 Jan 2024 15:04:17 +0100
Subject: [PATCH] Remove special, it's not correct enough (and not necessarily
 useful).

---
 router/src/server.rs     | 8 ++++----
 router/src/validation.rs | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/router/src/server.rs b/router/src/server.rs
index 42785025..678ea9c1 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -696,13 +696,13 @@ async fn tokenize(
             .get_ids()
             .iter()
             .zip(encoding.get_offsets())
-            .map(|(&id, (start, stop))| {
-                let text: String = input.chars().skip(*start).take(stop - start).collect();
+            .map(|(&id, &(start, stop))| {
+                let text: String = input.chars().skip(start).take(stop - start).collect();
                 SimpleToken {
                     id,
                     text,
-                    start: *start,
-                    stop: *stop,
+                    start,
+                    stop,
                 }
             })
             .collect();
diff --git a/router/src/validation.rs b/router/src/validation.rs
index e6dbcf81..adbe6bb2 100644
--- a/router/src/validation.rs
+++ b/router/src/validation.rs
@@ -365,7 +365,7 @@ fn prepare_input(
 ) -> Result<(tokenizers::Encoding, String), ValidationError> {
     // Get the number of tokens in the input
     let mut encoding = tokenizer
-        .encode(inputs.clone(), true)
+        .encode(inputs.clone(), false)
         .map_err(|err| ValidationError::Tokenizer(err.to_string()))?;
 
     // Optionally truncate