Fixing tokenization like https://github.com/huggingface/text-embeddings-inference/issues/525

router/src/server.rs

@@ -74,11 +74,8 @@ fn encoding_to_tokens(encoding: &tokenizers::Encoding, input: &str) -> Vec<Simpl
             .iter()
             .zip(offsets)
             .map(|(&id, &(start, stop))| {
-                let text = input
+                let text: Vec<u8> = input.bytes().skip(start).take(stop - start).collect();
-                    .chars()
+                let text: String = String::from_utf8_lossy(&text).to_string();
-                    .skip(start)
-                    .take(stop - start)
-                    .collect::<String>();
                 SimpleToken {
                     id,
                     text,