diff --git a/router/src/server.rs b/router/src/server.rs index 45d2b9f3..077c4102 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -74,11 +74,8 @@ fn encoding_to_tokens(encoding: &tokenizers::Encoding, input: &str) -> Vec(); + let text: Vec = input.bytes().skip(start).take(stop - start).collect(); + let text: String = String::from_utf8_lossy(&text).to_string(); SimpleToken { id, text,