diff --git a/router/src/server.rs b/router/src/server.rs index 42785025..678ea9c1 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -696,13 +696,13 @@ async fn tokenize( .get_ids() .iter() .zip(encoding.get_offsets()) - .map(|(&id, (start, stop))| { - let text: String = input.chars().skip(*start).take(stop - start).collect(); + .map(|(&id, &(start, stop))| { + let text: String = input.chars().skip(start).take(stop - start).collect(); SimpleToken { id, text, - start: *start, - stop: *stop, + start, + stop, } }) .collect(); diff --git a/router/src/validation.rs b/router/src/validation.rs index e6dbcf81..adbe6bb2 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -365,7 +365,7 @@ fn prepare_input( ) -> Result<(tokenizers::Encoding, String), ValidationError> { // Get the number of tokens in the input let mut encoding = tokenizer - .encode(inputs.clone(), true) + .encode(inputs.clone(), false) .map_err(|err| ValidationError::Tokenizer(err.to_string()))?; // Optionally truncate