From bfe062ee998b63418518db9941b0c9aac643c796 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Wed, 24 Jan 2024 12:31:12 +0000
Subject: [PATCH] Go back to the default of adding special tokens in
 validation.

---
 router/src/validation.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/router/src/validation.rs b/router/src/validation.rs
index adbe6bb2..e6dbcf81 100644
--- a/router/src/validation.rs
+++ b/router/src/validation.rs
@@ -365,7 +365,7 @@ fn prepare_input(
 ) -> Result<(tokenizers::Encoding, String), ValidationError> {
     // Get the number of tokens in the input
     let mut encoding = tokenizer
-        .encode(inputs.clone(), false)
+        .encode(inputs.clone(), true)
         .map_err(|err| ValidationError::Tokenizer(err.to_string()))?;
 
     // Optionally truncate