This commit is contained in:
Nicolas Patry 2024-01-25 10:33:51 +00:00
parent 211b9681e6
commit 46ca08a831

View File

@ -373,8 +373,8 @@ fn prepare_input(
if truncate < encoding.len() { if truncate < encoding.len() {
encoding.truncate(truncate, 0, TruncationDirection::Left); encoding.truncate(truncate, 0, TruncationDirection::Left);
inputs = tokenizer inputs = tokenizer
.decode(encoding.get_ids(), false) .decode(encoding.get_ids(), false)
.map_err(|err| ValidationError::Tokenizer(err.to_string()))?; .map_err(|err| ValidationError::Tokenizer(err.to_string()))?;
} }
} }