huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 00:12:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change things around when we don't have a tokenizer.

Browse Source
This commit is contained in:
Nicolas Patry 2024-04-11 18:44:23 +00:00
parent d43e10e097
commit 179ee4e2c2
1 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
router/src/validation.rs
View File

@ -161,17 +161,18 @@ impl Validation {
} else { } else {
return Err(ValidationError::UnsetMaxNewTokens); return Err(ValidationError::UnsetMaxNewTokens);
}; };
let input_length = truncate.unwrap_or(self.max_input_length); let mut input_length = truncate.unwrap_or(self.max_input_length);
// We don't have a tokenizer, therefore we have no idea how long is the query, let // We don't have a tokenizer, therefore we have no idea how long is the query, let
// them through and hope for the best. // them through and hope for the best.
// Validate MaxNewTokens // Validate MaxNewTokens
// if (input_length as u32 + max_new_tokens) > self.max_total_tokens as u32 { if (input_length as u32 + max_new_tokens) > self.max_total_tokens as u32 {
input_length = input_length.saturating_sub(max_new_tokens as usize);
// return Err(ValidationError::MaxNewTokens( // return Err(ValidationError::MaxNewTokens(
// self.max_total_tokens - self.max_input_length, // self.max_total_tokens - self.max_input_length,
// max_new_tokens, // max_new_tokens,
// )); // ));
// } }
Ok((inputs, input_length, max_new_tokens)) Ok((inputs, input_length, max_new_tokens))
} }
@ -666,8 +667,9 @@ mod tests {
.validate_input("Hello".to_string(), None, Some(max_new_tokens)) .validate_input("Hello".to_string(), None, Some(max_new_tokens))
.await .await
{ {
Err(ValidationError::MaxNewTokens(1, 10)) => (), // Err(ValidationError::MaxNewTokens(1, 10)) => (),
_ => panic!("Unexpected not max new tokens"), Ok((_s, 0, 10)) => (),
r => panic!("Unexpected not max new tokens: {r:?}"),
} }
} }