huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 23:42:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Correct input_length since habana extend input_length to max_input_length (#103)

Browse Source 
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi 2024-03-18 22:23:13 +08:00 committed by GitHub
parent b45f648483
commit d752317b5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
router/src/validation.rs
View File

@ -97,6 +97,12 @@ impl Validation {
// Unwrap is safe here
let (inputs, input_length) = response_receiver.await.unwrap()?;
let input_length = if self.skip_tokenizer_in_tgi {
inputs.chars().filter(|&c| c == ',').count() + 1
} else {
truncate.unwrap_or(self.max_input_length)
};
// Get total tokens
let max_new_tokens: u32 = if let Some(max_new_tokens) = max_new_tokens {
max_new_tokens