huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-27 21:12:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Truncating left for radix purposes.

Browse Source
This commit is contained in:
Nicolas Patry 2024-08-28 10:53:22 +02:00
parent 0a60973166
commit e6ee67f301
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
router/src/validation.rs
View File

@ -167,7 +167,8 @@ impl Validation {
));
}
let input_ids = encoding.get_ids()[..input_length].to_owned();
let ids = encoding.get_ids();
let input_ids = ids[ids.len().saturating_sub(input_length)..].to_owned();
metrics::histogram!("tgi_request_input_length").record(input_length as f64);
Ok((inputs, Some(input_ids), input_length, max_new_tokens))

3
server/text_generation_server/models/flash_causal_lm.py
View File

@ -272,6 +272,9 @@ class FlashCausalLMBatch(Batch):
prefix_len = r.prefix_len
assert prefix_len <= orig_input_length
if prefix_len == orig_input_length:
assert prefix_len > 0
prefix_len -= 1
prefix_ids.append(tokenized_input[:prefix_len])
tokenized_input = tokenized_input[prefix_len:]