huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 15:32:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add bucket for input seq len exactly same as --max-input-length (#178)

Browse Source
This commit is contained in:
Sun Choi 2024-07-05 01:30:26 -07:00 committed by GitHub
parent 1b4d80c03e
commit fff1d4f86f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
router/client/src/client.rs
View File

@ -137,8 +137,8 @@ impl Client {
let seq_bucket_size: u32 = read_env_var("PAD_SEQUENCE_TO_MULTIPLE_OF", 128); let seq_bucket_size: u32 = read_env_var("PAD_SEQUENCE_TO_MULTIPLE_OF", 128);
let mut seq_lengths: Vec<u32> = (seq_bucket_size..max_input_length+1).step_by(seq_bucket_size as usize).collect(); let mut seq_lengths: Vec<u32> = (seq_bucket_size..max_input_length+1).step_by(seq_bucket_size as usize).collect();
if let Some(&last) = seq_lengths.last() { if let Some(&last) = seq_lengths.last() {
if last < max_input_length { if last < (max_input_length + 1) {
seq_lengths.push(max_input_length); seq_lengths.push(max_input_length + 1);
} }
} }