huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

pad to block size

Browse Source
This commit is contained in:
OlivierDehaene 2023-07-18 12:04:38 +02:00
parent 086d0c2252
commit d2e3843588
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
router/src/queue.rs
View File

@ -187,10 +187,17 @@ impl State {
max_input_length = max_input_length.max(entry.request.input_length);
prefill_tokens = (batch_requests.len() + 1) as u32 * max_input_length
} else {
prefill_tokens += entry.request.input_length;
// pad to block size
prefill_tokens += ((entry.request.input_length + 16 - 1) / 16) * 16;
}
if self.requires_padding {
decode_tokens += entry.request.stopping_parameters.max_new_tokens;
} else {
// pad to block size
decode_tokens +=
((entry.request.stopping_parameters.max_new_tokens + 16 - 1) / 16) * 16;
}
if prefill_tokens > prefill_token_budget
|| (prefill_tokens + decode_tokens) > token_budget