huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

my b

Browse Source
This commit is contained in:
OlivierDehaene 2024-02-08 17:28:54 +01:00
parent 2af011a1c0
commit 55e29c9564
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
router/src/infer.rs
View File

@ -387,7 +387,7 @@ async fn batching_task(
}; };
let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens); let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
let max_size = max_batch_size.map(|max_size| batch_size as usize - max_size); let max_size = max_batch_size.map(|max_size| max_size - batch_size as usize);
// Try to get a new batch // Try to get a new batch
if let Some((mut new_entries, new_batch, span)) = queue if let Some((mut new_entries, new_batch, span)) = queue