From 61ff239724b62e33c691704f0112fb8e53d4ca41 Mon Sep 17 00:00:00 2001 From: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com> Date: Mon, 24 Apr 2023 16:19:54 +0200 Subject: [PATCH] refactor --- router/src/infer.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index e1f48fe3..8b44ec86 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -267,12 +267,13 @@ async fn batching_task( metrics::gauge!("tgi_batch_current_size", batch_size as f64); metrics::gauge!("tgi_batch_current_max_tokens", batch_max_tokens as f64); - let min_size = match waiting_tokens { + let min_size = if waiting_tokens >= max_waiting_tokens { // If we didn't onboard any new requests since >= max_waiting_tokens, we try // to add a new batch even though its size might be small - _ if waiting_tokens >= max_waiting_tokens => None, + None + } else { // Minimum batch size - _ => Some((batch_size as f32 * waiting_served_ratio).floor() as usize), + Some((batch_size as f32 * waiting_served_ratio).floor() as usize) }; let token_budget = max_batch_total_tokens - batch_max_tokens;