fix: update v3 scheduler and ensure max_batch_size > 0

backends/v3/src/backend.rs

@@ -168,7 +168,8 @@ pub(crate) async fn batching_task(
                 };
 
                 let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
-                let max_size = max_batch_size.map(|max_size| max_size - batch_size as usize);
+                let max_size =
+                    max_batch_size.map(|max_size| max_size.saturating_sub(batch_size as usize));
 
                 // Try to get a new batch
                 if let Some((mut new_entries, new_batch, span)) = queue

backends/v3/src/main.rs

@@ -150,6 +150,14 @@ async fn main() -> Result<(), RouterError> {
         }
     }
 
+    if let Some(max_batch_size) = max_batch_size {
+        if max_batch_size == 0 {
+            return Err(RouterError::ArgumentValidation(
+                "`max_batch_size` must be > 0".to_string(),
+            ));
+        }
+    }
+
     let (backend, _backend_info) = connect_backend(
         max_input_tokens,
         max_total_tokens,

backends/v3/src/queue.rs

@@ -226,6 +226,13 @@ impl State {
             }
         }
 
+        if let Some(max_size) = max_size {
+            if max_size == 0 {
+                tracing::debug!("No capacity");
+                return None;
+            }
+        }
+
         // Pad prefill_token_budget to be a multiple of block size
         let prefill_token_budget =
             ((prefill_token_budget + self.block_size - 1) / self.block_size) * self.block_size;

router/src/infer/v2/queue.rs

@@ -304,7 +304,7 @@ impl State {
             batch_entries.insert(id, entry);
 
             // Check if max_size
-            if Some(batch_requests.len()) >= max_size {
+            if Some(batch_requests.len()) == max_size {
                 break;
             }
         }

router/src/infer/v2/scheduler.rs

@@ -161,11 +161,8 @@ pub(crate) async fn batching_task(
                 };
 
                 let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
-                let max_size = max_batch_size.map(|max_size| {
+                let max_size =
-                    if batch_size as usize > max_size { 0 } else { max_size - batch_size as usize }
+                    max_batch_size.map(|max_size| max_size.saturating_sub(batch_size as usize));
-                });
-
-
                 // Try to get a new batch
                 if let Some((mut new_entries, new_batch, span)) = queue
                     .next_batch(min_size, max_size, max_batch_prefill_tokens, token_budget)