From dff8f7cde74cc8ff03ed64bcae9e0031174d632d Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 16 Jan 2023 21:59:37 -0800 Subject: [PATCH] Obey max batch size --- router/src/batcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/src/batcher.rs b/router/src/batcher.rs index 1484434c..c44dbc5b 100644 --- a/router/src/batcher.rs +++ b/router/src/batcher.rs @@ -128,7 +128,7 @@ async fn batching_task( // Try to get a new batch if let Some((new_request_ids, new_batch)) = - db.next_batch(min_size, max_batch_size) + db.next_batch(min_size, max_batch_size - batch_size as usize) { // Generate one token for this new batch to have the attention past in cache let new_cached_batch =