From 4cbba331399c3dcbddd397218500c9e9d1ac1fe8 Mon Sep 17 00:00:00 2001 From: "Wang, Yi" Date: Thu, 21 Nov 2024 22:06:26 +0800 Subject: [PATCH] Incomplete generation stream fix (#2754) entries.len() could > batch.size in prefill, so need to filter as well. Signed-off-by: Wang, Yi A --- backends/v3/src/backend.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/backends/v3/src/backend.rs b/backends/v3/src/backend.rs index a5c0f512..d83a38e6 100644 --- a/backends/v3/src/backend.rs +++ b/backends/v3/src/backend.rs @@ -369,11 +369,6 @@ async fn filter_batch( ) -> Option { let mut batch = next_batch?; - // No need to filter - if batch.size as usize == entries.len() { - return Some(batch); - } - let id = batch.id; // Retain only requests that are still in entries