From 291eaa99fbe8ff2a733070e48cc0d76b8c466c05 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 8 Aug 2024 00:33:10 +0200 Subject: [PATCH] use blocking_recv in looper to consume awaiting_requests at max before pulling in a single step --- backends/trtllm/src/looper.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/backends/trtllm/src/looper.rs b/backends/trtllm/src/looper.rs index 4bcf8a57..c287fa55 100644 --- a/backends/trtllm/src/looper.rs +++ b/backends/trtllm/src/looper.rs @@ -56,13 +56,10 @@ fn executor_status_poller( span!(Level::DEBUG, "[in-flight][submit]").in_scope(|| { // Is there any request pending to be scheduled? let awaiting_requests = waiting_requests.len(); - if awaiting_requests > 0 { + for _ in 0..awaiting_requests { // Retrieve all the requests - let mut requests = Vec::with_capacity(awaiting_requests); - let _ = waiting_requests.recv_many(&mut requests, awaiting_requests); - - // Submit all the request to the executor and move the context to the in-flight tracker - for ctx in requests { + if let Some(ctx) = waiting_requests.blocking_recv() { + // Submit all the request to the executor and move the context to the in-flight tracker let request = &ctx.request; let generation_params = &request.inner.parameters; let stopping_params = &request.inner.stopping_parameters;