huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-23 07:52:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

use blocking_recv in looper to consume awaiting_requests at max before pulling in a single step

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-08-08 00:33:10 +02:00
parent 7bebc629af
commit 291eaa99fb
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backends/trtllm/src/looper.rs
View File

@ -56,13 +56,10 @@ fn executor_status_poller(
span!(Level::DEBUG, "[in-flight][submit]").in_scope(|| {
// Is there any request pending to be scheduled?
let awaiting_requests = waiting_requests.len();
if awaiting_requests > 0 {
for _ in 0..awaiting_requests {
// Retrieve all the requests
let mut requests = Vec::with_capacity(awaiting_requests);
let _ = waiting_requests.recv_many(&mut requests, awaiting_requests);
if let Some(ctx) = waiting_requests.blocking_recv() {
// Submit all the request to the executor and move the context to the in-flight tracker
for ctx in requests {
let request = &ctx.request;
let generation_params = &request.inner.parameters;
let stopping_params = &request.inner.stopping_parameters;