From 300f6c6f94fb8ad76b019c37e2bc4e76b4222083 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 4 Dec 2024 17:32:14 +0100 Subject: [PATCH] feat(backend) fix moving backend when pulling --- backends/trtllm/src/looper.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backends/trtllm/src/looper.rs b/backends/trtllm/src/looper.rs index 0a4499b5d..272a04c09 100644 --- a/backends/trtllm/src/looper.rs +++ b/backends/trtllm/src/looper.rs @@ -121,8 +121,7 @@ fn executor_status_looper( } if backend.num_tokens_ready() > 0 { - let backend = backend.pin_mut(); - match backend.pull_tokens() { + match backend.pin_mut().pull_tokens() { Ok(responses) => { // Iterate through all the decoded token for step in responses.deref() { @@ -141,7 +140,7 @@ fn executor_status_looper( if posted.is_err() || step.is_final { debug!("Removing {}", step.request_id); - backend.cancel(step.request_id); + backend.pin_mut().cancel(step.request_id); let _ = in_flights.remove(&step.request_id); } } else {