huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-21 14:52:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(backend) fix moving backend when pulling

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-12-04 17:32:14 +01:00
parent 460f290d5b
commit 300f6c6f94
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backends/trtllm/src/looper.rs
View File

@ -121,8 +121,7 @@ fn executor_status_looper(
} }
if backend.num_tokens_ready() > 0 { if backend.num_tokens_ready() > 0 {
let backend = backend.pin_mut(); match backend.pin_mut().pull_tokens() {
match backend.pull_tokens() {
Ok(responses) => { Ok(responses) => {
// Iterate through all the decoded token // Iterate through all the decoded token
for step in responses.deref() { for step in responses.deref() {
@ -141,7 +140,7 @@ fn executor_status_looper(
if posted.is_err() || step.is_final { if posted.is_err() || step.is_final {
debug!("Removing {}", step.request_id); debug!("Removing {}", step.request_id);
backend.cancel(step.request_id); backend.pin_mut().cancel(step.request_id);
let _ = in_flights.remove(&step.request_id); let _ = in_flights.remove(&step.request_id);
} }
} else { } else {