huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-21 06:42:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(backend) fix moving backend when pulling

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-12-04 17:32:14 +01:00
parent 460f290d5b
commit 300f6c6f94
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backends/trtllm/src/looper.rs
View File

@ -121,8 +121,7 @@ fn executor_status_looper(
}
if backend.num_tokens_ready() > 0 {
let backend = backend.pin_mut();
match backend.pull_tokens() {
match backend.pin_mut().pull_tokens() {
Ok(responses) => {
// Iterate through all the decoded token
for step in responses.deref() {
@ -141,7 +140,7 @@ fn executor_status_looper(
if posted.is_err() || step.is_final {
debug!("Removing {}", step.request_id);
backend.cancel(step.request_id);
backend.pin_mut().cancel(step.request_id);
let _ = in_flights.remove(&step.request_id);
}
} else {