huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-09 19:34:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

small change to handling of cancelled requests

Browse Source
This commit is contained in:
Nick Hill 2023-05-02 12:53:13 +01:00
parent 7552483dde
commit c207b88b3a
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
router/src/queue.rs
View File

@ -332,11 +332,10 @@ impl<B: BatchType> State<B> {
self.last_seen_batch_size = total_count
}
// Filter entries where the response receiver was dropped (== entries where the request
// was dropped by the client)
let queue_len_before = self.entries.len();
self.entries.retain_mut(|(_, entry)| !entry.response_tx.is_disconnected());
if queue_len_before != self.entries.len() {
// Filter cancelled entries from the front of the queue,
// so that next-entry waiting time is accurate
while matches!(self.entries.front(), Some((_,entry)) if entry.response_tx.is_disconnected()) {
self.entries.pop_front();
// Reset the count of checked requests if any in the queue were cancelled since last check
self.checked_request_count = 0;
}
@ -354,7 +353,7 @@ impl<B: BatchType> State<B> {
}
}
// Indices into buffer of entries chosen to add to next batch or remove due to expiry
// Indices into buffer of entries chosen to add to next batch
let mut chosen_indices = vec![];
// Indices to drop due to client cancellation
let mut indices_to_drop = vec![];