huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 11:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix to pruning of cancelled requests from queue

Browse Source
This commit is contained in:
Nick Hill 2023-05-16 06:27:34 -07:00
parent c207b88b3a
commit 51355ea722
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
router/src/queue.rs
View File

@ -375,7 +375,7 @@ impl<B: BatchType> State<B> {
queue_index += 1; queue_index += 1;
if entry.response_tx.is_disconnected() { if entry.response_tx.is_disconnected() {
// Eject cancelled entry from queue // Eject cancelled entry from queue
indices_to_drop.push(queue_index); indices_to_drop.push(queue_index - 1);
continue continue
} }
// This is the index into the queue after cancelled entries // This is the index into the queue after cancelled entries
@ -465,7 +465,8 @@ impl<B: BatchType> State<B> {
// Drop any cancelled requests // Drop any cancelled requests
if !indices_to_drop.is_empty() { if !indices_to_drop.is_empty() {
indices_to_drop.iter().for_each(|i| { // Iterate in reverse so that indices remain correct
indices_to_drop.iter().rev().for_each(|i| {
self.entries.remove(*i); self.entries.remove(*i);
}); });
metrics::gauge!("tgi_queue_size", self.entries.len() as f64); metrics::gauge!("tgi_queue_size", self.entries.len() as f64);