huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-09 19:34:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use different id for batch vs liveness

Browse Source
This commit is contained in:
ehsanmok 2023-04-27 09:00:14 -07:00
parent 080542dbca
commit 6f608f0d84
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
router/src/health.rs
View File

@ -5,6 +5,7 @@ use text_generation_client::{
}; };
const LIVENESS_ID: u64 = u64::MAX; const LIVENESS_ID: u64 = u64::MAX;
const BATCH_ID: u64 = u64::MAX - 1;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(crate) struct Health { pub(crate) struct Health {
@ -49,7 +50,7 @@ impl Health {
}), }),
}; };
let batch = Batch { let batch = Batch {
id: u64::MAX, id: BATCH_ID,
requests: vec![liveness_request], requests: vec![liveness_request],
size: 1, size: 1,
max_tokens: 2, max_tokens: 2,