huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding max capacity metric.

Browse Source
This commit is contained in:
Nicolas Patry 2024-10-01 16:34:13 +02:00
parent 3b0b66bedb
commit 6e7189f2aa
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
2 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backends/v3/src/lib.rs
View File

@ -100,6 +100,7 @@ pub async fn connect_backend(
.map_err(V3Error::Warmup)?,
)?;
tracing::info!("Setting max batch total tokens to {max_batch_total_tokens}");
metrics::gauge!("tgi_batch_max_total_tokens").set(max_batch_total_tokens);
let backend_info = BackendInfo {
waiting_served_ratio,

2
router/src/server.rs
View File

@ -1993,8 +1993,6 @@ async fn start(
"Batch size of the next batch"
);
metrics::counter!("tgi_max_token_capacity").increment(u64::from(max_batch_total_tokens));
// CORS layer
let allow_origin = allow_origin.unwrap_or(AllowOrigin::any());
let cors_layer = CorsLayer::new()