huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-19 22:02:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

backend(vllm): disable metrics for now

Browse Source
This commit is contained in:
Morgan Funtowicz 2025-01-31 10:56:54 +01:00
parent 003163a2b9
commit 5452c1294c
2 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backends/vllm/src/backend.rs
View File

@ -63,6 +63,12 @@ impl TryFrom<&RequestOutput> for InferStreamResponse {
.unwrap(),
seed: None,
},
// start: STARTUP_INSTANT
// .checked_sub(Duration::from_secs_f32(metrics.first_scheduled_time))
// .unwrap_or_else(Instant::now),
// queued: STARTUP_INSTANT
// .checked_sub(Duration::from_secs_f32(metrics.arrival_time))
// .unwrap_or_else(Instant::now),
start: Instant::now(),
queued: Instant::now(),
})

2
backends/vllm/src/lib.rs
View File

@ -11,7 +11,7 @@ use pyo3::types::PyModule;
use pyo3::{Py, PyAny, PyErr, PyObject, Python};
use tokio::time::Instant;
pub(crate) const STARTUP_INSTANT: Instant = Instant::now();
pub(crate) static STARTUP_INSTANT: Instant = Instant::now();
static PY_TOKENS_PROMPT_CLASS: GILOnceCell<Py<PyAny>> = GILOnceCell::new();
static PY_SAMPLING_PARAMS_CLASS: GILOnceCell<Py<PyAny>> = GILOnceCell::new();