huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fmt.

Browse Source
This commit is contained in:
Nicolas Patry 2023-12-05 15:45:34 +00:00
parent cb8a1680fe
commit 5aa3a01971
1 changed files with 19 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
router/src/infer.rs
View File

@ -525,16 +525,15 @@ fn send_responses(
// Create last Token // Create last Token
let tokens_ = generation.tokens.expect("Non empty tokens in generation"); let tokens_ = generation.tokens.expect("Non empty tokens in generation");
let n = tokens_.ids.len(); let n = tokens_.ids.len();
metrics::histogram!( metrics::histogram!("tgi_request_skipped_tokens", (n - 1) as f64);
"tgi_request_skipped_tokens",
(n - 1) as f64
);
for (i, (((id, logprob), text), special)) in tokens_ for (i, (((id, logprob), text), special)) in tokens_
.ids .ids
.into_iter() .into_iter()
.zip(tokens_.logprobs.into_iter()) .zip(tokens_.logprobs.into_iter())
.zip(tokens_.texts.into_iter()) .zip(tokens_.texts.into_iter())
.zip(tokens_.is_special.into_iter()).enumerate() { .zip(tokens_.is_special.into_iter())
.enumerate()
{
let token = Token { let token = Token {
id, id,
text, text,
@ -553,10 +552,10 @@ fn send_responses(
text: text.to_string(), text: text.to_string(),
logprob, logprob,
special, special,
}).collect() })
.collect()
} else { } else {
vec![] vec![]
}; };
match (&generation.generated_text, i) { match (&generation.generated_text, i) {
(Some(generated_text), i) if i == n - 1 => { (Some(generated_text), i) if i == n - 1 => {