huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fmt.

Browse Source
This commit is contained in:
Nicolas Patry 2023-12-05 15:45:34 +00:00
parent cb8a1680fe
commit 5aa3a01971
1 changed files with 19 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
router/src/infer.rs
View File

@ -525,23 +525,22 @@ fn send_responses(
// Create last Token // Create last Token
let tokens_ = generation.tokens.expect("Non empty tokens in generation"); let tokens_ = generation.tokens.expect("Non empty tokens in generation");
let n = tokens_.ids.len(); let n = tokens_.ids.len();
metrics::histogram!( metrics::histogram!("tgi_request_skipped_tokens", (n - 1) as f64);
"tgi_request_skipped_tokens",
(n - 1) as f64
);
for (i, (((id, logprob), text), special)) in tokens_ for (i, (((id, logprob), text), special)) in tokens_
.ids .ids
.into_iter() .into_iter()
.zip(tokens_.logprobs.into_iter()) .zip(tokens_.logprobs.into_iter())
.zip(tokens_.texts.into_iter()) .zip(tokens_.texts.into_iter())
.zip(tokens_.is_special.into_iter()).enumerate() { .zip(tokens_.is_special.into_iter())
.enumerate()
{
let token = Token { let token = Token {
id, id,
text, text,
logprob, logprob,
special, special,
}; };
let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i){ let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i) {
top_tokens_ top_tokens_
.ids .ids
.iter() .iter()
@ -553,12 +552,12 @@ fn send_responses(
text: text.to_string(), text: text.to_string(),
logprob, logprob,
special, special,
}).collect() })
}else{ .collect()
} else {
vec![] vec![]
}; };
match (&generation.generated_text, i){ match (&generation.generated_text, i) {
(Some(generated_text), i) if i == n - 1 => { (Some(generated_text), i) if i == n - 1 => {
// Generation has ended // Generation has ended
stopped = true; stopped = true;