huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fmt.

Browse Source
This commit is contained in:
Nicolas Patry 2023-12-05 15:45:34 +00:00
parent cb8a1680fe
commit 5aa3a01971
1 changed files with 19 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
router/src/infer.rs
View File

@ -525,23 +525,22 @@ fn send_responses(
// Create last Token
let tokens_ = generation.tokens.expect("Non empty tokens in generation");
let n = tokens_.ids.len();
metrics::histogram!(
"tgi_request_skipped_tokens",
(n - 1) as f64
);
metrics::histogram!("tgi_request_skipped_tokens", (n - 1) as f64);
for (i, (((id, logprob), text), special)) in tokens_
.ids
.into_iter()
.zip(tokens_.logprobs.into_iter())
.zip(tokens_.texts.into_iter())
.zip(tokens_.is_special.into_iter()).enumerate() {
.zip(tokens_.is_special.into_iter())
.enumerate()
{
let token = Token {
id,
text,
logprob,
special,
};
let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i){
let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i) {
top_tokens_
.ids
.iter()
@ -553,12 +552,12 @@ fn send_responses(
text: text.to_string(),
logprob,
special,
}).collect()
}else{
})
.collect()
} else {
vec![]
};
match (&generation.generated_text, i){
match (&generation.generated_text, i) {
(Some(generated_text), i) if i == n - 1 => {
// Generation has ended
stopped = true;