huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove duplicate input_length on Details

Browse Source
This commit is contained in:
drbh 2024-01-11 12:10:34 -05:00
parent c63551fad7
commit 4555e8721c
2 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
router/src/lib.rs
View File

@ -232,9 +232,9 @@ impl ChatCompletion {
finish_reason: details.finish_reason.to_string(), finish_reason: details.finish_reason.to_string(),
}], }],
usage: Usage { usage: Usage {
prompt_tokens: details.input_length, prompt_tokens: details.prefill.len() as u32,
completion_tokens: details.generated_tokens, completion_tokens: details.generated_tokens,
total_tokens: details.input_length + details.generated_tokens, total_tokens: details.prefill.len() as u32 + details.generated_tokens,
}, },
} }
} }
@ -469,8 +469,6 @@ pub(crate) struct Details {
pub best_of_sequences: Option<Vec<BestOfSequence>>, pub best_of_sequences: Option<Vec<BestOfSequence>>,
#[serde(skip_serializing_if = "Vec::is_empty")] #[serde(skip_serializing_if = "Vec::is_empty")]
pub top_tokens: Vec<Vec<Token>>, pub top_tokens: Vec<Vec<Token>>,
#[schema(example = 1)]
pub input_length: u32,
} }
#[derive(Serialize, ToSchema)] #[derive(Serialize, ToSchema)]

3
router/src/server.rs
View File

@ -208,7 +208,6 @@ async fn generate(
seed: response.generated_text.seed, seed: response.generated_text.seed,
best_of_sequences, best_of_sequences,
top_tokens: response.top_tokens, top_tokens: response.top_tokens,
input_length: response.input_length,
}) })
} }
false => None, false => None,
@ -604,7 +603,7 @@ async fn chat_completions(
truncate: None, truncate: None,
watermark: false, watermark: false,
details: true, details: true,
decoder_input_details: false, decoder_input_details: true,
seed, seed,
top_n_tokens: None, top_n_tokens: None,
}, },