huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ignore_eos_token to HTTP interface

Browse Source
This commit is contained in:
Chirag Jain 2023-10-06 18:22:12 +05:30
parent 00b8f36fba
commit 017f3de464
No known key found for this signature in database
GPG Key ID: D6A7538AE0A6D9AF
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
router/src/lib.rs
View File

@ -138,6 +138,9 @@ pub(crate) struct GenerateParameters {
#[serde(default)]
#[schema(exclusive_minimum = 0, nullable = true, default = "null", example = 5)]
pub top_n_tokens: Option<u32>,
#[serde(default)]
#[schema(default = "false")]
pub ignore_eos_token: bool
}
fn default_max_new_tokens() -> Option<u32> {
@ -162,6 +165,7 @@ fn default_parameters() -> GenerateParameters {
decoder_input_details: false,
seed: None,
top_n_tokens: None,
ignore_eos_token: false,
}
}

3
router/src/validation.rs
View File

@ -156,6 +156,7 @@ impl Validation {
watermark,
decoder_input_details,
top_n_tokens,
ignore_eos_token,
..
} = request.parameters;
@ -274,7 +275,7 @@ impl Validation {
let stopping_parameters = StoppingCriteriaParameters {
max_new_tokens,
stop_sequences,
ignore_eos_token: false,
ignore_eos_token: ignore_eos_token,
};
metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64);