huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: adjust protos and router

Browse Source
This commit is contained in:
drbh 2024-02-10 02:17:28 +00:00
parent ffc228831c
commit 8413cc45ec
6 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/src/lib.rs
View File

@ -45,7 +45,7 @@ pub async fn run(
repetition_penalty: repetition_penalty.unwrap_or(1.0), repetition_penalty: repetition_penalty.unwrap_or(1.0),
frequency_penalty: frequency_penalty.unwrap_or(0.0), frequency_penalty: frequency_penalty.unwrap_or(0.0),
watermark, watermark,
fsm_grammar_state: Vec::new(), fsm_grammar_state: 0,
}; };
// Initialize terminal properties // Initialize terminal properties

4
proto/generate.proto
View File

@ -71,9 +71,9 @@ message NextTokenChooserParameters {
/// token watermarking using "A Watermark for Large Language Models" /// token watermarking using "A Watermark for Large Language Models"
bool watermark = 8; bool watermark = 8;
/// grammar (applied if not empty) /// grammar (applied if not empty)
repeated string grammar = 10; string grammar = 10;
/// fsm_grammar_state /// fsm_grammar_state
repeated uint32 fsm_grammar_state = 11; uint32 fsm_grammar_state = 11;
} }
message StoppingCriteriaParameters { message StoppingCriteriaParameters {

4
router/client/src/client.rs
View File

@ -128,8 +128,8 @@ impl Client {
repetition_penalty: 1.2, repetition_penalty: 1.2,
frequency_penalty: 0.1, frequency_penalty: 0.1,
watermark: true, watermark: true,
grammar: Vec::new(), grammar: String::new(),
fsm_grammar_state: Vec::new(), fsm_grammar_state: 0,
}), }),
stopping_parameters: Some(StoppingCriteriaParameters { stopping_parameters: Some(StoppingCriteriaParameters {
max_new_tokens: max_total_tokens - truncate, max_new_tokens: max_total_tokens - truncate,

4
router/src/health.rs
View File

@ -45,8 +45,8 @@ impl Health {
repetition_penalty: 1.0, repetition_penalty: 1.0,
frequency_penalty: 0.0, frequency_penalty: 0.0,
watermark: false, watermark: false,
grammar: Vec::new(), grammar: String::new(),
fsm_grammar_state: Vec::new(), fsm_grammar_state: 0,
}), }),
stopping_parameters: Some(StoppingCriteriaParameters { stopping_parameters: Some(StoppingCriteriaParameters {
max_new_tokens: 1, max_new_tokens: 1,

4
router/src/queue.rs
View File

@ -368,8 +368,8 @@ mod tests {
repetition_penalty: 0.0, repetition_penalty: 0.0,
frequency_penalty: 0.0, frequency_penalty: 0.0,
watermark: false, watermark: false,
grammar: Vec::new(), grammar: String::new(),
fsm_grammar_state: Vec::new(), fsm_grammar_state: 0,
}, },
stopping_parameters: StoppingCriteriaParameters { stopping_parameters: StoppingCriteriaParameters {
ignore_eos_token: false, ignore_eos_token: false,

4
router/src/validation.rs
View File

@ -294,9 +294,9 @@ impl Validation {
.await?; .await?;
// initialize the grammar parameter // initialize the grammar parameter
let grammar = vec![grammar]; let grammar = grammar;
// init the start state of the grammar // init the start state of the grammar
let fsm_grammar_state = vec![0]; let fsm_grammar_state = 0;
let parameters = NextTokenChooserParameters { let parameters = NextTokenChooserParameters {
temperature, temperature,