huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: adjust protos and router

Browse Source
This commit is contained in:
drbh 2024-02-10 02:17:28 +00:00
parent ffc228831c
commit 8413cc45ec
6 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/src/lib.rs
View File

@ -45,7 +45,7 @@ pub async fn run(
repetition_penalty: repetition_penalty.unwrap_or(1.0),
frequency_penalty: frequency_penalty.unwrap_or(0.0),
watermark,
fsm_grammar_state: Vec::new(),
fsm_grammar_state: 0,
};
// Initialize terminal properties

4
proto/generate.proto
View File

@ -71,9 +71,9 @@ message NextTokenChooserParameters {
/// token watermarking using "A Watermark for Large Language Models"
bool watermark = 8;
/// grammar (applied if not empty)
repeated string grammar = 10;
string grammar = 10;
/// fsm_grammar_state
repeated uint32 fsm_grammar_state = 11;
uint32 fsm_grammar_state = 11;
}
message StoppingCriteriaParameters {

4
router/client/src/client.rs
View File

@ -128,8 +128,8 @@ impl Client {
repetition_penalty: 1.2,
frequency_penalty: 0.1,
watermark: true,
grammar: Vec::new(),
fsm_grammar_state: Vec::new(),
grammar: String::new(),
fsm_grammar_state: 0,
}),
stopping_parameters: Some(StoppingCriteriaParameters {
max_new_tokens: max_total_tokens - truncate,

4
router/src/health.rs
View File

@ -45,8 +45,8 @@ impl Health {
repetition_penalty: 1.0,
frequency_penalty: 0.0,
watermark: false,
grammar: Vec::new(),
fsm_grammar_state: Vec::new(),
grammar: String::new(),
fsm_grammar_state: 0,
}),
stopping_parameters: Some(StoppingCriteriaParameters {
max_new_tokens: 1,

4
router/src/queue.rs
View File

@ -368,8 +368,8 @@ mod tests {
repetition_penalty: 0.0,
frequency_penalty: 0.0,
watermark: false,
grammar: Vec::new(),
fsm_grammar_state: Vec::new(),
grammar: String::new(),
fsm_grammar_state: 0,
},
stopping_parameters: StoppingCriteriaParameters {
ignore_eos_token: false,

4
router/src/validation.rs
View File

@ -294,9 +294,9 @@ impl Validation {
.await?;
// initialize the grammar parameter
let grammar = vec![grammar];
let grammar = grammar;
// init the start state of the grammar
let fsm_grammar_state = vec![0];
let fsm_grammar_state = 0;
let parameters = NextTokenChooserParameters {
temperature,