huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fmt

Browse Source
This commit is contained in:
OlivierDehaene 2023-10-20 09:47:53 +02:00
parent d20576ae0c
commit 47e0620ab6
3 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
router/client/src/client.rs
View File

@ -128,7 +128,7 @@ impl Client {
watermark: true,
}),
stopping_parameters: Some(StoppingCriteriaParameters {
max_new_tokens: max_total_tokens-truncate,
max_new_tokens: max_total_tokens - truncate,
stop_sequences: vec![],
ignore_eos_token: true,
}),

4
router/client/src/sharded_client.rs
View File

@ -100,7 +100,9 @@ impl ShardedClient {
let futures: Vec<_> = self
.clients
.iter_mut()
.map(|client| Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens)))
.map(|client| {
Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens))
})
.collect();
// Take the minimum value
let results = join_all(futures)

2
router/src/validation.rs
View File

@ -122,7 +122,7 @@ impl Validation {
if let Some(truncate) = truncate {
self.max_total_tokens.saturating_sub(truncate) as u32
} else {
return Err(ValidationError::UnsetMaxNewTokens)
return Err(ValidationError::UnsetMaxNewTokens);
}
};
let input_length = truncate.unwrap_or(self.max_input_length);