huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fmt

Browse Source
This commit is contained in:
OlivierDehaene 2023-10-20 09:47:53 +02:00
parent d20576ae0c
commit 47e0620ab6
3 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
router/client/src/client.rs
View File

@ -128,7 +128,7 @@ impl Client {
watermark: true, watermark: true,
}), }),
stopping_parameters: Some(StoppingCriteriaParameters { stopping_parameters: Some(StoppingCriteriaParameters {
max_new_tokens: max_total_tokens-truncate, max_new_tokens: max_total_tokens - truncate,
stop_sequences: vec![], stop_sequences: vec![],
ignore_eos_token: true, ignore_eos_token: true,
}), }),

4
router/client/src/sharded_client.rs
View File

@ -100,7 +100,9 @@ impl ShardedClient {
let futures: Vec<_> = self let futures: Vec<_> = self
.clients .clients
.iter_mut() .iter_mut()
.map(|client| Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens))) .map(|client| {
Box::pin(client.warmup(max_input_length, max_prefill_tokens, max_total_tokens))
})
.collect(); .collect();
// Take the minimum value // Take the minimum value
let results = join_all(futures) let results = join_all(futures)

2
router/src/validation.rs
View File

@ -122,7 +122,7 @@ impl Validation {
if let Some(truncate) = truncate { if let Some(truncate) = truncate {
self.max_total_tokens.saturating_sub(truncate) as u32 self.max_total_tokens.saturating_sub(truncate) as u32
} else { } else {
return Err(ValidationError::UnsetMaxNewTokens) return Err(ValidationError::UnsetMaxNewTokens);
} }
}; };
let input_length = truncate.unwrap_or(self.max_input_length); let input_length = truncate.unwrap_or(self.max_input_length);