huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix default value

Browse Source
This commit is contained in:
OlivierDehaene 2023-07-18 11:39:14 +02:00
parent b165f8b7b7
commit 4201a8be46
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
router/src/main.rs
View File

@ -220,7 +220,8 @@ fn main() -> Result<(), RouterError> {
.map_err(RouterError::Warmup)? .map_err(RouterError::Warmup)?
{ {
// Older models do not support automatic max-batch-total-tokens // Older models do not support automatic max-batch-total-tokens
None => max_batch_total_tokens.unwrap_or(16000), None => max_batch_total_tokens
.unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens))),
// Flash attention models return their max supported total tokens // Flash attention models return their max supported total tokens
Some(max_supported_batch_total_tokens) => { Some(max_supported_batch_total_tokens) => {
// Warn if user added his own max-batch-total-tokens as we will ignore it // Warn if user added his own max-batch-total-tokens as we will ignore it