diff --git a/router/src/main.rs b/router/src/main.rs index 03e920c6..6449e647 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -220,8 +220,14 @@ fn main() -> Result<(), RouterError> { .map_err(RouterError::Warmup)? { // Older models do not support automatic max-batch-total-tokens - None => max_batch_total_tokens - .unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens))), + None => { + let max_batch_total_tokens = max_batch_total_tokens.unwrap_or( + 16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens)), + ); + tracing::warn!("Model does not support automatic max batch total tokens"); + tracing::warn!("Setting max batch total tokens to {max_batch_total_tokens}"); + max_batch_total_tokens + } // Flash attention models return their max supported total tokens Some(max_supported_batch_total_tokens) => { // Warn if user added his own max-batch-total-tokens as we will ignore it