diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 1320b1c8..2ba34732 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -1332,11 +1332,14 @@ fn main() -> Result<(), LauncherError> { } }; let max_batch_prefill_tokens = { - // TODO get config. match args.max_batch_prefill_tokens { Some(max_batch_prefill_tokens) => max_batch_prefill_tokens, None => { - let value = config.max_position_embeddings as u32 - 1; + let value: u32 = if let Some(max_batch_size) = args.max_batch_size { + max_batch_size * max_input_tokens + } else { + max_input_tokens + } as u32; tracing::info!("Default `max_batch_prefill_tokens` to {value}"); value } diff --git a/router/src/main.rs b/router/src/main.rs index 16a031ae..f3a6c46f 100644 --- a/router/src/main.rs +++ b/router/src/main.rs @@ -402,12 +402,15 @@ async fn main() -> Result<(), RouterError> { /// - otlp_endpoint is an optional URL to an Open Telemetry collector /// - LOG_LEVEL may be TRACE, DEBUG, INFO, WARN or ERROR (default to INFO) /// - LOG_FORMAT may be TEXT or JSON (default to TEXT) +/// - LOG_COLORIZE may be "false" or "true" (default to "true" or ansi supported platforms) fn init_logging(otlp_endpoint: Option, json_output: bool) { let mut layers = Vec::new(); // STDOUT/STDERR layer + let ansi = std::env::var("LOG_COLORIZE") != Ok("1".to_string()); let fmt_layer = tracing_subscriber::fmt::layer() .with_file(true) + .with_ansi(ansi) .with_line_number(true); let fmt_layer = match json_output {