huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 00:12:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better defaults (and LOG_COLORIZE).

Browse Source
This commit is contained in:
Nicolas Patry 2024-04-11 15:35:48 +00:00
parent 3c71d2f1c4
commit bd01d448d7
2 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
launcher/src/main.rs
View File

@ -1332,11 +1332,14 @@ fn main() -> Result<(), LauncherError> {
}
};
let max_batch_prefill_tokens = {
// TODO get config.
match args.max_batch_prefill_tokens {
Some(max_batch_prefill_tokens) => max_batch_prefill_tokens,
None => {
let value = config.max_position_embeddings as u32 - 1;
let value: u32 = if let Some(max_batch_size) = args.max_batch_size {
max_batch_size * max_input_tokens
} else {
max_input_tokens
} as u32;
tracing::info!("Default `max_batch_prefill_tokens` to {value}");
value
}

3
router/src/main.rs
View File

@ -402,12 +402,15 @@ async fn main() -> Result<(), RouterError> {
/// - otlp_endpoint is an optional URL to an Open Telemetry collector
/// - LOG_LEVEL may be TRACE, DEBUG, INFO, WARN or ERROR (default to INFO)
/// - LOG_FORMAT may be TEXT or JSON (default to TEXT)
/// - LOG_COLORIZE may be "false" or "true" (default to "true" or ansi supported platforms)
fn init_logging(otlp_endpoint: Option<String>, json_output: bool) {
let mut layers = Vec::new();
// STDOUT/STDERR layer
let ansi = std::env::var("LOG_COLORIZE") != Ok("1".to_string());
let fmt_layer = tracing_subscriber::fmt::layer()
.with_file(true)
.with_ansi(ansi)
.with_line_number(true);
let fmt_layer = match json_output {