huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 11:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cargo fmt.

Browse Source
This commit is contained in:
Nicolas Patry 2023-07-31 12:42:18 +00:00
parent edbba4ea36
commit aa44a3d1f0
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
launcher/src/main.rs
View File

@ -61,7 +61,7 @@ impl std::fmt::Display for Dtype {
} }
#[derive(Clone, Copy, Debug, ValueEnum)] #[derive(Clone, Copy, Debug, ValueEnum)]
enum RopeScaling{ enum RopeScaling {
Linear, Linear,
Dynamic, Dynamic,
} }
@ -271,11 +271,11 @@ struct Args {
cuda_memory_fraction: f32, cuda_memory_fraction: f32,
/// Rope scaling will only be used for RoPE models /// Rope scaling will only be used for RoPE models
/// and allow rescaling the position rotary to accomodate for /// and allow rescaling the position rotary to accomodate for
/// larger prompts. /// larger prompts.
/// ///
/// Goes together with `rope_factor`. /// Goes together with `rope_factor`.
/// ///
/// `--rope-factor 2.0` gives linear scaling with a factor of 2.0 /// `--rope-factor 2.0` gives linear scaling with a factor of 2.0
/// `--rope-scaling dynamic` gives dynamic scaling with a factor of 1.0 /// `--rope-scaling dynamic` gives dynamic scaling with a factor of 1.0
/// `--rope-scaling linear` gives linear scaling with a factor of 1.0 (Nothing will be changed /// `--rope-scaling linear` gives linear scaling with a factor of 1.0 (Nothing will be changed
@ -290,7 +290,6 @@ struct Args {
#[clap(long, env)] #[clap(long, env)]
rope_factor: Option<f32>, rope_factor: Option<f32>,
/// Outputs the logs in JSON format (useful for telemetry) /// Outputs the logs in JSON format (useful for telemetry)
#[clap(long, env)] #[clap(long, env)]
json_output: bool, json_output: bool,
@ -448,12 +447,11 @@ fn shard_manager(
// Sending as env instead of CLI args to not bloat everything // Sending as env instead of CLI args to not bloat everything
// those only can be used by RoPE models, so passing information around // those only can be used by RoPE models, so passing information around
// for all models will complexify code unnecessarily // for all models will complexify code unnecessarily
if let Some((scaling, factor)) = rope{ if let Some((scaling, factor)) = rope {
envs.push(("ROPE_SCALING".into(), scaling.to_string().into())); envs.push(("ROPE_SCALING".into(), scaling.to_string().into()));
envs.push(("ROPE_FACTOR".into(), factor.to_string().into())); envs.push(("ROPE_FACTOR".into(), factor.to_string().into()));
} }
// If huggingface_hub_cache is some, pass it to the shard // If huggingface_hub_cache is some, pass it to the shard
// Useful when running inside a docker container // Useful when running inside a docker container
if let Some(huggingface_hub_cache) = huggingface_hub_cache { if let Some(huggingface_hub_cache) = huggingface_hub_cache {