huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Propagate speculate

Browse Source
This commit is contained in:
Nicolas Patry 2023-12-04 13:50:59 +00:00
parent 7ed07bcc05
commit bdd9596b6c
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
launcher/src/main.rs
View File

@ -383,6 +383,7 @@ fn shard_manager(
model_id: String, model_id: String,
revision: Option<String>, revision: Option<String>,
quantize: Option<Quantization>, quantize: Option<Quantization>,
speculate: Option<usize>,
dtype: Option<Dtype>, dtype: Option<Dtype>,
trust_remote_code: bool, trust_remote_code: bool,
uds_path: String, uds_path: String,
@ -895,6 +896,7 @@ fn spawn_shards(
let shutdown_sender = shutdown_sender.clone(); let shutdown_sender = shutdown_sender.clone();
let otlp_endpoint = args.otlp_endpoint.clone(); let otlp_endpoint = args.otlp_endpoint.clone();
let quantize = args.quantize; let quantize = args.quantize;
let speculate = args.speculate;
let dtype = args.dtype; let dtype = args.dtype;
let trust_remote_code = args.trust_remote_code; let trust_remote_code = args.trust_remote_code;
let master_port = args.master_port; let master_port = args.master_port;
@ -909,6 +911,7 @@ fn spawn_shards(
model_id, model_id,
revision, revision,
quantize, quantize,
speculate,
dtype, dtype,
trust_remote_code, trust_remote_code,
uds_path, uds_path,