huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cargo fmt

Browse Source
This commit is contained in:
Nicolas Patry 2023-12-04 14:56:38 +00:00
parent d99f281050
commit 269792094b
3 changed files with 34 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
launcher/src/main.rs
View File

@ -162,7 +162,6 @@ struct Args {
#[clap(long, env)] #[clap(long, env)]
speculate: Option<usize>, speculate: Option<usize>,
/// The dtype to be forced upon the model. This option cannot be used with `--quantize`. /// The dtype to be forced upon the model. This option cannot be used with `--quantize`.
#[clap(long, env, value_enum)] #[clap(long, env, value_enum)]
dtype: Option<Dtype>, dtype: Option<Dtype>,

2
router/client/src/lib.rs
View File

@ -10,7 +10,7 @@ pub use pb::generate::v1::HealthResponse;
pub use pb::generate::v1::InfoResponse as ShardInfo; pub use pb::generate::v1::InfoResponse as ShardInfo;
pub use pb::generate::v1::{ pub use pb::generate::v1::{
Batch, CachedBatch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters, Batch, CachedBatch, FinishReason, GeneratedText, Generation, NextTokenChooserParameters,
Tokens, Request, StoppingCriteriaParameters, Request, StoppingCriteriaParameters, Tokens,
}; };
pub use sharded_client::ShardedClient; pub use sharded_client::ShardedClient;
use thiserror::Error; use thiserror::Error;

9
router/src/infer.rs
View File

@ -9,7 +9,7 @@ use std::sync::{
Arc, Arc,
}; };
use text_generation_client::{ use text_generation_client::{
Batch, CachedBatch, ClientError, GeneratedText, Generation, Tokens, ShardedClient, Batch, CachedBatch, ClientError, GeneratedText, Generation, ShardedClient, Tokens,
}; };
use thiserror::Error; use thiserror::Error;
use tokio::sync::mpsc::error::SendError; use tokio::sync::mpsc::error::SendError;
@ -525,7 +525,9 @@ fn send_responses(
// Create last Token // Create last Token
let tokens: Vec<Token> = if let Some(tokens_) = generation.tokens { let tokens: Vec<Token> = if let Some(tokens_) = generation.tokens {
tokens_.ids.into_iter() tokens_
.ids
.into_iter()
.zip(tokens_.logprobs.into_iter()) .zip(tokens_.logprobs.into_iter())
.zip(tokens_.texts.into_iter()) .zip(tokens_.texts.into_iter())
.zip(tokens_.is_special.into_iter()) .zip(tokens_.is_special.into_iter())
@ -534,7 +536,8 @@ fn send_responses(
text, text,
logprob, logprob,
special, special,
}).collect() })
.collect()
} else { } else {
vec![] vec![]
}; };