huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 08:22:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge b9467b95a0 into 095775e05c

Browse Source
This commit is contained in:
jrc2139 2025-03-17 14:43:47 -04:00 committed by GitHub
commit 3ca71c6422
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 23 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
router/src/server.rs
View File

@ -294,10 +294,10 @@ pub(crate) async fn generate_internal(
); );
let compute_characters = req.inputs.chars().count(); let compute_characters = req.inputs.chars().count();
let mut add_prompt = None; //let mut add_prompt = None;
if req.parameters.return_full_text.unwrap_or(false) { //if req.parameters.return_full_text.unwrap_or(false) {
add_prompt = Some(req.inputs.clone()); // add_prompt = Some(req.inputs.clone());
} //}
let details: bool = req.parameters.details || req.parameters.decoder_input_details; let details: bool = req.parameters.details || req.parameters.decoder_input_details;
@ -320,10 +320,10 @@ pub(crate) async fn generate_internal(
.into_iter() .into_iter()
.map(|response: InferResponse| { .map(|response: InferResponse| {
// Add prompt if return_full_text // Add prompt if return_full_text
let mut output_text = response.generated_text.text; let output_text = response.generated_text.text.clone();
if let Some(prompt) = &add_prompt { //if let Some(prompt) = &add_prompt {
output_text = prompt.clone() + &output_text; // output_text = prompt.clone() + &output_text;
} //}
BestOfSequence { BestOfSequence {
generated_text: output_text, generated_text: output_text,
@ -415,10 +415,10 @@ pub(crate) async fn generate_internal(
.record(response.generated_text.generated_tokens as f64); .record(response.generated_text.generated_tokens as f64);
// Send response // Send response
let mut output_text = response.generated_text.text; let output_text = response.generated_text.text.clone();
if let Some(prompt) = add_prompt { //if let Some(prompt) = add_prompt {
output_text = prompt + &output_text; // output_text = prompt + &output_text;
} //}
tracing::debug!("Output: {}", output_text); tracing::debug!("Output: {}", output_text);
tracing::info!("Success"); tracing::info!("Success");
@ -521,10 +521,10 @@ async fn generate_stream_internal(
let mut end_reached = false; let mut end_reached = false;
let mut error = false; let mut error = false;
let mut add_prompt = None; //let mut add_prompt = None;
if req.parameters.return_full_text.unwrap_or(false) { //if req.parameters.return_full_text.unwrap_or(false) {
add_prompt = Some(req.inputs.clone()); // add_prompt = Some(req.inputs.clone());
} //}
let details = req.parameters.details; let details = req.parameters.details;
let best_of = req.parameters.best_of.unwrap_or(1); let best_of = req.parameters.best_of.unwrap_or(1);
@ -615,10 +615,10 @@ async fn generate_stream_internal(
// StreamResponse // StreamResponse
end_reached = true; end_reached = true;
let mut output_text = generated_text.text; let output_text = generated_text.text;
if let Some(prompt) = add_prompt { //if let Some(prompt) = add_prompt {
output_text = prompt + &output_text; // output_text = prompt + &output_text;
} //}
tracing::debug!(parent: &span, "Output: {}", output_text); tracing::debug!(parent: &span, "Output: {}", output_text);
tracing::info!(parent: &span, "Success"); tracing::info!(parent: &span, "Success");
@ -1522,10 +1522,9 @@ pub async fn run(
// Shared API builder initialization // Shared API builder initialization
let api_builder = || { let api_builder = || {
let mut builder = ApiBuilder::new().with_progress(false); let mut builder = ApiBuilder::new()
if let Some(token) = authorization_token { .with_progress(false)
builder = builder.with_token(Some(token)); .with_token(authorization_token);
}
if let Ok(cache_dir) = std::env::var("HUGGINGFACE_HUB_CACHE") { if let Ok(cache_dir) = std::env::var("HUGGINGFACE_HUB_CACHE") {
builder = builder.with_cache_dir(cache_dir.into()); builder = builder.with_cache_dir(cache_dir.into());