huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 04:14:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

add in Buffering..

Browse Source 
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi A 2025-02-09 19:28:37 -08:00
parent efeef0bed6
commit ecbd956a4c
1 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
router/src/server.rs
View File

@ -1345,6 +1345,18 @@ pub(crate) async fn chat_completions(
model_id.clone(),
);
yield Ok::<Event, Infallible>(event);
if stream_token.details.is_some() && stream_options
.as_ref()
.map(|s| s.include_usage)
.unwrap_or(false) {
let usage_event = create_usage_event_from_stream_token(
stream_token,
stream_options.clone(),
system_fingerprint.clone(),
model_id.clone(),
);
yield Ok::<Event, Infallible>(usage_event);
}
}
}
}