huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 11:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding additional response header X-Total-Tokens

Browse Source
This commit is contained in:
Julian Bright 2023-07-11 03:17:09 +10:00
parent b4024edd45
commit 1e62237d44
2 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
load_tests/starcoder_load.js
View File

@ -55,6 +55,7 @@ export default function () {
if (res.status === 200) {
totalTime.add(res.headers["X-Total-Time"]);
totalTokens.add(res.headers["X-Total-Tokens"]);
validationTime.add(res.headers["X-Validation-Time"]);
queueTime.add(res.headers["X-Queue-Time"]);
inferenceTime.add(res.headers["X-Inference-Time"]);

4
router/src/server.rs
View File

@ -241,6 +241,10 @@ async fn generate(
"x-total-time",
total_time.as_millis().to_string().parse().unwrap(),
);
headers.insert(
"x-total-tokens",
response.generated_text.generated_tokens.to_string().parse().unwrap(),
);
headers.insert(
"x-validation-time",
validation_time.as_millis().to_string().parse().unwrap(),