huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 12:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

add docker load_tests

Browse Source
This commit is contained in:
Xuan Son Nguyen 2024-08-09 13:16:49 +02:00
parent 7830de1566
commit cd1e2cd2cf
2 changed files with 17 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
load_tests/common.js
View File

@ -36,7 +36,7 @@ export function get_options() {
load_test: { load_test: {
executor: 'constant-arrival-rate', executor: 'constant-arrival-rate',
duration: '60s', duration: '60s',
preAllocatedVUs: 100, preAllocatedVUs: 50, // not enough RAM for 100 VUs
rate: 1, rate: 1,
timeUnit: '1s', timeUnit: '1s',
}, },
@ -68,13 +68,16 @@ export default function run() {
const headers = { 'Content-Type': 'application/json' }; const headers = { 'Content-Type': 'application/json' };
const query = shareGPT[scenario.iterationInTest % shareGPT.length]; const query = shareGPT[scenario.iterationInTest % shareGPT.length];
const payload = JSON.stringify(generate_payload(query, max_new_tokens)); const payload = JSON.stringify(generate_payload(query, max_new_tokens));
const res = http.post(`http://${host}/v1/chat/completions`, payload, { const res = http.post(`https://${host}/v1/chat/completions`, payload, {
headers, headers,
}); });
if (res.status >= 400 && res.status < 500) { if (res.status >= 400 && res.status < 500) {
return; return;
} }
if (res.status !== 200) {
console.error(res.body);
}
check(res, { check(res, {
'Post status is 200': (res) => res.status === 200, 'Post status is 200': (res) => res.status === 200,

12
load_tests/docker-compose.yml Normal file
View File

@ -0,0 +1,12 @@
services:
# How to run:
# HOST=..... docker compose up
tgi_load_test:
image: grafana/k6
command: run /load_tests/common.js
environment:
- HOST=${HOST}
- MODEL_ID=${MODEL_ID}
volumes:
- ./:/load_tests:Z