huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-06-19 15:52:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

add image

Browse Source
This commit is contained in:
OlivierDehaene 2023-03-30 13:13:23 +02:00
parent c15922b132
commit b2d1276c16
5 changed files with 24 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
assets/benchmark.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 100 KiB

6
benchmark/README.md
View File

@ -1,5 +1,11 @@
<div align="center">
# Text Generation Inference benchmarking tool
![benchmark](../assets/benchmark.png)
</div>
A lightweight benchmarking tool based inspired by [oha](https://github.com/hatoo/oha)
and powered by [tui](https://github.com/tui-rs-revival/ratatui).

11
benchmark/src/app.rs
View File

@ -19,6 +19,7 @@ pub(crate) struct App {
completed_batch: usize,
current_batch: usize,
current_tab: usize,
touched_tab: bool,
zoom: bool,
is_error: bool,
data: Data,
@ -53,6 +54,7 @@ impl App {
completed_batch,
current_batch,
current_tab,
touched_tab: false,
zoom: false,
is_error,
data,
@ -76,6 +78,7 @@ impl App {
| KeyEvent {
code: KeyCode::Tab, ..
} => {
self.touched_tab=true;
self.current_tab = (self.current_tab + 1) % self.batch_size.len();
}
// Decrease and wrap tab
@ -83,6 +86,7 @@ impl App {
code: KeyCode::Left,
..
} => {
self.touched_tab=true;
if self.current_tab > 0 {
self.current_tab -= 1;
} else {
@ -131,9 +135,14 @@ impl App {
}
Message::EndBatch => {
self.data.end_batch(self.current_batch);
self.completed_batch += 1;
if self.current_batch < self.batch_size.len() - 1 {
// Only go to next tab if the user never touched the tab keys
if !self.touched_tab {
self.current_tab += 1;
}
self.current_batch += 1;
}
}

14
benchmark/src/main.rs
View File

@ -14,19 +14,19 @@ use tracing_subscriber::EnvFilter;
#[derive(Parser, Debug)]
#[clap(author, version, about, long_about = None)]
struct Args {
#[clap(long, env)]
#[clap(short, long, env)]
tokenizer_name: String,
#[clap(long)]
#[clap(short, long)]
batch_size: Option<Vec<u32>>,
#[clap(default_value = "10", long, env)]
#[clap(default_value = "10", short, long, env)]
sequence_length: u32,
#[clap(default_value = "64", long, env)]
#[clap(default_value = "8", short,long, env)]
decode_length: u32,
#[clap(default_value = "10", long, env)]
#[clap(default_value = "10", short,long, env)]
runs: usize,
#[clap(default_value = "1", long, env)]
#[clap(default_value = "1", short,long, env)]
warmups: usize,
#[clap(default_value = "/tmp/text-generation-server-0", long, env)]
#[clap(default_value = "/tmp/text-generation-server-0", short, long, env)]
master_shard_uds_path: String,
}

1
router/src/queue.rs
View File

@ -237,6 +237,7 @@ mod tests {
watermark: false,
},
stopping_parameters: StoppingCriteriaParameters {
ignore_eos_token: false,
max_new_tokens: 0,
stop_sequences: vec![],
},