huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix typo in batch concatination

Browse Source
This commit is contained in:
Vincent Brouwers 2023-08-09 08:39:18 +00:00
parent e30f4f61e7
commit ea78c5c26a
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/models/flash_causal_lm.py
View File

@ -646,7 +646,7 @@ class FlashCausalLMBatch(Batch):
next_token_chooser_parameters.extend([r.parameters for r in batch.requests]) next_token_chooser_parameters.extend([r.parameters for r in batch.requests])
stopping_criterias.extend(batch.stopping_criterias) stopping_criterias.extend(batch.stopping_criterias)
top_n_tokens.extend(top_n_tokens) top_n_tokens.extend(batch.top_n_tokens)
# Update # Update
cumulative_batch_size += len(batch) cumulative_batch_size += len(batch)