huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-07-13 11:20:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify the warmup process (#173)

Browse Source 
Signed-off-by: yuanwu <yuan.wu@intel.com>
This commit is contained in:
yuanwu2017 2024-08-15 18:04:14 +08:00 committed by GitHub
parent e3f0f85b70
commit 369e499a66
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 0 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
server/text_generation_server/models/causal_lm.py
View File

@ -1192,14 +1192,6 @@ class CausalLM(Model):
if len(request_ids) < len(decode_batch.requests): if len(request_ids) < len(decode_batch.requests):
decode_batch = decode_batch.filter(request_ids) decode_batch = decode_batch.filter(request_ids)
while decode_batch is not None:
# filter finished requests
request_ids = get_unfinished_requests(decode_batch.requests)
if len(request_ids) < len(decode_batch.requests):
decode_batch = decode_batch.filter(request_ids)
# decode
_, decode_batch, _ = self.generate_token([decode_batch])
def shifting_warmup(self, batch: CausalLMBatch) -> None: def shifting_warmup(self, batch: CausalLMBatch) -> None:
chunk_sizes = CHUNK_SIZES.copy() chunk_sizes = CHUNK_SIZES.copy()
chunk_sizes.extend([-chunk for chunk in chunk_sizes]) chunk_sizes.extend([-chunk for chunk in chunk_sizes])