huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-21 06:42:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

work with the latest vllm extension ops

Browse Source 
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi A 2025-04-10 19:56:58 -07:00
parent 4de8fb0127
commit a83e9fe003
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backends/gaudi/server/text_generation_server/models/flash_causal_lm.py
View File

@ -1512,9 +1512,10 @@ class FlashCausalLM(Model):
self.bucketing_ctx = HPUBucketingContext( self.bucketing_ctx = HPUBucketingContext(
os.getenv("DECODE_MAX_BS", 128), # self.max_num_seqs, #TODO os.getenv("DECODE_MAX_BS", 128), # self.max_num_seqs, #TODO
os.getenv("PREFILL_MAX_BS", 16), # self.max_num_prefill_seqs, #TODO os.getenv("PREFILL_MAX_BS", 64), # self.max_num_prefill_seqs, #TODO
BLOCK_SIZE, BLOCK_SIZE,
num_blocks * BLOCK_SIZE, num_blocks * BLOCK_SIZE,
False,
) )
self.bucketing_ctx.num_hpu_blocks = num_blocks self.bucketing_ctx.num_hpu_blocks = num_blocks
if os.getenv("VLLM_SKIP_WARMUP", "false").lower() == "true": if os.getenv("VLLM_SKIP_WARMUP", "false").lower() == "true":