This commit is contained in:
regisss 2025-06-18 07:52:59 +00:00
parent 9dbaa176fd
commit 1acc96c82a
10 changed files with 16 additions and 10 deletions
backends/gaudi/server/text_generation_server

View File

@ -1798,7 +1798,9 @@ class FlashCausalLM(Model):
total_batch_seq = 0.001
total_mem = 0
available_mem = free_mem - self.mem_reserved
log_master(logger.info, f"Decode batch size list:{[bsz[0] for bsz in buckets]}\n")
log_master(
logger.info, f"Decode batch size list:{[bsz[0] for bsz in buckets]}\n"
)
for i, (batch_size, block_num) in enumerate(buckets):
if batch_size > block_num:
continue

View File

@ -822,7 +822,9 @@ class FlashVlmCausalLM(FlashCausalLM):
total_batch_seq = 0.001
total_mem = 0
available_mem = decode_available_memory
log_master(logger.info, f"Decode batch size list:{[bsz[0] for bsz in buckets]}\n")
log_master(
logger.info, f"Decode batch size list:{[bsz[0] for bsz in buckets]}\n"
)
for i, (batch_size, block_num) in enumerate(buckets):
if batch_size > block_num:
continue

View File

@ -442,7 +442,9 @@ class FlashMllamaCausalLM(FlashVlmCausalLM):
total_batch_seq = 0.001
total_mem = 0
available_mem = free_mem - self.mem_reserved
log_master(logger.info, f"Decode batch size list:{[bsz[0] for bsz in buckets]}\n")
log_master(
logger.info, f"Decode batch size list:{[bsz[0] for bsz in buckets]}\n"
)
for i, (batch_size, block_num) in enumerate(buckets):
if batch_size > block_num:
continue

View File

@ -8,7 +8,7 @@ import torch
def find_segments(
adapter_indices: Union[torch.Tensor, List[int]]
adapter_indices: Union[torch.Tensor, List[int]],
) -> Tuple[List[int], List[int]]:
segments = [0]
segment_indices = []