huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-08-01 21:00:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Gaudi] Fix the OOM issue of Llama-4-Scout-17B-16E-Instruct (#3245)

Browse Source 
Signed-off-by: yuanwu <yuan.wu@intel.com>
This commit is contained in:
Yuan Wu 2025-05-29 15:58:24 +08:00 committed by GitHub
parent f14044009a
commit 70217ac345
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
backends/gaudi/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
View File

@ -143,12 +143,14 @@ class FlashLlamaAttention(torch.nn.Module):
config.num_key_value_heads = getattr( config.num_key_value_heads = getattr(
config, "num_key_value_heads", config.num_attention_heads config, "num_key_value_heads", config.num_attention_heads
) )
self.rotary_emb = PositionRotaryEmbedding.static(
config=config, if config.model_type != "llama4_text":
dim=self.head_size, self.rotary_emb = PositionRotaryEmbedding.static(
base=config.rope_theta, config=config,
device=weights.device, dim=self.head_size,
) base=config.rope_theta,
device=weights.device,
)
# `config.attention_multiplier` is used in Granite # `config.attention_multiplier` is used in Granite
self.softmax_scale = getattr( self.softmax_scale = getattr(