huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 00:12:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix qwen2.

Browse Source
This commit is contained in:
Nicolas Patry 2025-03-18 10:36:54 +01:00
parent febc488e0e
commit 078084286a
No known key found for this signature in database
GPG Key ID: 4242CEF24CB6DBF9
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
View File

@ -409,7 +409,7 @@ class Qwen2ForCausalLM(torch.nn.Module):
self.window_size = config.sliding_window self.window_size = config.sliding_window
self.window_size_tensor = ( self.window_size_tensor = (
torch.tensor(config.sliding_window, device=weights.device) torch.tensor(config.sliding_window, device=weights.device)
if self.max_past is not None if self.window_size is not None
else None else None
) )
@ -431,7 +431,7 @@ class Qwen2ForCausalLM(torch.nn.Module):
if prefill_cache_indices is not None: if prefill_cache_indices is not None:
# Slots also need to be sliced as it has the same size as the whole kv tensor # Slots also need to be sliced as it has the same size as the whole kv tensor
slots = slots[prefill_cache_indices] slots = slots[prefill_cache_indices]
elif self.max_past is not None: elif self.window_size is not None:
# Clamp in decode mode as paged attention requires clamped values whereas the flash attention # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
# kernel requires the true values # kernel requires the true values
seqlen = seqlen.clamp(max=self.window_size_tensor) seqlen = seqlen.clamp(max=self.window_size_tensor)