huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-19 22:02:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix Qwen VL break in intel platform (#3002)

Browse Source 
* fix Qwen VL break in intel platform

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* could use PositionRotaryEmbedding impl so rocm and ipex could all work

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

---------

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi 2025-02-12 18:31:34 +08:00 committed by GitHub
parent b86c3947ab
commit 76bcb4948d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 0 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
server/text_generation_server/layers/rotary.py
View File

@ -570,22 +570,6 @@ class RotaryPositionEmbeddingMultimodalSections(PositionRotaryEmbedding):
.to(inv_freq.device) .to(inv_freq.device)
) )
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
):
# rotate half the sequence length
rot = cos.shape[-1] // 2
q2 = torch.cat([-query[..., rot:], query[..., :rot]], dim=-1)
k2 = torch.cat([-key[..., rot:], key[..., :rot]], dim=-1)
# apply the rotation
rotary_emb.apply_rotary(query, q2, cos, sin, query, q2, True)
rotary_emb.apply_rotary(key, k2, cos, sin, key, k2, True)
def _update_cos_sin_cache( def _update_cos_sin_cache(
self, dtype: torch.dtype, device: torch.device, seqlen: int self, dtype: torch.dtype, device: torch.device, seqlen: int
): ):
@ -614,7 +598,4 @@ class RotaryPositionEmbeddingMultimodalSections(PositionRotaryEmbedding):
cos = self._cos_cached[position_ids].gather(1, self._sections[:slen]) cos = self._cos_cached[position_ids].gather(1, self._sections[:slen])
sin = self._sin_cached[position_ids].gather(1, self._sections[:slen]) sin = self._sin_cached[position_ids].gather(1, self._sections[:slen])
cos = torch.cat([cos, cos], dim=-1)
sin = torch.cat([sin, sin], dim=-1)
return cos, sin return cos, sin