huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 15:32:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pr 2337 ci branch (#2379)

Browse Source 
* hotfix: fix xpu crash brought by code refine. torch.xpu rely on import ipex

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* reable gemma2 in xpu

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* fix in regression in ipex flashattention

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

---------

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
Co-authored-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
drbh 2024-08-08 12:30:29 -04:00 committed by yuanwu
parent 3893d00927
commit 1057f28128
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
server/text_generation_server/layers/attention/ipex.py
View File

@ -2,6 +2,7 @@ import intel_extension_for_pytorch as ipex
import torch import torch
from text_generation_server.models.flash_causal_lm import BLOCK_SIZE from text_generation_server.models.flash_causal_lm import BLOCK_SIZE
from text_generation_server.layers.attention import Seqlen from text_generation_server.layers.attention import Seqlen
from typing import Optional
SUPPORTS_WINDOWING = False SUPPORTS_WINDOWING = False
@ -15,11 +16,12 @@ def attention(
softmax_scale, softmax_scale,
window_size_left=-1, window_size_left=-1,
causal=True, causal=True,
softcap: Optional[float] = None,
): ):
out = torch.empty_like(q) out = torch.empty_like(q)
# We do not need to check window_size_left (not supported) here, so it is already checked ahead of time at model load. # We do not need to check window_size_left (not supported) here, so it is already checked ahead of time at model load.
return ipex.llm.functional.varlen_attention( ipex.llm.functional.varlen_attention(
q, q,
k, k,
v, v,
@ -36,6 +38,8 @@ def attention(
None, None,
) )
return out
def reshape_and_cache( def reshape_and_cache(
key: torch.Tensor, key: torch.Tensor,
@ -58,6 +62,7 @@ def paged_attention(
block_tables: torch.Tensor, block_tables: torch.Tensor,
seqlen: Seqlen, seqlen: Seqlen,
max_s: int, max_s: int,
softcap: Optional[float] = None,
): ):
out = torch.empty_like(query) out = torch.empty_like(query)
ipex.llm.modules.PagedAttention.single_query_cached_kv_attention( ipex.llm.modules.PagedAttention.single_query_cached_kv_attention(

2
server/text_generation_server/utils/import_utils.py
View File

@ -56,6 +56,8 @@ elif torch.version.cuda is not None and torch.cuda.is_available():
get_free_memory = get_cuda_free_memory get_free_memory = get_cuda_free_memory
elif is_ipex_available(): elif is_ipex_available():
SYSTEM = "ipex" SYSTEM = "ipex"
import intel_extension_for_pytorch # noqa: F401
if hasattr(torch, "xpu") and torch.xpu.is_available(): if hasattr(torch, "xpu") and torch.xpu.is_available():
empty_cache = torch.xpu.empty_cache empty_cache = torch.xpu.empty_cache
synchronize = torch.xpu.synchronize synchronize = torch.xpu.synchronize