huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 04:44:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update kv_cache.py

Browse Source
This commit is contained in:
Florian Zimmermeister 2024-10-05 09:12:12 +02:00 committed by GitHub
parent 2358c2bb54
commit e03a7167ee
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/text_generation_server/layers/attention/kv_cache.py
View File

@ -26,8 +26,8 @@ class KVCache:
if ( if (
dtype == torch.float8_e5m2 dtype == torch.float8_e5m2
and ATTENTION != "flashinfer" and (ATTENTION != "flashinfer"
and SYSTEM != "cuda" or SYSTEM != "cuda")
): ):
raise ValueError( raise ValueError(
"float8_e5m2 KV cache is currently only supported for flashinfer on CUDA" "float8_e5m2 KV cache is currently only supported for flashinfer on CUDA"