mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-21 14:52:20 +00:00
eab07f746c
* Add support for FP8 KV cache scales Since FP8 only has limited dynamic range, we can scale keys/values before storing them into the cache (and unscale them in attention). To avoid rescaling the cache as the absmax values change, good scales are usually determined per layer using calibration calibration data and stored in the checkpoint. This change adds support for for using key-value scales and loading them from checkpoints in the two most common formats: - Separate per-layer `k_scale` and `v_scale` scalars. - Per-layer `kv_scale` scalar (older format). Currently, scales are only used with an `float8_e4m3fn` cache. Besides adding support for key/value scales, the `fp8_quantize` function is also extended to support quantization with a kernel vendored from vLLM. This is slightly faster than the PyTorch implementation, but also scales in FP32, potentially improving accuracy. * Update FP8 KV cache test to use checkpoint with scales * `can_scale`: check that the attention is flashinfer
41 lines
929 B
Python
41 lines
929 B
Python
import os
|
|
|
|
from text_generation_server.utils.import_utils import SYSTEM
|
|
|
|
from .common import Seqlen
|
|
|
|
if os.getenv("USE_FLASH_ATTENTION", "").lower() == "false":
|
|
raise ImportError("`USE_FLASH_ATTENTION` is false.")
|
|
if SYSTEM == "cuda":
|
|
from .cuda import (
|
|
SUPPORTS_WINDOWING,
|
|
attention,
|
|
paged_attention,
|
|
)
|
|
elif SYSTEM == "rocm":
|
|
from .rocm import (
|
|
SUPPORTS_WINDOWING,
|
|
attention,
|
|
paged_attention,
|
|
)
|
|
elif SYSTEM == "ipex":
|
|
from .ipex import (
|
|
SUPPORTS_WINDOWING,
|
|
attention,
|
|
paged_attention,
|
|
)
|
|
else:
|
|
raise ImportError(f"System {SYSTEM} doesn't support flash/paged attention")
|
|
|
|
# KVCache needs `reshape_and_cache`, so ensure that it is defined already.
|
|
from .kv_cache import KVCache, get_kv_scales
|
|
|
|
__all__ = [
|
|
"attention",
|
|
"get_kv_scales",
|
|
"paged_attention",
|
|
"SUPPORTS_WINDOWING",
|
|
"KVCache",
|
|
"Seqlen",
|
|
]
|