huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

raise error if needed

Browse Source
This commit is contained in:
Cyril Vallez 2025-01-20 11:29:51 +01:00
parent f01014de37
commit 2659b5998b
No known key found for this signature in database
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/models/__init__.py
View File

@ -380,6 +380,8 @@ def get_model(
transformers_model_class = getattr(transformers, modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[model_type])
if transformers_model_class.is_backend_compatible():
transformers_causal_lm_class = TransformersFlashCausalLM
if not FLASH_ATTENTION and lora_adapter_ids is not None and len(lora_adapter_ids) > 0:
raise ValueError("Transformers backend AutoModel do not support `lora_adapter_ids`.")
quantization_config = config_dict.get("quantization_config", None)
if quantization_config is None:

2
server/text_generation_server/models/transformers_flash_causal_lm.py
View File

@ -48,7 +48,7 @@ def tgi_flash_attention_forward(
softmax_scale: Optional[float] = None,
sliding_window: Optional[int] = None,
softcap: Optional[float] = None,
**kwargs, # This is needed to "absorb" other args passed by Transformers modeling
**_kwargs, # This is needed to "absorb" other args passed by Transformers modeling
):
kv_cache = kv_cache[module.layer_idx]