huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 15:32:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

hotfix: Fix number of KV heads (#2202)

Browse Source 
Fix number of KV heads
This commit is contained in:
Daniël de Kok 2024-07-08 09:52:12 +02:00 committed by yuanwu
parent 8e3d1e6c3f
commit f11fd699b6
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/text_generation_server/models/flash_causal_lm.py
View File

@ -906,8 +906,8 @@ class FlashCausalLM(Model):
# Validation is done in the model itself
if num_kv_heads is None:
# Order is important here.
for attr in ["num_key_value_heads", "num_key_value_heads", "n_head"]:
num_kv_heads = getattr(config, "num_attention_heads", None)
for attr in ["num_key_value_heads", "num_attention_heads", "n_head"]:
num_kv_heads = getattr(config, attr, None)
if num_kv_heads is not None:
break
if num_kv_heads is None: