huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 12:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

none g_idx

Browse Source
This commit is contained in:
IlyasMoutawwakil 2024-02-02 14:42:42 +01:00 committed by Nicolas Patry
parent bbe5bedea5
commit 76834c9989
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/text_generation_server/utils/weights.py
View File

@ -177,6 +177,8 @@ class Weights:
dtype=torch.int32, dtype=torch.int32,
device=qweight.device, device=qweight.device,
) )
else:
g_idx = None
weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False)
else: else:
@ -242,6 +244,8 @@ class Weights:
dtype=torch.int32, dtype=torch.int32,
device=qweight.device, device=qweight.device,
) )
else:
g_idx = None
from text_generation_server.utils.layers import HAS_EXLLAMA from text_generation_server.utils.layers import HAS_EXLLAMA