huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-05-07 02:12:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing the other pathways.

Browse Source
This commit is contained in:
Nicolas Patry 2024-08-14 08:53:17 +02:00
parent c9047667ad
commit 5c598cc7ed
No known key found for this signature in database
GPG Key ID: 64AF4752B2967863
1 changed files with 13 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
server/text_generation_server/layers/gptq/exllamav2.py
View File

@ -115,13 +115,14 @@ def ext_make_q_matrix(
w.qweight,
extra.q_perm,
extra.q_invperm,
none_tensor,
none_tensor,
none_tensor,
none_tensor,
none_tensor, # q_scale
none_tensor, # q_scale_max
none_tensor, # q_groups
none_tensor, # q_group_map
w.qzeros,
w.scales,
w.g_idx.cpu(),
none_tensor, # bias
temp_dq,
max_dq_rows,
)
@ -129,15 +130,16 @@ def ext_make_q_matrix(
else:
return make_q_matrix(
w.qweight,
none_tensor,
none_tensor,
none_tensor,
none_tensor,
none_tensor,
none_tensor,
none_tensor, # q_perm
none_tensor, # q_invperm
none_tensor, # q_scale
none_tensor, # q_scale_max
none_tensor, # q_groups
none_tensor, # q_group_map
w.qzeros,
w.scales,
none_tensor,
none_tensor, # g_idx
none_tensor, # bias
temp_dq,
max_dq_rows,
)