huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-24 00:12:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

support bits different than 4

Browse Source
This commit is contained in:
Felix Marty 2023-07-12 16:19:25 +00:00
parent 67d687609b
commit f90c61a340
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/utils/layers.py
View File

@ -151,7 +151,7 @@ def get_linear(weight, bias, quantize):
f"The passed weight is not `gptq` compatible, loader needs to be updated." f"The passed weight is not `gptq` compatible, loader needs to be updated."
) )
if use_triton_kernel: if use_triton_kernel or bits != 4:
linear = QuantLinear( linear = QuantLinear(
qweight, qweight,
qzeros, qzeros,