From a217b4df5a55a4cdb12d54c6bdf486ed704bfe00 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 7 Aug 2023 12:19:02 +0200 Subject: [PATCH] Fixing non 4bits quantization. --- server/text_generation_server/utils/gptq/quant_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/gptq/quant_linear.py b/server/text_generation_server/utils/gptq/quant_linear.py index 54fa2014..bfc91c00 100644 --- a/server/text_generation_server/utils/gptq/quant_linear.py +++ b/server/text_generation_server/utils/gptq/quant_linear.py @@ -263,7 +263,7 @@ class QuantLinear(nn.Module): self.groupsize = groupsize self.outfeatures = qweight.shape[1] - self.infeatures = qweight.shape[0] * 32 // 4 + self.infeatures = qweight.shape[0] * 32 // bits @classmethod def new(cls, bits, groupsize, infeatures, outfeatures, bias):