diff --git a/server/text_generation_server/utils/gptq/quant_linear.py b/server/text_generation_server/utils/gptq/quant_linear.py
index 54fa2014..bfc91c00 100644
--- a/server/text_generation_server/utils/gptq/quant_linear.py
+++ b/server/text_generation_server/utils/gptq/quant_linear.py
@@ -263,7 +263,7 @@ class QuantLinear(nn.Module):
         self.groupsize = groupsize
 
         self.outfeatures = qweight.shape[1]
-        self.infeatures = qweight.shape[0] * 32 // 4
+        self.infeatures = qweight.shape[0] * 32 // bits
 
     @classmethod
     def new(cls, bits, groupsize, infeatures, outfeatures, bias):