Removing dead code.

server/text_generation_server/utils/gptq/quant_linear.py

@@ -318,127 +318,8 @@ class QuantLinear(nn.Module):
         self.outfeatures = qweight.shape[1]
         self.infeatures = qweight.shape[0] * 32 // 4
 
-        # expected = (math.ceil(self.infeatures / self.groupsize), self.outfeatures // 32 * self.bits)
-        # assert tuple(self.qzeros.shape) == expected, f"{self.qzeros.shape} != {expected}"
-        # expected = (math.ceil(self.infeatures / self.groupsize), self.outfeatures)
-        # assert tuple(self.scales.shape) == expected, f"{self.scales.shape} != {expected}"
-        # assert self.g_idx.shape == (math.ceil(self.infeatures / self.group_size), outfeatures)
-
-    # def new(cls, bits, groupsize, infeatures, outfeatures, bias):
-    #     if bits not in [2, 4, 8]:
-    #         raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-    #     self.infeatures = infeatures
-    #     self.outfeatures = outfeatures
-    #     self.bits = bits
-    #     self.maxq = 2**self.bits - 1
-    #     self.groupsize = groupsize if groupsize != -1 else infeatures
-
-    #     self.register_buffer('qweight', torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32))
-    #     self.register_buffer('qzeros', torch.zeros((math.ceil(infeatures / self.groupsize), outfeatures // 32 * self.bits), dtype=torch.int32))
-    #     self.register_buffer('scales', torch.zeros((math.ceil(infeatures / self.groupsize), outfeatures), dtype=torch.float16))
-    #     self.register_buffer('g_idx', torch.tensor([i // self.groupsize for i in range(infeatures)], dtype=torch.int32))
-    #     if bias:
-    #         self.register_buffer('bias', torch.zeros((outfeatures), dtype=torch.float16))
-    #     else:
-    #         self.bias = None
-
-    def pack(self, linear, scales, zeros, g_idx=None):
-        self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
-
-        scales = scales.t().contiguous()
-        zeros = zeros.t().contiguous()
-        scale_zeros = zeros * scales
-        self.scales = scales.clone().half()
-        if linear.bias is not None:
-            self.bias = linear.bias.clone().half()
-
-        intweight = []
-        for idx in range(self.infeatures):
-            intweight.append(torch.round((linear.weight.data[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[:, None])
-        intweight = torch.cat(intweight, dim=1)
-        intweight = intweight.t().contiguous()
-        intweight = intweight.numpy().astype(np.uint32)
-        qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)
-        i = 0
-        row = 0
-        while row < qweight.shape[0]:
-            if self.bits in [2, 4, 8]:
-                for j in range(i, i + (32 // self.bits)):
-                    qweight[row] |= intweight[j] << (self.bits * (j - i))
-                i += 32 // self.bits
-                row += 1
-            else:
-                raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qweight = qweight.astype(np.int32)
-        self.qweight = torch.from_numpy(qweight)
-
-        zeros -= 1
-        zeros = zeros.numpy().astype(np.uint32)
-        qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
-        i = 0
-        col = 0
-        while col < qzeros.shape[1]:
-            if self.bits in [2, 4, 8]:
-                for j in range(i, i + (32 // self.bits)):
-                    qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
-                i += 32 // self.bits
-                col += 1
-            else:
-                raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qzeros = qzeros.astype(np.int32)
-        self.qzeros = torch.from_numpy(qzeros)
-
     def forward(self, x):
         out_shape = x.shape[:-1] + (self.outfeatures, )
         out = QuantLinearFunction.apply(x.reshape(-1, x.shape[-1]), self.qweight, self.scales, self.qzeros, self.g_idx, self.bits, self.maxq)
         out = out + self.bias if self.bias is not None else out
         return out.reshape(out_shape)
-
-
-def make_quant_linear(module, names, bits, groupsize, name=''):
-    if isinstance(module, QuantLinear):
-        return
-    for attr in dir(module):
-        tmp = getattr(module, attr)
-        name1 = name + '.' + attr if name != '' else attr
-        if name1 in names:
-            delattr(module, attr)
-            setattr(module, attr, QuantLinear(bits, groupsize, tmp.in_features, tmp.out_features, tmp.bias is not None))
-    for name1, child in module.named_children():
-        make_quant_linear(child, names, bits, groupsize, name + '.' + name1 if name != '' else name1)
-
-
-def autotune_warmup_linear(model, transpose=False):
-    """
-    Pre-tunes the quantized kernel
-    """
-    from tqdm import tqdm
-
-    kn_values = {}
-
-    for _, m in model.named_modules():
-        if not isinstance(m, QuantLinear):
-            continue
-
-        k = m.infeatures
-        n = m.outfeatures
-
-        if (k, n) not in kn_values:
-            kn_values[(k, n)] = (m.qweight.cuda(), m.scales.cuda(), m.qzeros.cuda(), m.g_idx.cuda(), m.bits, m.maxq)
-
-    print(f'Found {len(kn_values)} unique KN Linear values.')
-
-    print('Warming up autotune cache ...')
-    with torch.no_grad():
-        for m in tqdm(range(0, 12)):
-            m = 2**m  # [1, 2048]
-            for (k, n), (qweight, scales, qzeros, g_idx, bits, maxq) in kn_values.items():
-                a = torch.randn(m, k, dtype=torch.float16, device='cuda')
-                matmul248(a, qweight, scales, qzeros, g_idx, bits, maxq)
-                if transpose:
-                    a = torch.randn(m, n, dtype=torch.float16, device='cuda')
-                    transpose_matmul248(a, qweight, scales, qzeros, g_idx, bits, maxq)
-    del kn_values