huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-09 11:24:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

support g_idx

Browse Source 
Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
This commit is contained in:
Wang, Yi A 2025-07-16 22:46:05 -07:00
parent fc2405c549
commit d31bdfdbae
1 changed files with 23 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
backends/gaudi/server/text_generation_server/layers/gptq/hpu.py
View File

@ -89,11 +89,29 @@ class QuantLinear(nn.Module):
g_idx_trivial = torch.tensor( g_idx_trivial = torch.tensor(
g_idx_trivial, dtype=torch.int32, device=self.g_idx.device g_idx_trivial, dtype=torch.int32, device=self.g_idx.device
) )
assert torch.equal( sort_zeros = not (torch.equal(self.g_idx, g_idx_trivial))
self.g_idx, g_idx_trivial
), "Non-trivial tensor g_idx is not supported"
self.qzeros = self.qzeros.cpu() self.qzeros = self.qzeros.cpu()
zeros = self.unpack_zeros_from_cuda_old_format() zeros = self.unpack_zeros_from_cuda_old_format()
if sort_zeros:
zeros_group_1 = torch.zeros(
(self.infeatures, self.outfeatures),
dtype=zeros.dtype,
device=zeros.device,
)
scales = self.scales.cpu()
scale_group_1 = torch.zeros(
(self.infeatures, self.outfeatures),
dtype=scales.dtype,
device=scales.device,
)
for i in range(self.infeatures):
zeros_group_1[i] = zeros[self.g_idx[i]]
scale_group_1[i] = self.scales[self.g_idx[i]]
self.qzeros = pack_tensor(zeros_group_1).to(orig_device)
self.scales = scale_group_1.to(orig_device)
self.groupsize = 1
self.g_idx = None
else:
new_qzeros = pack_tensor(zeros) new_qzeros = pack_tensor(zeros)
self.qzeros = new_qzeros.to(orig_device) self.qzeros = new_qzeros.to(orig_device)