From 69569f0c2a79355682debf4d720c9ff51cb21069 Mon Sep 17 00:00:00 2001 From: "Wang, Yi A" Date: Mon, 4 Nov 2024 03:41:31 -0800 Subject: [PATCH] fix incorrect output of Qwen2-7B-Instruct-GPTQ-Int4 and Qwen2-7B-Instruct-AWQ ipex kernel provide func like add_bias, so no need add it outside Signed-off-by: Wang, Yi A --- server/text_generation_server/layers/awq/quantize/ipex.py | 1 - server/text_generation_server/layers/gptq/ipex.py | 1 - 2 files changed, 2 deletions(-) diff --git a/server/text_generation_server/layers/awq/quantize/ipex.py b/server/text_generation_server/layers/awq/quantize/ipex.py index 84cd7a21..842e9623 100644 --- a/server/text_generation_server/layers/awq/quantize/ipex.py +++ b/server/text_generation_server/layers/awq/quantize/ipex.py @@ -44,5 +44,4 @@ class WQLinear(nn.Module): def forward(self, x): out_shape = x.shape[:-1] + (self.out_features,) out = self.woq_linear(x.reshape(-1, x.shape[-1])) - out = out + self.bias if self.bias is not None else out return out.reshape(out_shape) diff --git a/server/text_generation_server/layers/gptq/ipex.py b/server/text_generation_server/layers/gptq/ipex.py index ab9c9e24..48584e90 100644 --- a/server/text_generation_server/layers/gptq/ipex.py +++ b/server/text_generation_server/layers/gptq/ipex.py @@ -122,5 +122,4 @@ class QuantLinear(nn.Module): def forward(self, x): out_shape = x.shape[:-1] + (self.outfeatures,) out = self.woq_linear(x.reshape(-1, x.shape[-1])) - out = out + self.bias if self.bias is not None else out return out.reshape(out_shape)