huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing quantize script on models with non parameters buffers.

Browse Source
This commit is contained in:
Nicolas Patry 2023-07-20 11:16:34 +00:00
parent fe80f5360c
commit 929e374753
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/text_generation_server/utils/gptq/quantize.py
View File

@ -812,10 +812,13 @@ def load_weights_pre_hook(module_name, weights, recursive=False):
tensor = weights.get_tensor(tensor_name) tensor = weights.get_tensor(tensor_name)
setdeepattr(module, local_param, nn.Parameter(tensor)) setdeepattr(module, local_param, nn.Parameter(tensor))
else: else:
tensor = current_tensor.to(device=torch.device("cuda:0"))
if current_tensor.requires_grad:
tensor = nn.Parameter(tensor)
setdeepattr( setdeepattr(
module, module,
local_param, local_param,
nn.Parameter(current_tensor.to(device=torch.device("cuda:0"))), tensor
) )
return inner return inner