From af2c589cef9dd482720695d87a5353d46eee2de2 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 2 Feb 2024 14:42:42 +0100 Subject: [PATCH] none g_idx --- server/text_generation_server/utils/weights.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 759ea602..f600a296 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -177,6 +177,8 @@ class Weights: dtype=torch.int32, device=qweight.device, ) + else: + g_idx = None weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False) else: @@ -242,6 +244,8 @@ class Weights: dtype=torch.int32, device=qweight.device, ) + else: + g_idx = None from text_generation_server.utils.layers import HAS_EXLLAMA