From cbf047b4aef7e77c1a7e0724184f2c0285a0d039 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Mon, 25 Sep 2023 10:02:49 +0000 Subject: [PATCH] Support TheBloke exported models. --- server/text_generation_server/utils/weights.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 7b492ee7..45807949 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -345,4 +345,15 @@ class Weights: self.gptq_bits = data["bits"] self.gptq_groupsize = data["group_size"] except Exception: - pass + filename = "quant_config.json" + try: + if os.path.exists(os.path.join(model_id, filename)): + filename = os.path.join(model_id, filename) + else: + filename = hf_hub_download(model_id, filename=filename) + with open(filename, "r") as f: + data = json.load(f) + self.gptq_bits = data["w_bit"] + self.gptq_groupsize = data["q_group_size"] + except Exception: + pass