From 987b0fff3aa8082c340726543db6b40782294b12 Mon Sep 17 00:00:00 2001 From: Antoni Baum Date: Thu, 27 Jul 2023 18:03:04 -0700 Subject: [PATCH] Load `quantize_config.json` from local path --- server/text_generation_server/utils/weights.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py index 0330402d..bad68735 100644 --- a/server/text_generation_server/utils/weights.py +++ b/server/text_generation_server/utils/weights.py @@ -220,9 +220,13 @@ class Weights: return bits, groupsize - def _set_gptq_params(self, model_id): + def _set_gptq_params(self, model_id: str): try: - filename = hf_hub_download(model_id, filename="quantize_config.json") + potential_local_path = Path(model_id, "quantize_config.json") + if potential_local_path.exists(): + filename = potential_local_path + else: + filename = hf_hub_download(model_id, filename="quantize_config.json") with open(filename, "r") as f: data = json.load(f) self.gptq_bits = data["bits"]