From 780198b9e4f884ebd46923c80458613ac092bdaa Mon Sep 17 00:00:00 2001 From: Florian Zimmermeister Date: Tue, 11 Jul 2023 18:37:09 +0200 Subject: [PATCH] Update santacoder.py --- server/text_generation_server/models/santacoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py index a2b38737..d73c07f8 100644 --- a/server/text_generation_server/models/santacoder.py +++ b/server/text_generation_server/models/santacoder.py @@ -56,7 +56,7 @@ class SantaCoder(CausalLM): model_id, revision=revision, torch_dtype=dtype, - load_in_8bit=quantize == "bitsandbytes", + load_in_4bit=quantize == "bitsandbytes", trust_remote_code=trust_remote_code, ).to(device)