huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-25 20:12:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

server quantize: expose groupsize option (#2225)

Browse Source
This commit is contained in:
Daniël de Kok 2024-07-16 08:36:05 +02:00 committed by yuanwu
parent e955f7b536
commit 7177da0df6
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/text_generation_server/cli.py
View File

@ -341,6 +341,7 @@ def quantize(
upload_to_model_id: Optional[str] = None,
percdamp: float = 0.01,
act_order: bool = False,
groupsize: int = 128,
):
if revision is None:
revision = "main"
@ -355,7 +356,7 @@ def quantize(
quantize(
model_id=model_id,
bits=4,
groupsize=128,
groupsize=groupsize,
output_dir=output_dir,
revision=revision,
trust_remote_code=trust_remote_code,