`server quantize`: expose groupsize option (#2225)

2cb18428 · Daniël de Kok · GitHub · 06d0e880 · 2cb18428
Unverified Commit 2cb18428 authored Jul 16, 2024 by Daniël de Kok Committed by GitHub Jul 16, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 1 deletion

server/text_generation_server/cli.py server/text_generation_server/cli.py +2 -1

No files found.
--- a/server/text_generation_server/cli.py
+++ b/server/text_generation_server/cli.py
@@ -341,6 +341,7 @@ def quantize(
    upload_to_model_id: Optional[str] = None,
    percdamp: float = 0.01,
    act_order: bool = False,
+    groupsize: int = 128,
 ):
    if revision is None:
        revision = "main"
@@ -355,7 +356,7 @@ def quantize(
    quantize(
        model_id=model_id,
        bits=4,
-        groupsize=128,
+        groupsize=groupsize,
        output_dir=output_dir,
        revision=revision,
        trust_remote_code=trust_remote_code,