Skip to content

GitLab

Unverified Commit cfb3bf25 authored by yansh97's avatar yansh97 Committed by GitHub
Browse files

[bugfix] fix the default value of llm_int8_threshold in BitsAndBytesConfig (#10657)

parent 1bf905dd
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
vllm/model_executor/layers/quantization/bitsandbytes.py
View file @ cfb3bf25
...@@ -26,7 +26,7 @@ class BitsAndBytesConfig(QuantizationConfig): ...@@ -26,7 +26,7 @@ class BitsAndBytesConfig(QuantizationConfig):
llm_int8_enable_fp32_cpu_offload: bool = False, llm_int8_enable_fp32_cpu_offload: bool = False,
llm_int8_has_fp16_weight: bool = False, llm_int8_has_fp16_weight: bool = False,
llm_int8_skip_modules: Optional[List[str]] = None, llm_int8_skip_modules: Optional[List[str]] = None,
llm_int8_threshold: float = 0.0, llm_int8_threshold: float = 6.0,
) -> None: ) -> None:
self.load_in_8bit = load_in_8bit self.load_in_8bit = load_in_8bit
...@@ -103,7 +103,7 @@ class BitsAndBytesConfig(QuantizationConfig): ...@@ -103,7 +103,7 @@ class BitsAndBytesConfig(QuantizationConfig):
["llm_int8_skip_modules"], ["llm_int8_skip_modules"],
default_value=[]) default_value=[])
llm_int8_threshold = get_safe_value(config, ["llm_int8_threshold"], llm_int8_threshold = get_safe_value(config, ["llm_int8_threshold"],
default_value=0.0) default_value=6.0)
return cls( return cls(
load_in_8bit=load_in_8bit, load_in_8bit=load_in_8bit,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment