enforce eager mode with bnb quantization temporarily (#6846)

bb549467 · chenqianfzh · GitHub · b5f49ee5 · bb549467
Unverified Commit bb549467 authored Jul 26, 2024 by chenqianfzh Committed by GitHub Jul 27, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 0 deletions

vllm/config.py vllm/config.py +4 -0

No files found.
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -282,6 +282,10 @@ class ModelConfig:
            raise ValueError(
                "BitAndBytes quantization with TP or PP is not supported yet.")

+        if self.quantization == "bitsandbytes" and self.enforce_eager is False:
+            raise ValueError(
+                "BitAndBytes with enforce_eager = False is not supported yet.")
+
    def get_hf_config_sliding_window(self) -> Optional[int]:
        """Get the sliding window size, or None if disabled."""