Temporarily enforce eager mode for GPTQ models (#2154)

3a765bd5 · Woosuk Kwon · GitHub · 26c52a5e · 3a765bd5
Unverified Commit 3a765bd5 authored Dec 17, 2023 by Woosuk Kwon Committed by GitHub Dec 17, 2023
Show whitespace changes
Inline Side-by-side

Showing with 5 additions and 0 deletions

vllm/config.py vllm/config.py +5 -0

No files found.
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -185,6 +185,11 @@ class ModelConfig:
            self.max_context_len_to_capture = self.max_model_len
        self.max_context_len_to_capture = min(self.max_context_len_to_capture,
                                              self.max_model_len)
+        if self.quantization == "gptq" and not self.enforce_eager:
+            # Related issue: https://github.com/vllm-project/vllm/issues/2147
+            logger.warning("GPTQ does not support CUDA graph yet. Disabling "
+                           "CUDA graph.")
+            self.enforce_eager = True
    def verify_with_parallel_config(
        self,