Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "c907d22158dbf31508e73e89cb2bb272bb59e37e"
Unverified Commit a049b107 authored by Jee Jee Li's avatar Jee Jee Li Committed by GitHub
Browse files

[Misc] Temporarily resolve the error of BitAndBytes (#7308)

parent 8334c39f
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 2 deletions
vllm/config.py
View file @ a049b107
...@@ -322,8 +322,9 @@ class ModelConfig: ...@@ -322,8 +322,9 @@ class ModelConfig:
"BitAndBytes quantization with TP or PP is not supported yet.") "BitAndBytes quantization with TP or PP is not supported yet.")
if self.quantization == "bitsandbytes" and self.enforce_eager is False: if self.quantization == "bitsandbytes" and self.enforce_eager is False:
raise ValueError( logger.warning("CUDA graph is not supported on BitAndBytes yet, "
"BitAndBytes with enforce_eager = False is not supported yet.") "fallback to the eager mode.")
self.enforce_eager = True
def get_hf_config_sliding_window(self) -> Optional[int]: def get_hf_config_sliding_window(self) -> Optional[int]:
"""Get the sliding window size, or None if disabled.""" """Get the sliding window size, or None if disabled."""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment