Unverified Commit d7a4f220 authored by Woosuk Kwon's avatar Woosuk Kwon Committed by GitHub
Browse files

[V1] Do not use inductor for piecewise CUDA graphs (#10225)


Signed-off-by: default avatarWoosuk Kwon <woosuk.kwon@berkeley.edu>
parent f9dadfbe
...@@ -404,15 +404,14 @@ class GPUModelRunner: ...@@ -404,15 +404,14 @@ class GPUModelRunner:
def load_model(self) -> None: def load_model(self) -> None:
if self.use_cuda_graph: if self.use_cuda_graph:
# FIXME(woosuk): Currently, the custom ops are not supported # FIXME(woosuk): Currently, we do not use inductor to reduce the
# in the piecewise compilation mode. We rely on TorchInductor # compilation time and any potential issues with the inductor.
# to optimize the model.
os.environ["VLLM_CUSTOM_OPS"] = "none" os.environ["VLLM_CUSTOM_OPS"] = "none"
set_compilation_config( set_compilation_config(
CompilationConfig( CompilationConfig(
use_cudagraph=True, use_cudagraph=True,
non_cudagraph_ops=["vllm.unified_v1_flash_attention"], non_cudagraph_ops=["vllm.unified_v1_flash_attention"],
use_inductor=True, use_inductor=False,
)) ))
logger.info("Starting to load model %s...", self.model_config.model) logger.info("Starting to load model %s...", self.model_config.model)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment