Skip to content

GitLab

Commit 65225897 authored by zhuwenwen's avatar zhuwenwen
Browse files

support adaptive max_num_seqs and cuda_graph_sizes

parent 9b67a1e0
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
vllm/config.py
View file @ 65225897
...@@ -4754,8 +4754,8 @@ class VllmConfig: ...@@ -4754,8 +4754,8 @@ class VllmConfig:
batch_size_capture_list = [] batch_size_capture_list = []
if self.model_config is not None and \ if self.model_config is not None and \
not self.model_config.enforce_eager: not self.model_config.enforce_eager:
if self.model_config.use_mla and self.compilation_config.full_cuda_graph: if self.model_config.use_mla and self.compilation_config.full_cuda_graph and self.scheduler_config.max_num_seqs<=256:
cuda_graph_sizes = [256] cuda_graph_sizes = [self.scheduler_config.max_num_seqs]
else: else:
cuda_graph_sizes = self.scheduler_config.cuda_graph_sizes cuda_graph_sizes = self.scheduler_config.cuda_graph_sizes
if len(cuda_graph_sizes) == 1: if len(cuda_graph_sizes) == 1:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment