[BugFix] Fix a bug in counting running sequences (#266)

526df28f · Woosuk Kwon · GitHub · 2cf1a333 · 526df28f
Unverified Commit 526df28f authored Jun 26, 2023 by Woosuk Kwon Committed by GitHub Jun 26, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 6 additions and 2 deletions

vllm/core/scheduler.py vllm/core/scheduler.py +6 -2

No files found.
--- a/vllm/core/scheduler.py
+++ b/vllm/core/scheduler.py
@@ -156,7 +156,9 @@ class Scheduler:
            # The total number of sequences in the RUNNING state should not
            # exceed the maximum number of sequences.
            num_new_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
-            num_curr_seqs = len(self.running)
+            num_curr_seqs = sum(
+                seq_group.num_seqs(status=SequenceStatus.RUNNING)
+                for seq_group in self.running)
            if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
                break

@@ -198,7 +200,9 @@ class Scheduler:
                # The total number of sequences in the RUNNING state should not
                # exceed the maximum number of sequences.
                num_new_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
-                num_curr_seqs = len(self.running)
+                num_curr_seqs = sum(
+                    seq_group.num_seqs(status=SequenceStatus.RUNNING)
+                    for seq_group in self.running)
                if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
                    break