scheduler.py 107 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
14
15
"""A scheduler that manages a tensor parallel GPU worker."""

16
import faulthandler
17
import logging
18
import os
19
import signal
20
import sys
Lianmin Zheng's avatar
Lianmin Zheng committed
21
import threading
22
import time
23
from collections import defaultdict, deque
Lianmin Zheng's avatar
Lianmin Zheng committed
24
from concurrent import futures
25
from dataclasses import dataclass
26
from pathlib import Path
27
from types import SimpleNamespace
28
from typing import Dict, List, Optional, Tuple, Union
29

30
import psutil
31
import setproctitle
32
import torch
33
import zmq
34
from torch.distributed import barrier
35

36
from sglang.global_config import global_config
Lianmin Zheng's avatar
Lianmin Zheng committed
37
from sglang.srt.configs.model_config import ModelConfig
38
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE, GPU_MEMORY_TYPE_WEIGHTS
39
40
41
42
from sglang.srt.constrained.base_grammar_backend import (
    INVALID_GRAMMAR_OBJ,
    create_grammar_backend,
)
Byron Hsu's avatar
Byron Hsu committed
43
44
45
46
47
from sglang.srt.disaggregation.decode import (
    DecodePreallocQueue,
    DecodeTransferQueue,
    SchedulerDisaggregationDecodeMixin,
)
48
from sglang.srt.disaggregation.kv_events import EventPublisherFactory, KVEventBatch
Byron Hsu's avatar
Byron Hsu committed
49
50
51
52
53
54
from sglang.srt.disaggregation.prefill import (
    PrefillBootstrapQueue,
    SchedulerDisaggregationPrefillMixin,
)
from sglang.srt.disaggregation.utils import (
    DisaggregationMode,
55
    MetadataBuffers,
Byron Hsu's avatar
Byron Hsu committed
56
    ReqToMetadataIdxAllocator,
57
    TransferBackend,
58
    prepare_abort,
Byron Hsu's avatar
Byron Hsu committed
59
)
60
from sglang.srt.distributed import get_pp_group, get_world_group
xm:D's avatar
xm:D committed
61
62
63
64
65
from sglang.srt.hf_transformers_utils import (
    get_processor,
    get_tokenizer,
    get_tokenizer_from_processor,
)
66
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
67
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
68
69
70
from sglang.srt.managers.expert_distribution import (
    get_global_expert_distribution_recorder,
)
71
72
from sglang.srt.managers.io_struct import (
    AbortReq,
73
    CloseSessionReqInput,
74
    ExpertDistributionReq,
75
    ExpertDistributionReqOutput,
76
77
    FlushCacheReqInput,
    FlushCacheReqOutput,
78
79
    GetInternalStateReq,
    GetInternalStateReqOutput,
80
81
    GetWeightsByNameReqInput,
    GetWeightsByNameReqOutput,
82
    HealthCheckOutput,
83
84
    InitWeightsUpdateGroupReqInput,
    InitWeightsUpdateGroupReqOutput,
85
86
    OpenSessionReqInput,
    OpenSessionReqOutput,
87
    ProfileReq,
88
89
    ProfileReqOutput,
    ProfileReqType,
90
91
92
93
    ReleaseMemoryOccupationReqInput,
    ReleaseMemoryOccupationReqOutput,
    ResumeMemoryOccupationReqInput,
    ResumeMemoryOccupationReqOutput,
94
95
    RpcReqInput,
    RpcReqOutput,
96
97
    SetInternalStateReq,
    SetInternalStateReqOutput,
98
99
    SlowDownReqInput,
    SlowDownReqOutput,
100
101
    TokenizedEmbeddingReqInput,
    TokenizedGenerateReqInput,
Chayenne's avatar
Chayenne committed
102
103
    UpdateWeightFromDiskReqInput,
    UpdateWeightFromDiskReqOutput,
104
105
    UpdateWeightsFromDistributedReqInput,
    UpdateWeightsFromDistributedReqOutput,
106
107
    UpdateWeightsFromTensorReqInput,
    UpdateWeightsFromTensorReqOutput,
108
)
109
from sglang.srt.managers.mm_utils import init_embedding_cache
110
111
from sglang.srt.managers.schedule_batch import (
    FINISH_ABORT,
Mick's avatar
Mick committed
112
    MultimodalInputs,
113
114
    Req,
    ScheduleBatch,
115
    global_server_args_dict,
116
)
117
118
119
120
121
from sglang.srt.managers.schedule_policy import (
    AddReqResult,
    PrefillAdder,
    SchedulePolicy,
)
122
123
124
from sglang.srt.managers.scheduler_output_processor_mixin import (
    SchedulerOutputProcessorMixin,
)
125
from sglang.srt.managers.session_controller import Session
126
from sglang.srt.managers.tp_worker import TpModelWorker
127
from sglang.srt.managers.tp_worker_overlap_thread import TpModelWorkerClient
128
from sglang.srt.managers.utils import validate_input_length
129
from sglang.srt.mem_cache.chunk_cache import ChunkCache
130
from sglang.srt.mem_cache.hiradix_cache import HiRadixCache
131
from sglang.srt.mem_cache.radix_cache import RadixCache
132
from sglang.srt.metrics.collector import SchedulerMetricsCollector, SchedulerStats
Lianmin Zheng's avatar
Lianmin Zheng committed
133
from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors
134
from sglang.srt.reasoning_parser import ReasoningParser
135
from sglang.srt.server_args import PortArgs, ServerArgs
136
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
137
from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter
138
from sglang.srt.two_batch_overlap import TboDPAttentionPreparer
139
from sglang.srt.utils import (
140
    DeepEPMode,
141
    DynamicGradMode,
142
    broadcast_pyobj,
fzyzcjy's avatar
fzyzcjy committed
143
    configure_gc_logger,
144
    configure_logger,
Lianmin Zheng's avatar
Lianmin Zheng committed
145
    disable_request_logging,
146
    get_available_gpu_memory,
147
    get_bool_env_var,
148
    get_zmq_socket,
Lianmin Zheng's avatar
Lianmin Zheng committed
149
    kill_itself_when_parent_died,
150
    point_to_point_pyobj,
151
    pyspy_dump_schedulers,
152
153
    require_mlp_sync,
    require_mlp_tp_gather,
154
    set_gpu_proc_affinity,
155
156
157
    set_random_seed,
    suppress_other_loggers,
)
158
from sglang.utils import TypeBasedDispatcher, get_exception_traceback
159
160
161

logger = logging.getLogger(__name__)

162
# Test retract decode for debugging purposes
163
164
TEST_RETRACT = get_bool_env_var("SGLANG_TEST_RETRACT")
RECORD_STEP_TIME = get_bool_env_var("SGLANG_RECORD_STEP_TIME")
165
GRAMMAR_TIMEOUT = float(os.environ.get("SGLANG_GRAMMAR_TIMEOUT", 300))
166

167

168
169
@dataclass
class GenerationBatchResult:
170
171
172
    logits_output: Optional[LogitsProcessorOutput]
    pp_hidden_states_proxy_tensors: Optional[torch.Tensor]
    next_token_ids: Optional[List[int]]
173
174
    extend_input_len_per_req: List[int]
    extend_logprob_start_len_per_req: List[int]
175
    bid: int
176
    can_run_cuda_graph: bool
177
178
179
180
181
182
183
184


@dataclass
class EmbeddingBatchResult:
    embeddings: torch.Tensor
    bid: int


185
186
187
188
189
190
191
192
193
194
195
196
class KvMetrics:
    def __init__(self):
        self.request_active_slots = None
        self.request_total_slots = None
        self.kv_active_blocks = None
        self.kv_total_blocks = None
        self.num_requests_waiting = None
        self.gpu_cache_usage_perc = None
        self.gpu_prefix_cache_hit_rate = None
        self.data_parallel_rank = None


197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
class IdleSleeper:
    """
    In setups which have long inactivity periods it is desirable to reduce
    system power consumption when sglang does nothing. This would lead not only
    to power savings, but also to more CPU thermal headroom when a request
    eventually comes. This is important in cases when multiple GPUs are connected
    as each GPU would otherwise pin one thread at 100% CPU usage.

    The simplest solution is to use zmq.Poller on all sockets that may receive
    data that needs handling immediately.
    """

    def __init__(self, sockets):
        self.poller = zmq.Poller()
        for s in sockets:
            self.poller.register(s, zmq.POLLIN)

    def maybe_sleep(self):
        self.poller.poll(1000)


Byron Hsu's avatar
Byron Hsu committed
218
219
220
221
222
class Scheduler(
    SchedulerOutputProcessorMixin,
    SchedulerDisaggregationDecodeMixin,
    SchedulerDisaggregationPrefillMixin,
):
223
224
225
226
227
228
229
230
    """A scheduler that manages a tensor parallel GPU worker."""

    def __init__(
        self,
        server_args: ServerArgs,
        port_args: PortArgs,
        gpu_id: int,
        tp_rank: int,
231
        pp_rank: int,
232
        dp_rank: Optional[int],
233
234
    ):
        # Parse args
235
        self.server_args = server_args
236
        self.tp_rank = tp_rank
237
        self.pp_rank = pp_rank
238
        self.dp_rank = dp_rank
239
        self.tp_size = server_args.tp_size
240
241
        self.pp_size = server_args.pp_size
        self.dp_size = server_args.dp_size
242
243
244
        self.schedule_policy = server_args.schedule_policy
        self.lora_paths = server_args.lora_paths
        self.max_loras_per_batch = server_args.max_loras_per_batch
245
        self.enable_overlap = not server_args.disable_overlap_schedule
246
        self.skip_tokenizer_init = server_args.skip_tokenizer_init
247
        self.enable_metrics = server_args.enable_metrics
248
        self.enable_kv_cache_events = server_args.kv_events_config is not None
249
        self.stream_interval = server_args.stream_interval
250
251
252
        self.spec_algorithm = SpeculativeAlgorithm.from_string(
            server_args.speculative_algorithm
        )
253
254
        self.gpu_id = gpu_id
        self.enable_hierarchical_cache = server_args.enable_hierarchical_cache
Lianmin Zheng's avatar
Lianmin Zheng committed
255
        self.page_size = server_args.page_size
256
257
        self.dp_size = server_args.dp_size
        self.attn_tp_rank, self.attn_tp_size, self.attn_dp_rank = (
258
259
260
261
262
263
264
265
            compute_dp_attention_world_info(
                server_args.enable_dp_attention,
                self.tp_rank,
                self.tp_size,
                self.dp_size,
            )
        )

266
267
        # Init inter-process communication
        context = zmq.Context(2)
268
269
        self.idle_sleeper = None

270
        if self.pp_rank == 0 and self.attn_tp_rank == 0:
271
            self.recv_from_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
272
                context, zmq.PULL, port_args.scheduler_input_ipc_name, False
273
            )
274
            self.send_to_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
275
                context, zmq.PUSH, port_args.tokenizer_ipc_name, False
276
            )
277
278
279
            self.send_metrics_from_scheduler = get_zmq_socket(
                context, zmq.PUSH, port_args.metrics_ipc_name, False
            )
280

281
            if server_args.skip_tokenizer_init:
282
                # Directly send to the TokenizerManager
283
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
284
                    context, zmq.PUSH, port_args.tokenizer_ipc_name, False
285
286
                )
            else:
287
                # Send to the DetokenizerManager
288
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
289
                    context, zmq.PUSH, port_args.detokenizer_ipc_name, False
290
                )
291
292
293
294

            self.recv_from_rpc = get_zmq_socket(
                context, zmq.DEALER, port_args.rpc_ipc_name, False
            )
295
296
297
298
299
300
301
            if self.server_args.sleep_on_idle:
                self.idle_sleeper = IdleSleeper(
                    [
                        self.recv_from_tokenizer,
                        self.recv_from_rpc,
                    ]
                )
302
        else:
303
            self.recv_from_tokenizer = None
304
            self.recv_from_rpc = None
305
            self.send_metrics_from_scheduler = None
306
307
            self.send_to_tokenizer = SimpleNamespace(send_pyobj=lambda x: None)
            self.send_to_detokenizer = SimpleNamespace(send_pyobj=lambda x: None)
308
309

        # Init tokenizer
310
        self.init_tokenizer()
311

312
313
314
315
316
317
318
319
320
        # Set reasoning_parser and think_end_id if --reasoning_parser is enabled
        if self.server_args.reasoning_parser and self.tokenizer:
            reasoning_parser = ReasoningParser(
                model_type=self.server_args.reasoning_parser, stream_reasoning=False
            )
            self.tokenizer.think_end_id = self.tokenizer.encode(
                reasoning_parser.detector.think_end_token, add_special_tokens=False
            )[0]

321
322
323
324
        # Check whether overlap can be enabled
        if not self.is_generation:
            self.enable_overlap = False
            logger.info("Overlap scheduler is disabled for embedding models.")
325

326
        # Launch a tensor parallel worker
327
        if self.enable_overlap:
328
            TpWorkerClass = TpModelWorkerClient
329
330
        else:
            TpWorkerClass = TpModelWorker
331

332
        self.tp_worker = TpWorkerClass(
333
            server_args=server_args,
334
335
            gpu_id=gpu_id,
            tp_rank=tp_rank,
336
            pp_rank=pp_rank,
337
            dp_rank=dp_rank,
338
            nccl_port=port_args.nccl_port,
339
        )
340

341
        # Launch a draft worker for speculative decoding
342
343
344
345
346
347
348
349
350
351
352
353
354
355
        if self.spec_algorithm.is_eagle():
            from sglang.srt.speculative.eagle_worker import EAGLEWorker

            self.draft_worker = EAGLEWorker(
                gpu_id=gpu_id,
                tp_rank=tp_rank,
                server_args=server_args,
                nccl_port=port_args.nccl_port,
                target_worker=self.tp_worker,
                dp_rank=dp_rank,
            )
        else:
            self.draft_worker = None

356
        # Get token and memory info from the model worker
357
358
359
360
        (
            self.max_total_num_tokens,
            self.max_prefill_tokens,
            self.max_running_requests,
361
            self.max_req_len,
362
363
            self.max_req_input_len,
            self.random_seed,
364
            self.device,
365
366
367
368
369
            worker_global_server_args_dict,
            _,
            _,
            _,
        ) = self.tp_worker.get_worker_info()
370
371
372
373
374
375
376
377
        if global_server_args_dict["max_micro_batch_size"] is None:
            global_server_args_dict["max_micro_batch_size"] = max(
                self.max_running_requests // server_args.pp_size, 1
            )

        self.tp_group = self.tp_worker.get_tp_group()
        self.tp_cpu_group = self.tp_group.cpu_group
        self.attn_tp_group = self.tp_worker.get_attention_tp_group()
378
        self.attn_tp_cpu_group = self.tp_worker.get_attention_tp_cpu_group()
379
380
381
        self.pp_group = get_pp_group()
        self.world_group = get_world_group()

382
        self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
383
        global_server_args_dict.update(worker_global_server_args_dict)
384
        set_random_seed(self.random_seed)
385

386
        # Print debug info
387
        if tp_rank == 0:
388
389
390
            avail_mem = get_available_gpu_memory(
                self.device, self.gpu_id, empty_cache=False
            )
391
392
393
394
395
            logger.info(
                f"max_total_num_tokens={self.max_total_num_tokens}, "
                f"chunked_prefill_size={server_args.chunked_prefill_size}, "
                f"max_prefill_tokens={self.max_prefill_tokens}, "
                f"max_running_requests={self.max_running_requests}, "
396
397
                f"context_len={self.model_config.context_len}, "
                f"available_gpu_mem={avail_mem:.2f} GB"
398
            )
399

Lianmin Zheng's avatar
Lianmin Zheng committed
400
        # Init memory pool and cache
401
        self.init_memory_pool_and_cache()
402
403
404

        # Init running status
        self.waiting_queue: List[Req] = []
405
        # The running decoding batch for continuous batching
Lianmin Zheng's avatar
Lianmin Zheng committed
406
        self.running_batch: ScheduleBatch = ScheduleBatch(reqs=[], batch_is_full=False)
407
        # The current forward batch
Lianmin Zheng's avatar
Lianmin Zheng committed
408
        self.cur_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
409
        # The last forward batch
410
        self.last_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
411
412
        self.forward_ct = 0
        self.forward_ct_decode = 0
413
        self.num_generated_tokens = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
414
        self.last_prefill_tokens = 0
415
416
        self.last_decode_stats_tic = time.perf_counter()
        self.last_prefill_stats_tic = time.perf_counter()
417
        self.return_health_check_ct = 0
418
        self.current_stream = torch.get_device_module(self.device).current_stream()
419
420
        if self.device == "cpu":
            self.current_stream.synchronize = lambda: None  # No-op for CPU
421
        self.forward_sleep_time = None
422

423
        # Init session info
424
        self.sessions: Dict[str, Session] = {}
425
426
427

        # Init chunked prefill
        self.chunked_prefill_size = server_args.chunked_prefill_size
428
429
        if self.chunked_prefill_size <= 0:  # -1 means disable
            self.chunked_prefill_size = None
430
        self.chunked_req = None
431
432
433
434
        self.is_mixed_chunk = (
            self.chunked_prefill_size is not None and server_args.enable_mixed_chunk
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
435
        # Init the grammar backend for constrained generation
436
        self.grammar_queue: List[Req] = []
437
        if not server_args.skip_tokenizer_init:
438
439
440
            self.grammar_backend = create_grammar_backend(
                server_args, self.tokenizer, self.model_config.vocab_size
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
441
442
        else:
            self.grammar_backend = None
443

444
        # Init schedule policy and new token estimation
445
        self.policy = SchedulePolicy(
Lianmin Zheng's avatar
Lianmin Zheng committed
446
447
448
            self.schedule_policy,
            self.tree_cache,
            self.enable_hierarchical_cache,
449
        )
450
451
452
        assert (
            server_args.schedule_conservativeness >= 0
        ), "Invalid schedule_conservativeness"
453
454
        self.init_new_token_ratio = min(
            global_config.default_init_new_token_ratio
455
456
            * server_args.schedule_conservativeness,
            1.0,
457
        )
458
459
460
461
462
463
464
465
466
467
        self.min_new_token_ratio = min(
            self.init_new_token_ratio
            * global_config.default_min_new_token_ratio_factor,
            1.0,
        )
        self.new_token_ratio_decay = (
            self.init_new_token_ratio - self.min_new_token_ratio
        ) / global_config.default_new_token_ratio_decay_steps
        self.new_token_ratio = self.init_new_token_ratio

Lianmin Zheng's avatar
Lianmin Zheng committed
468
469
470
471
        # Init watchdog thread
        self.watchdog_timeout = server_args.watchdog_timeout
        t = threading.Thread(target=self.watchdog_thread, daemon=True)
        t.start()
472
        self.parent_process = psutil.Process().parent()
473
474
475
476
        self.memory_saver_adapter = TorchMemorySaverAdapter.create(
            enable=server_args.enable_memory_saver
        )

477
        # Init profiler
478
479
        self.torch_profiler = None
        self.torch_profiler_output_dir: Optional[str] = None
480
        self.profiler_activities: Optional[List[str]] = None
481
        self.profile_id: Optional[str] = None
482
        self.profiler_target_forward_ct: Optional[int] = None
483
484
485
486
487
488
489
490
        self.profiler_target_prefill_ct: Optional[int] = None
        self.profiler_target_decode_ct: Optional[int] = None
        self.profiler_prefill_ct: Optional[int] = None
        self.profiler_decode_ct: Optional[int] = None
        self.profile_by_stage: bool = False
        self.profile_steps: Optional[int] = None
        self.profile_in_progress: bool = False
        self.rpd_profiler = None
491

492
        # Init metrics stats
493
        self.init_metrics()
494
        self.init_kv_events(server_args.kv_events_config)
495

496
497
        # Init request dispatcher
        self._request_dispatcher = TypeBasedDispatcher(
498
499
500
            [
                (TokenizedGenerateReqInput, self.handle_generate_request),
                (TokenizedEmbeddingReqInput, self.handle_embedding_request),
501
                (FlushCacheReqInput, self.flush_cache_wrapped),
502
                (AbortReq, self.abort_request),
503
504
                (OpenSessionReqInput, self.open_session),
                (CloseSessionReqInput, self.close_session),
505
506
507
508
509
510
511
512
                (UpdateWeightFromDiskReqInput, self.update_weights_from_disk),
                (InitWeightsUpdateGroupReqInput, self.init_weights_update_group),
                (
                    UpdateWeightsFromDistributedReqInput,
                    self.update_weights_from_distributed,
                ),
                (UpdateWeightsFromTensorReqInput, self.update_weights_from_tensor),
                (GetWeightsByNameReqInput, self.get_weights_by_name),
513
514
                (ReleaseMemoryOccupationReqInput, self.release_memory_occupation),
                (ResumeMemoryOccupationReqInput, self.resume_memory_occupation),
515
                (SlowDownReqInput, self.slow_down),
516
                (ProfileReq, self.profile),
517
                (GetInternalStateReq, self.get_internal_state),
518
                (SetInternalStateReq, self.set_internal_state),
519
                (RpcReqInput, self.handle_rpc_request),
520
                (ExpertDistributionReq, self.expert_distribution_handle),
521
522
523
            ]
        )

Byron Hsu's avatar
Byron Hsu committed
524
525
526
527
528
        self.disaggregation_mode = DisaggregationMode(
            self.server_args.disaggregation_mode
        )
        self.init_disaggregation()

fzyzcjy's avatar
fzyzcjy committed
529
530
531
        if get_bool_env_var("SGLANG_GC_LOG"):
            configure_gc_logger()

532
533
534
535
    def maybe_sleep_on_idle(self):
        if self.idle_sleeper is not None:
            self.idle_sleeper.maybe_sleep()

536
537
    def init_tokenizer(self):
        server_args = self.server_args
Lianmin Zheng's avatar
Lianmin Zheng committed
538

539
        self.model_config = ModelConfig.from_server_args(server_args)
540
        self.is_generation = self.model_config.is_generation
541

542
543
544
545
546
547
548
549
550
        if server_args.skip_tokenizer_init:
            self.tokenizer = self.processor = None
        else:
            if self.model_config.is_multimodal:
                self.processor = get_processor(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
551
                    use_fast=not server_args.disable_fast_image_processor,
552
                )
xm:D's avatar
xm:D committed
553
                self.tokenizer = get_tokenizer_from_processor(self.processor)
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
            else:
                self.tokenizer = get_tokenizer(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
                )

    def init_memory_pool_and_cache(self):
        server_args = self.server_args

        self.req_to_token_pool, self.token_to_kv_pool_allocator = (
            self.tp_worker.get_memory_pool()
        )

        if (
            server_args.chunked_prefill_size is not None
            and server_args.disable_radix_cache
        ):
            self.tree_cache = ChunkCache(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
576
                page_size=self.page_size,
577
578
579
580
581
582
            )
        else:
            if self.enable_hierarchical_cache:
                self.tree_cache = HiRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
583
584
585
586
587
                    tp_cache_group=(
                        self.attn_tp_cpu_group
                        if self.server_args.enable_dp_attention
                        else self.tp_cpu_group
                    ),
588
                    page_size=self.page_size,
589
                    hicache_ratio=server_args.hicache_ratio,
Zhiqiang Xie's avatar
Zhiqiang Xie committed
590
591
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
592
                )
593
594
595
596
                self.tp_worker.register_hicache_layer_transfer_counter(
                    self.tree_cache.cache_controller.layer_done_counter
                )

597
598
599
600
            else:
                self.tree_cache = RadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Lianmin Zheng's avatar
Lianmin Zheng committed
601
                    page_size=self.page_size,
602
                    disable=server_args.disable_radix_cache,
603
                    enable_kv_cache_events=self.enable_kv_cache_events,
604
605
606
607
608
609
610
611
612
613
614
615
                )

        self.decode_mem_cache_buf_multiplier = (
            1
            if self.spec_algorithm.is_none()
            else (
                server_args.speculative_num_draft_tokens
                + (
                    server_args.speculative_eagle_topk
                    * server_args.speculative_num_steps
                )
            )
616
        )
617
618
619

    def init_metrics(self):
        self.last_gen_throughput: float = 0.0
Lianmin Zheng's avatar
Lianmin Zheng committed
620
        self.last_input_throughput: float = 0.0
621
622
623
624
625
626
627
628
629
630
631
632
633
634
        self.step_time_dict = defaultdict(list)  # Dict[batch size -> step time]
        self.spec_num_total_accepted_tokens = 0
        self.spec_num_total_forward_ct = 0
        self.cum_spec_accept_length = 0
        self.cum_spec_accept_count = 0
        self.stats = SchedulerStats()
        if self.enable_metrics:
            engine_type = "unified"
            self.metrics_collector = SchedulerMetricsCollector(
                labels={
                    "model_name": self.server_args.served_model_name,
                    "engine_type": engine_type,
                },
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
635

636
637
    def init_kv_events(self, kv_events_config: Optional[str]):
        if self.enable_kv_cache_events:
638
639
640
            self.kv_event_publisher = EventPublisherFactory.create(
                kv_events_config, self.attn_dp_rank
            )
641

Byron Hsu's avatar
Byron Hsu committed
642
    def init_disaggregation(self):
643
644
645
646
        self.transfer_backend = TransferBackend(
            self.server_args.disaggregation_transfer_backend
        )

Byron Hsu's avatar
Byron Hsu committed
647
648
649
650
        if (
            self.disaggregation_mode == DisaggregationMode.DECODE
        ):  # *2 for the headroom.
            buffer_size = (self.req_to_token_pool.size) * 2
Byron Hsu's avatar
Byron Hsu committed
651
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
652
653
                buffer_size
            )
654
655
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
656
657
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
658
659
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
660
661
662

            # The decode requests polling kv cache
            self.disagg_decode_transfer_queue = DecodeTransferQueue(
663
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
664
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
665
                tp_rank=self.tp_rank,
666
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
667
668
                scheduler=self,
                tree_cache=self.tree_cache,
Byron Hsu's avatar
Byron Hsu committed
669
670
671
672
673
674
            )

            # The decode requests pending for pre-allocation
            self.disagg_decode_prealloc_queue = DecodePreallocQueue(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Byron Hsu's avatar
Byron Hsu committed
675
676
677
678
679
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
680
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
681
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
682
683
684
                scheduler=self,
                transfer_queue=self.disagg_decode_transfer_queue,
                tree_cache=self.tree_cache,
685
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
686
687
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
688
689
                dp_size=self.server_args.dp_size,
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
690
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
691
692
                max_total_num_tokens=self.max_total_num_tokens,
                prefill_pp_size=self.server_args.disaggregation_prefill_pp,
693
                num_reserved_decode_tokens=self.server_args.num_reserved_decode_tokens,
694
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
695
            )
Liangsheng Yin's avatar
Liangsheng Yin committed
696
697
698
699

            # Metric for pre-allocation
            self.num_tokens_pre_allocated = 0

Byron Hsu's avatar
Byron Hsu committed
700
701
702
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            # *2 for the headroom.
            buffer_size = self.max_running_requests * 2
Byron Hsu's avatar
Byron Hsu committed
703
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
704
705
                buffer_size
            )
706
707
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
708
709
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
710
711
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
712

Liangsheng Yin's avatar
Liangsheng Yin committed
713
            self.disagg_prefill_bootstrap_queue = PrefillBootstrapQueue(
Byron Hsu's avatar
Byron Hsu committed
714
                token_to_kv_pool=self.token_to_kv_pool_allocator.get_kvcache(),
Byron Hsu's avatar
Byron Hsu committed
715
716
717
718
719
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
720
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
721
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
722
723
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
Byron Hsu's avatar
Byron Hsu committed
724
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
725
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
726
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
727
728
729
                max_total_num_tokens=self.max_total_num_tokens,
                decode_tp_size=self.server_args.disaggregation_decode_tp,
                decode_dp_size=self.server_args.disaggregation_decode_dp,
730
                scheduler=self,
Byron Hsu's avatar
Byron Hsu committed
731
732
733
                pp_rank=self.pp_rank,
                pp_size=self.pp_size,
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
734
735
            )
            # The prefill requests that are in the middle of kv sending
736
            self.disagg_prefill_inflight_queue: List[Req] = []
Byron Hsu's avatar
Byron Hsu committed
737

738
    @DynamicGradMode()
739
    def event_loop_normal(self):
740
        """A normal scheduler loop."""
741
        while True:
Lianmin Zheng's avatar
Lianmin Zheng committed
742
743
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)
744

745
            batch = self.get_next_batch_to_run()
Lianmin Zheng's avatar
Lianmin Zheng committed
746
            self.cur_batch = batch
747
748
749
750

            if batch:
                result = self.run_batch(batch)
                self.process_batch_result(batch, result)
Lianmin Zheng's avatar
Lianmin Zheng committed
751
            else:
Lianmin Zheng's avatar
Lianmin Zheng committed
752
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
753
                self.check_memory()
754
                self.new_token_ratio = self.init_new_token_ratio
755
                self.maybe_sleep_on_idle()
756
757

            self.last_batch = batch
758

759
    @DynamicGradMode()
Lianmin Zheng's avatar
Lianmin Zheng committed
760
    def event_loop_overlap(self):
761
        """A scheduler loop that overlaps the CPU processing and GPU computation."""
762
        self.result_queue = deque()
Lianmin Zheng's avatar
Lianmin Zheng committed
763
764
765
766
767
768
769

        while True:
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)

            batch = self.get_next_batch_to_run()
            self.cur_batch = batch
770

Lianmin Zheng's avatar
Lianmin Zheng committed
771
            if batch:
772
                batch.launch_done = threading.Event()
Lianmin Zheng's avatar
Lianmin Zheng committed
773
                result = self.run_batch(batch)
774
                self.result_queue.append((batch.copy(), result))
Lianmin Zheng's avatar
Lianmin Zheng committed
775

776
                if self.last_batch is None:
777
                    # Create a dummy first batch to start the pipeline for overlap schedule.
778
779
780
781
782
783
                    # It is now used for triggering the sampling_info_done event.
                    tmp_batch = ScheduleBatch(
                        reqs=None,
                        forward_mode=ForwardMode.DUMMY_FIRST,
                        next_batch_sampling_info=self.tp_worker.cur_sampling_info,
                    )
784
                    self.process_batch_result(tmp_batch, None, batch.launch_done)
785

Lianmin Zheng's avatar
Lianmin Zheng committed
786
            if self.last_batch:
787
                # Process the results of the last batch
788
                tmp_batch, tmp_result = self.result_queue.popleft()
789
790
791
                tmp_batch.next_batch_sampling_info = (
                    self.tp_worker.cur_sampling_info if batch else None
                )
792
793
794
795
                # NOTE: we should use current launched batch's launch_done event Instead of the last batch's
                self.process_batch_result(
                    tmp_batch, tmp_result, batch.launch_done if batch else None
                )
Lianmin Zheng's avatar
Lianmin Zheng committed
796
            elif batch is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
797
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
798
                self.check_memory()
799
                self.new_token_ratio = self.init_new_token_ratio
800
                self.maybe_sleep_on_idle()
Lianmin Zheng's avatar
Lianmin Zheng committed
801
802
803

            self.last_batch = batch

804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
    @DynamicGradMode()
    def event_loop_pp(self):
        """A non-overlap scheduler loop for pipeline parallelism."""
        mbs = [None] * self.pp_size
        last_mbs = [None] * self.pp_size
        self.running_mbs = [
            ScheduleBatch(reqs=[], batch_is_full=False) for _ in range(self.pp_size)
        ]
        bids = [None] * self.pp_size
        pp_outputs: Optional[PPProxyTensors] = None
        while True:
            server_is_idle = True
            for mb_id in range(self.pp_size):
                self.running_batch = self.running_mbs[mb_id]
                self.last_batch = last_mbs[mb_id]

                recv_reqs = self.recv_requests()
                self.process_input_requests(recv_reqs)
                mbs[mb_id] = self.get_next_batch_to_run()
                self.running_mbs[mb_id] = self.running_batch

                self.cur_batch = mbs[mb_id]
                if self.cur_batch:
                    server_is_idle = False
                    result = self.run_batch(self.cur_batch)

830
                # (last rank) send the outputs to the next step
831
832
833
834
835
836
                if self.pp_group.is_last_rank:
                    if self.cur_batch:
                        next_token_ids, bids[mb_id] = (
                            result.next_token_ids,
                            result.bid,
                        )
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
                        if self.cur_batch.return_logprob:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                    "extend_input_len_per_req": result.extend_input_len_per_req,
                                    "extend_logprob_start_len_per_req": result.extend_logprob_start_len_per_req,
                                }
                                | (
                                    {
                                        f"logits_output.{k}": v
                                        for k, v in result.logits_output.__dict__.items()
                                    }
                                    if result.logits_output is not None
                                    else {}
                                )
                            )
                        else:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                }
                            )
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
                        # send the output from the last round to let the next stage worker run post processing
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                # receive outputs and post-process (filter finished reqs) the coming microbatch
                next_mb_id = (mb_id + 1) % self.pp_size
                next_pp_outputs = None
                if mbs[next_mb_id] is not None:
                    next_pp_outputs: Optional[PPProxyTensors] = PPProxyTensors(
                        self.pp_group.recv_tensor_dict(
                            all_gather_group=self.attn_tp_group
                        )
                    )
                    mbs[next_mb_id].output_ids = next_pp_outputs["next_token_ids"]
875
876
877
878
879
880
881
882
883
                    logits_output_args = {
                        k[len("logits_output.") :]: v
                        for k, v in next_pp_outputs.tensors.items()
                        if k.startswith("logits_output.")
                    }
                    if len(logits_output_args) > 0:
                        logits_output = LogitsProcessorOutput(**logits_output_args)
                    else:
                        logits_output = None
884
                    output_result = GenerationBatchResult(
885
                        logits_output=logits_output,
886
887
                        pp_hidden_states_proxy_tensors=None,
                        next_token_ids=next_pp_outputs["next_token_ids"],
888
889
890
891
892
893
                        extend_input_len_per_req=next_pp_outputs.tensors.get(
                            "extend_input_len_per_req", None
                        ),
                        extend_logprob_start_len_per_req=next_pp_outputs.tensors.get(
                            "extend_logprob_start_len_per_req", None
                        ),
894
                        bid=bids[next_mb_id],
895
                        can_run_cuda_graph=result.can_run_cuda_graph,
896
897
898
899
                    )
                    self.process_batch_result(mbs[next_mb_id], output_result)
                    last_mbs[next_mb_id] = mbs[next_mb_id]

900
                # (not last rank)
901
902
903
                if not self.pp_group.is_last_rank:
                    if self.cur_batch:
                        bids[mb_id] = result.bid
904
905
                    # carry the outputs to the next stage
                    # send the outputs from the last round to let the next stage worker run post processing
906
907
908
909
910
911
912
                    if pp_outputs:
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                    # send out reqs to the next stage
913
                    dp_offset = self.attn_dp_rank * self.attn_tp_size
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
                    if self.attn_tp_rank == 0:
                        point_to_point_pyobj(
                            recv_reqs,
                            self.pp_rank * self.tp_size + dp_offset,
                            self.world_group.cpu_group,
                            self.pp_rank * self.tp_size + dp_offset,
                            (self.pp_rank + 1) * self.tp_size + dp_offset,
                        )

                    # send out proxy tensors to the next stage
                    if self.cur_batch:
                        self.pp_group.send_tensor_dict(
                            result.pp_hidden_states_proxy_tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                pp_outputs = next_pp_outputs

            # When the server is idle, self-check and re-init some states
            if server_is_idle:
                self.check_memory()
                self.new_token_ratio = self.init_new_token_ratio
936
                self.maybe_sleep_on_idle()
937

938
939
    def recv_requests(self) -> List[Req]:
        """Receive results at tp_rank = 0 and broadcast it to all other TP ranks."""
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
        if self.pp_rank == 0:
            if self.attn_tp_rank == 0:
                recv_reqs = []

                while True:
                    try:
                        recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_req)

                while True:
                    try:
                        recv_rpc = self.recv_from_rpc.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_rpc)
            else:
                recv_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
959
        else:
960
            if self.attn_tp_rank == 0:
961
                dp_offset = self.attn_dp_rank * self.attn_tp_size
962
963
964
965
966
967
968
969
970
                recv_reqs = point_to_point_pyobj(
                    [],
                    self.pp_rank * self.tp_size + dp_offset,
                    self.world_group.cpu_group,
                    (self.pp_rank - 1) * self.tp_size + dp_offset,
                    self.pp_rank * self.tp_size + dp_offset,
                )
            else:
                recv_reqs = None
971

972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
        if self.server_args.enable_dp_attention:
            if self.attn_tp_rank == 0:
                work_reqs = [
                    req
                    for req in recv_reqs
                    if isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
                control_reqs = [
                    req
                    for req in recv_reqs
                    if not isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
            else:
                work_reqs = None
                control_reqs = None

            if self.attn_tp_size != 1:
                work_reqs = broadcast_pyobj(
                    work_reqs,
995
                    self.attn_tp_group.rank,
996
                    self.attn_tp_cpu_group,
997
                    src=self.attn_tp_group.ranks[0],
998
999
1000
                )
            if self.tp_size != 1:
                control_reqs = broadcast_pyobj(
1001
1002
1003
1004
                    control_reqs,
                    self.tp_group.rank,
                    self.tp_cpu_group,
                    src=self.tp_group.ranks[0],
1005
1006
1007
                )
            recv_reqs = work_reqs + control_reqs
        elif self.tp_size != 1:
1008
1009
1010
1011
1012
1013
            recv_reqs = broadcast_pyobj(
                recv_reqs,
                self.tp_group.rank,
                self.tp_cpu_group,
                src=self.tp_group.ranks[0],
            )
1014
1015
        return recv_reqs

Lianmin Zheng's avatar
Lianmin Zheng committed
1016
    def process_input_requests(self, recv_reqs: List):
1017
        for recv_req in recv_reqs:
1018
1019
            # If it is a health check generation request and there are running requests, ignore it.
            if is_health_check_generate_req(recv_req) and (
Lianmin Zheng's avatar
Lianmin Zheng committed
1020
                self.chunked_req is not None or not self.running_batch.is_empty()
1021
1022
1023
1024
            ):
                self.return_health_check_ct += 1
                continue

1025
            output = self._request_dispatcher(recv_req)
1026
            if output is not None:
1027
1028
1029
1030
1031
                if isinstance(output, RpcReqOutput):
                    if self.recv_from_rpc is not None:
                        self.recv_from_rpc.send_pyobj(output)
                else:
                    self.send_to_tokenizer.send_pyobj(output)
1032
1033
1034
1035
1036

    def handle_generate_request(
        self,
        recv_req: TokenizedGenerateReqInput,
    ):
1037
        # Create a new request
1038
1039
1040
1041
1042
        if (
            recv_req.session_params is None
            or recv_req.session_params.id is None
            or recv_req.session_params.id not in self.sessions
        ):
Rin Intachuen's avatar
Rin Intachuen committed
1043
1044
1045
1046
1047
1048
            if recv_req.input_embeds is not None:
                # Generate fake input_ids based on the length of input_embeds
                seq_length = len(recv_req.input_embeds)
                fake_input_ids = [1] * seq_length
                recv_req.input_ids = fake_input_ids

1049
1050
1051
1052
            if recv_req.bootstrap_port is None:
                # Use default bootstrap port
                recv_req.bootstrap_port = self.server_args.disaggregation_bootstrap_port

1053
1054
1055
1056
1057
            req = Req(
                recv_req.rid,
                recv_req.input_text,
                recv_req.input_ids,
                recv_req.sampling_params,
Lianmin Zheng's avatar
Lianmin Zheng committed
1058
1059
                return_logprob=recv_req.return_logprob,
                top_logprobs_num=recv_req.top_logprobs_num,
1060
                token_ids_logprob=recv_req.token_ids_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1061
                stream=recv_req.stream,
1062
                lora_path=recv_req.lora_path,
Rin Intachuen's avatar
Rin Intachuen committed
1063
                input_embeds=recv_req.input_embeds,
Lianmin Zheng's avatar
Lianmin Zheng committed
1064
                custom_logit_processor=recv_req.custom_logit_processor,
1065
                return_hidden_states=recv_req.return_hidden_states,
1066
                eos_token_ids=self.model_config.hf_eos_token_id,
1067
                bootstrap_host=recv_req.bootstrap_host,
1068
                bootstrap_port=recv_req.bootstrap_port,
1069
                bootstrap_room=recv_req.bootstrap_room,
1070
                data_parallel_rank=recv_req.data_parallel_rank,
1071
1072
            )
            req.tokenizer = self.tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
1073

1074
1075
1076
            if self.disaggregation_mode != DisaggregationMode.NULL:
                # Invalid request for disaggregated mode
                if recv_req.bootstrap_room is None:
1077
                    error_msg = (
1078
1079
1080
                        f"Invalid request: Disaggregated request received without "
                        f"boostrap room id. {req.rid=}"
                    )
1081
1082
                    logger.error(error_msg)
                    prepare_abort(req, error_msg)
1083
1084
1085
                    self.stream_output([req], req.return_logprob)
                    return

1086
1087
1088
1089
            if (
                recv_req.session_params is not None
                and recv_req.session_params.id is not None
            ):
1090
                req.finished_reason = FINISH_ABORT(
1091
                    f"Invalid request: session id {recv_req.session_params.id} does not exist"
1092
                )
1093
                self._add_request_to_queue(req)
1094
1095
                return
        else:
1096
1097
            # Create a new request from a previous session
            session = self.sessions[recv_req.session_params.id]
1098
            req = session.create_req(recv_req, self.tokenizer)
1099
            if isinstance(req.finished_reason, FINISH_ABORT):
1100
                self._add_request_to_queue(req)
1101
                return
1102

1103
        # Handle multimodal inputs
Mick's avatar
Mick committed
1104
1105
        if recv_req.mm_inputs is not None:
            image_inputs = MultimodalInputs.from_dict(recv_req.mm_inputs)
1106
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
1107
            req.origin_input_ids = self.pad_input_ids_func(
1108
                req.origin_input_ids, image_inputs
1109
            )
1110
            req.extend_image_inputs(image_inputs)
1111

1112
            if len(req.origin_input_ids) >= self.max_req_input_len:
1113
1114
1115
1116
1117
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1118
                )
1119
                self._add_request_to_queue(req)
1120
1121
                return

1122
        # Validate prompt length
1123
1124
1125
1126
1127
1128
        error_msg = validate_input_length(
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
        if error_msg:
1129
            req.set_finish_with_abort(error_msg)
1130
            self._add_request_to_queue(req)
1131
            return
1132

1133
        # Copy more attributes
1134
        if recv_req.logprob_start_len == -1 or not recv_req.return_logprob:
1135
1136
1137
1138
1139
            # By default, only return the logprobs for output tokens
            req.logprob_start_len = len(req.origin_input_ids) - 1
        else:
            req.logprob_start_len = recv_req.logprob_start_len

1140
        if req.logprob_start_len >= len(req.origin_input_ids):
1141
            error_msg = f"{req.logprob_start_len=} is higher than the number of input tokens {len(req.origin_input_ids)=}. Please use a smaller logprob_start_len."
1142
            req.logprob_start_len = len(req.origin_input_ids) - 1
1143
            req.set_finish_with_abort(error_msg)
1144
1145
1146
            self._add_request_to_queue(req)
            return

1147
1148
1149
1150
1151
1152
        req.sampling_params.max_new_tokens = min(
            (
                req.sampling_params.max_new_tokens
                if req.sampling_params.max_new_tokens is not None
                else 1 << 30
            ),
1153
            self.max_req_len - len(req.origin_input_ids) - 1,
1154
1155
        )

1156
1157
1158
1159
1160
        # Init grammar cache for this request
        add_to_grammar_queue = False
        if (
            req.sampling_params.json_schema is not None
            or req.sampling_params.regex is not None
1161
            or req.sampling_params.ebnf is not None
1162
            or req.sampling_params.structural_tag is not None
1163
1164
1165
1166
1167
1168
        ):
            assert self.grammar_backend is not None
            if req.sampling_params.json_schema is not None:
                key = ("json", req.sampling_params.json_schema)
            elif req.sampling_params.regex is not None:
                key = ("regex", req.sampling_params.regex)
1169
1170
            elif req.sampling_params.ebnf is not None:
                key = ("ebnf", req.sampling_params.ebnf)
1171
1172
            elif req.sampling_params.structural_tag:
                key = ("structural_tag", req.sampling_params.structural_tag)
1173

1174
1175
1176
1177
1178
            value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
            req.grammar = value

            if not cache_hit:
                req.grammar_key = key
1179
                add_to_grammar_queue = True
1180
1181
1182
1183
            else:
                if value is INVALID_GRAMMAR_OBJ:  # We hit a cached invalid grammar.
                    error_msg = f"Invalid grammar request with cache hit: {key=}"
                    req.set_finish_with_abort(error_msg)
1184
1185

        if add_to_grammar_queue:
1186
            req.queue_time_start = time.perf_counter()
1187
1188
            self.grammar_queue.append(req)
        else:
1189
1190
1191
            self._add_request_to_queue(req)

    def _add_request_to_queue(self, req: Req):
1192
        req.queue_time_start = time.perf_counter()
Byron Hsu's avatar
Byron Hsu committed
1193
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1194
1195
1196
            self.disagg_prefill_bootstrap_queue.add(
                req, self.model_config.num_key_value_heads
            )
Byron Hsu's avatar
Byron Hsu committed
1197
1198
1199
1200
1201
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            self.disagg_decode_prealloc_queue.add(req)
        else:
            self.waiting_queue.append(req)

1202
    def _extend_requests_to_queue(self, reqs: List[Req], is_retracted: bool = False):
1203
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1204
1205
1206
            self.disagg_prefill_bootstrap_queue.extend(
                reqs, self.model_config.num_key_value_heads
            )
1207
1208
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            # If this is a decode server, we put the request to the decode pending prealloc queue
1209
            self.disagg_decode_prealloc_queue.extend(reqs, is_retracted)
Byron Hsu's avatar
Byron Hsu committed
1210
1211
        else:
            self.waiting_queue.extend(reqs)
1212
1213
1214

    def handle_embedding_request(
        self,
1215
        recv_req: TokenizedEmbeddingReqInput,
1216
1217
1218
1219
1220
1221
    ):
        req = Req(
            recv_req.rid,
            recv_req.input_text,
            recv_req.input_ids,
            recv_req.sampling_params,
woodx's avatar
woodx committed
1222
            token_type_ids=recv_req.token_type_ids,
1223
1224
1225
        )
        req.tokenizer = self.tokenizer

1226
1227
        # Handle multimodal inputs
        if recv_req.image_inputs is not None:
Mick's avatar
Mick committed
1228
            image_inputs = MultimodalInputs.from_dict(recv_req.image_inputs)
1229
1230
1231
1232
1233
1234
1235
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
            req.origin_input_ids = self.pad_input_ids_func(
                req.origin_input_ids, image_inputs
            )
            req.extend_image_inputs(image_inputs)

            if len(req.origin_input_ids) >= self.max_req_input_len:
1236
1237
1238
1239
1240
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1241
                )
1242
                self._add_request_to_queue(req)
1243
1244
                return

1245
        # Validate prompts length
1246
        error_msg = validate_input_length(
1247
1248
1249
1250
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
1251
        if error_msg:
1252
            self._add_request_to_queue(req)
1253
            return
1254

1255
1256
        # Copy more attributes
        req.logprob_start_len = len(req.origin_input_ids) - 1
1257
        self._add_request_to_queue(req)
1258

1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
    def _emit_kv_metrics(self):
        kv_metrics = KvMetrics()
        kv_metrics.request_active_slots = self.stats.num_running_reqs
        kv_metrics.request_total_slots = self.max_running_requests
        kv_metrics.kv_active_blocks = int(
            self.stats.token_usage * self.max_total_num_tokens
        )
        kv_metrics.kv_total_blocks = self.max_total_num_tokens
        kv_metrics.num_requests_waiting = self.stats.num_queue_reqs
        kv_metrics.gpu_cache_usage_perc = self.stats.token_usage
        kv_metrics.gpu_prefix_cache_hit_rate = self.stats.cache_hit_rate
        kv_metrics.data_parallel_rank = self.dp_rank if self.dp_rank is not None else 0

        if not self.send_metrics_from_scheduler.closed:
            self.send_metrics_from_scheduler.send_pyobj(kv_metrics)

1275
1276
1277
1278
    def log_prefill_stats(
        self,
        adder: PrefillAdder,
        can_run_list: List[Req],
1279
        running_bs: int,
1280
    ):
1281
1282
        gap_latency = time.perf_counter() - self.last_prefill_stats_tic
        self.last_prefill_stats_tic = time.perf_counter()
Liangsheng Yin's avatar
Liangsheng Yin committed
1283
1284
        self.last_input_throughput = self.last_prefill_tokens / gap_latency
        self.last_prefill_tokens = adder.log_input_tokens
Lianmin Zheng's avatar
Lianmin Zheng committed
1285

1286
        num_used = self.max_total_num_tokens - (
1287
1288
            self.token_to_kv_pool_allocator.available_size()
            + self.tree_cache.evictable_size()
1289
1290
        )

1291
        num_new_seq = len(can_run_list)
1292
        f = (
1293
            f"Prefill batch. "
1294
            f"#new-seq: {num_new_seq}, "
1295
1296
1297
1298
            f"#new-token: {adder.log_input_tokens}, "
            f"#cached-token: {adder.log_hit_tokens}, "
            f"token usage: {num_used / self.max_total_num_tokens:.2f}, "
        )
Liangsheng Yin's avatar
Liangsheng Yin committed
1299
1300
1301
1302

        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            f += f"#unbootstrapped-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, "
            f += f"#queue-req: {len(self.waiting_queue)}, "
fzyzcjy's avatar
fzyzcjy committed
1303
            f += f"#transferring-req: {len(self.disagg_prefill_inflight_queue)}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1304
            f += f"input throughput (token/s): {self.last_input_throughput:.2f} "
Liangsheng Yin's avatar
Liangsheng Yin committed
1305
        else:
Liangsheng Yin's avatar
Liangsheng Yin committed
1306
            f += f"#running-req: {running_bs}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1307
1308
            f += f"#queue-req: {len(self.waiting_queue)}"

1309
        logger.info(f)
1310
1311

        if self.enable_metrics:
1312
1313
1314
            cache_hit_rate = adder.log_hit_tokens / (
                adder.log_input_tokens + adder.log_hit_tokens
            )
1315
1316
1317
            self.stats.num_running_reqs = running_bs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = round(num_used / self.max_total_num_tokens, 2)
1318
1319
            self.stats.num_queue_reqs = len(self.waiting_queue)
            self.stats.cache_hit_rate = cache_hit_rate
1320
1321
1322
1323
1324
1325

            total_queue_latency = 0
            for req in can_run_list:
                total_queue_latency += req.queue_time_end - req.queue_time_start
            self.stats.avg_request_queue_latency = total_queue_latency / num_new_seq

1326
            self.metrics_collector.log_stats(self.stats)
1327
            self._emit_kv_metrics()
1328
        self._publish_kv_events()
1329

1330
1331
1332
    def log_decode_stats(
        self, can_run_cuda_graph: bool, running_batch: ScheduleBatch = None
    ):
1333
1334
        batch = running_batch or self.running_batch

1335
1336
        gap_latency = time.perf_counter() - self.last_decode_stats_tic
        self.last_decode_stats_tic = time.perf_counter()
1337
1338
        self.last_gen_throughput = self.num_generated_tokens / gap_latency
        self.num_generated_tokens = 0
1339
        num_running_reqs = len(batch.reqs)
Lianmin Zheng's avatar
Lianmin Zheng committed
1340
        num_used = self.max_total_num_tokens - (
1341
1342
            self.token_to_kv_pool_allocator.available_size()
            + self.tree_cache.evictable_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1343
        )
1344
1345
1346
1347
1348

        if RECORD_STEP_TIME:
            self.step_time_dict[num_running_reqs].append(
                gap_latency / self.server_args.decode_log_interval
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1349

Liangsheng Yin's avatar
Liangsheng Yin committed
1350
1351
1352
1353
1354
1355
1356
        msg = (
            f"Decode batch. "
            f"#running-req: {num_running_reqs}, "
            f"#token: {num_used}, "
            f"token usage: {num_used / self.max_total_num_tokens:.2f}, "
        )

1357
        if self.spec_algorithm.is_none():
1358
            spec_accept_length = 0
1359
        else:
1360
            spec_accept_length = (
1361
1362
                self.spec_num_total_accepted_tokens / self.spec_num_total_forward_ct
            )
1363
1364
            self.cum_spec_accept_length += self.spec_num_total_accepted_tokens
            self.cum_spec_accept_count += self.spec_num_total_forward_ct
1365
            self.spec_num_total_accepted_tokens = self.spec_num_total_forward_ct = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
1366
1367
1368
1369
            msg += f"accept len: {spec_accept_length:.2f}, "

        if self.disaggregation_mode == DisaggregationMode.DECODE:
            msg += f"pre-allocated usage: {self.num_tokens_pre_allocated / self.max_total_num_tokens:.2f}, "
1370
            msg += f"#retracted-req: {len(self.disagg_decode_prealloc_queue.retracted_queue)}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1371
1372

        msg += (
1373
            f"cuda graph: {can_run_cuda_graph}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1374
1375
1376
            f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
            f"#queue-req: {len(self.waiting_queue)}"
        )
1377
1378

        logger.info(msg)
1379
1380
1381
1382
        if self.enable_metrics:
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = num_used / self.max_total_num_tokens
1383
1384
            self.stats.cache_hit_rate = 0.0
            self.stats.gen_throughput = self.last_gen_throughput
1385
            self.stats.num_queue_reqs = len(self.waiting_queue)
1386
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1387
            self.stats.spec_accept_length = spec_accept_length
1388
            self.metrics_collector.log_stats(self.stats)
1389
            self._emit_kv_metrics()
1390
        self._publish_kv_events()
1391

Lianmin Zheng's avatar
Lianmin Zheng committed
1392
1393
    def check_memory(self):
        available_size = (
1394
1395
            self.token_to_kv_pool_allocator.available_size()
            + self.tree_cache.evictable_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1396
        )
1397
1398
1399
1400
1401
1402
1403
        protected_size = self.tree_cache.protected_size()
        memory_leak = available_size != (
            self.max_total_num_tokens
            if not self.enable_hierarchical_cache
            else self.max_total_num_tokens - protected_size
        )
        if memory_leak:
1404
            msg = (
1405
                "token_to_kv_pool_allocator memory leak detected! "
1406
                f"{available_size=}, {protected_size=}, {self.max_total_num_tokens=}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1407
1408
                f"{self.token_to_kv_pool_allocator.available_size()=}\n"
                f"{self.tree_cache.evictable_size()=}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1409
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1410
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1411

1412
1413
1414
1415
1416
1417
1418
1419
        if self.disaggregation_mode == DisaggregationMode.DECODE:
            req_total_size = (
                self.req_to_token_pool.size + self.req_to_token_pool.pre_alloc_size
            )
        else:
            req_total_size = self.req_to_token_pool.size

        if len(self.req_to_token_pool.free_slots) != req_total_size:
1420
            msg = (
1421
                "req_to_token_pool memory leak detected!"
1422
1423
                f"available_size={len(self.req_to_token_pool.free_slots)}, "
                f"total_size={self.req_to_token_pool.size}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1424
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1425
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1426

1427
1428
1429
        if (
            self.enable_metrics
            and self.attn_tp_rank == 0
1430
            and time.perf_counter() > self.metrics_collector.last_log_time + 30
1431
1432
1433
        ):
            # During idle time, also collect metrics every 30 seconds.
            num_used = self.max_total_num_tokens - (
1434
                self.token_to_kv_pool_allocator.available_size()
1435
1436
                + self.tree_cache.evictable_size()
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1437
            num_running_reqs = len(self.running_batch.reqs)
1438
1439
1440
1441
1442
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = num_used / self.max_total_num_tokens
            self.stats.gen_throughput = 0
            self.stats.num_queue_reqs = len(self.waiting_queue)
1443
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1444
            self.metrics_collector.log_stats(self.stats)
1445
        self._publish_kv_events()
1446

1447
    def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
1448
        # Merge the prefill batch into the running batch
1449
1450
1451
1452
1453
1454
1455
1456
        chunked_req_to_exclude = set()
        if self.chunked_req:
            # Move the chunked request out of the batch so that we can merge
            # only finished requests to running_batch.
            chunked_req_to_exclude.add(self.chunked_req)
            self.tree_cache.cache_unfinished_req(self.chunked_req)
            # chunked request keeps its rid but will get a new req_pool_idx
            self.req_to_token_pool.free(self.chunked_req.req_pool_idx)
Lianmin Zheng's avatar
Lianmin Zheng committed
1457
        if self.last_batch and self.last_batch.forward_mode.is_extend():
1458
1459
1460
1461
            if self.last_batch.chunked_req is not None:
                # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
                # We need to discard it.
                chunked_req_to_exclude.add(self.last_batch.chunked_req)
Lianmin Zheng's avatar
Lianmin Zheng committed
1462

1463
            # Filter batch
1464
            last_bs = self.last_batch.batch_size()
1465
1466
1467
            self.last_batch.filter_batch(
                chunked_req_to_exclude=list(chunked_req_to_exclude)
            )
1468
            if self.last_batch.batch_size() < last_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1469
                self.running_batch.batch_is_full = False
1470

1471
            # Merge the new batch into the running batch
1472
            if not self.last_batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1473
                if self.running_batch.is_empty():
1474
1475
                    self.running_batch = self.last_batch
                else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1476
                    # Merge running_batch with prefill batch
1477
                    self.running_batch.merge_batch(self.last_batch)
1478

1479
        new_batch = self.get_new_batch_prefill()
1480

1481
1482
1483
1484
1485
        need_dp_attn_preparation = require_mlp_sync(self.server_args)

        if need_dp_attn_preparation and not self.spec_algorithm.is_none():
            # In speculative decoding, prefill batches and decode batches cannot be processed in the same DP attention group.
            # We prepare idle batches in advance to skip preparing decode batches when there are prefill batches in the group.
Cheng Wan's avatar
Cheng Wan committed
1486
            new_batch, _ = self.prepare_mlp_sync_batch(new_batch)
1487
1488
1489
            need_dp_attn_preparation = new_batch is None

        if new_batch is not None:
1490
1491
1492
1493
            # Run prefill first if possible
            ret = new_batch
        else:
            # Run decode
Lianmin Zheng's avatar
Lianmin Zheng committed
1494
            if not self.running_batch.is_empty():
1495
                self.running_batch = self.update_running_batch(self.running_batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1496
1497
1498
                ret = self.running_batch if not self.running_batch.is_empty() else None
            else:
                ret = None
1499

1500
1501
        # Handle DP attention
        if need_dp_attn_preparation:
Cheng Wan's avatar
Cheng Wan committed
1502
            ret, _ = self.prepare_mlp_sync_batch(ret)
1503
1504

        return ret
1505

1506
1507
1508
1509
1510
1511
    def get_num_allocatable_reqs(self, running_bs):
        res = global_server_args_dict["max_micro_batch_size"] - running_bs
        if self.pp_size > 1:
            res = min(res, self.req_to_token_pool.available_size())
        return res

Lianmin Zheng's avatar
Lianmin Zheng committed
1512
    def get_new_batch_prefill(self) -> Optional[ScheduleBatch]:
Lianmin Zheng's avatar
Lianmin Zheng committed
1513
        # Check if the grammar is ready in the grammar queue
1514
        if self.grammar_queue:
1515
            self.move_ready_grammar_requests()
1516

Lianmin Zheng's avatar
Lianmin Zheng committed
1517
1518
        # Handle the cases where prefill is not allowed
        if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1519
            self.running_batch.batch_is_full or len(self.waiting_queue) == 0
1520
        ) and self.chunked_req is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
1521
1522
            return None

Lianmin Zheng's avatar
Lianmin Zheng committed
1523
        running_bs = len(self.running_batch.reqs)
1524
        # Ignore the check if self.chunked_req is not None.
1525
1526
1527
1528
1529
        # In the non-PP case, when self.chunked_req is not None, num_allocatable_reqs should always be greater than 0,
        # as the space for the chunked request has just been released.
        # In PP case, a chunked req can start in one microbatch and end in another microbatch, so the max_running_requests per microbatch should not be strict.
        # Instead, we should always allow chunked request to be added, otherwise, there will be a memory leak.
        if self.get_num_allocatable_reqs(running_bs) <= 0 and not self.chunked_req:
Lianmin Zheng's avatar
Lianmin Zheng committed
1530
            self.running_batch.batch_is_full = True
1531
1532
            return None

1533
        if self.enable_hierarchical_cache:
1534
            self.tree_cache.check_hicache_events()
1535

1536
        # Get priority queue
1537
        self.policy.calc_priority(self.waiting_queue)
1538

Lianmin Zheng's avatar
Lianmin Zheng committed
1539
        # Prefill policy
1540
        adder = PrefillAdder(
1541
            self.page_size,
1542
            self.tree_cache,
1543
            self.token_to_kv_pool_allocator,
1544
1545
1546
1547
            self.running_batch,
            self.new_token_ratio,
            self.max_prefill_tokens,
            self.chunked_prefill_size,
1548
            running_bs if self.is_mixed_chunk else 0,
1549
1550
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1551
        if self.chunked_req is not None:
1552
1553
            self.chunked_req.init_next_round_input()
            self.chunked_req = adder.add_chunked_req(self.chunked_req)
1554

Lianmin Zheng's avatar
Lianmin Zheng committed
1555
        if self.lora_paths:
Lianmin Zheng's avatar
Lianmin Zheng committed
1556
1557
            lora_set = set([req.lora_path for req in self.running_batch.reqs])

1558
        # Get requests from the waiting queue to a new prefill batch
1559
1560
        for req in self.waiting_queue:
            if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1561
                self.lora_paths
1562
1563
1564
1565
1566
1567
1568
                and len(
                    lora_set
                    | set([req.lora_path for req in adder.can_run_list])
                    | set([req.lora_path])
                )
                > self.max_loras_per_batch
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1569
                self.running_batch.batch_is_full = True
1570
1571
                break

1572
            if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):
Lianmin Zheng's avatar
Lianmin Zheng committed
1573
                self.running_batch.batch_is_full = True
1574
                break
1575

Byron Hsu's avatar
Byron Hsu committed
1576
1577
1578
1579
1580
1581
1582
            if self.disaggregation_mode == DisaggregationMode.PREFILL:
                # In prefill mode, prealloc queue and transfer queue can also take memory,
                # so we need to check if the available size for the actual available size.
                if len(adder.can_run_list) >= self.req_to_token_pool.available_size():
                    self.running_batch.batch_is_full = True
                    break

1583
1584
            req.init_next_round_input(self.tree_cache)
            res = adder.add_one_req(req, has_chunked_req=(self.chunked_req is not None))
1585

1586
1587
            if res != AddReqResult.CONTINUE:
                if res == AddReqResult.NO_TOKEN:
1588
1589
                    if self.enable_hierarchical_cache:
                        # Set batch_is_full after making sure there are requests that can be served
Lianmin Zheng's avatar
Lianmin Zheng committed
1590
1591
                        self.running_batch.batch_is_full = len(
                            adder.can_run_list
1592
                        ) > 0 or (not self.running_batch.is_empty())
1593
                    else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1594
                        self.running_batch.batch_is_full = True
1595
1596
                break

Lianmin Zheng's avatar
Lianmin Zheng committed
1597
        # Update waiting queue
1598
        can_run_list: List[Req] = adder.can_run_list
Lianmin Zheng's avatar
Lianmin Zheng committed
1599
1600
        if len(can_run_list) == 0:
            return None
1601
1602
1603
1604

        if self.enable_metrics:
            # only record queue time when enable_metrics is True to avoid overhead
            for req in can_run_list:
1605
                req.queue_time_end = time.perf_counter()
1606

Lianmin Zheng's avatar
Lianmin Zheng committed
1607
1608
1609
        self.waiting_queue = [
            x for x in self.waiting_queue if x not in set(can_run_list)
        ]
1610

1611
1612
1613
        if adder.new_chunked_req is not None:
            assert self.chunked_req is None
            self.chunked_req = adder.new_chunked_req
1614

1615
1616
        if self.chunked_req:
            self.chunked_req.is_chunked += 1
Lianmin Zheng's avatar
Lianmin Zheng committed
1617

1618
        # Print stats
1619
        if self.attn_tp_rank == 0:
1620
            self.log_prefill_stats(adder, can_run_list, running_bs)
1621

Lianmin Zheng's avatar
Lianmin Zheng committed
1622
        # Create a new batch
1623
1624
1625
        new_batch = ScheduleBatch.init_new(
            can_run_list,
            self.req_to_token_pool,
1626
            self.token_to_kv_pool_allocator,
1627
            self.tree_cache,
1628
            self.model_config,
1629
            self.enable_overlap,
1630
            self.spec_algorithm,
1631
            self.server_args.enable_custom_logit_processor,
1632
            chunked_req=self.chunked_req,
1633
        )
1634
1635
        if self.enable_hierarchical_cache:
            # todo (zhiqiang): disable cuda graph execution if hicache loading triggered
1636
1637
1638
            new_batch.hicache_consumer_index = (
                self.tree_cache.ready_to_load_host_cache()
            )
1639

1640
        new_batch.prepare_for_extend()
1641

Lianmin Zheng's avatar
Lianmin Zheng committed
1642
        # Mixed-style chunked prefill
1643
1644
        if (
            self.is_mixed_chunk
Lianmin Zheng's avatar
Lianmin Zheng committed
1645
            and not self.running_batch.is_empty()
1646
1647
1648
            and not (new_batch.return_logprob or self.running_batch.return_logprob)
        ):
            # TODO (lianmin): support return_logprob + mixed chunked prefill
1649
1650
            self.running_batch.filter_batch()
            if not self.running_batch.is_empty():
1651
                self.running_batch.prepare_for_decode()
1652
1653
                new_batch.mix_with_running(self.running_batch)
                new_batch.decoding_reqs = self.running_batch.reqs
Lianmin Zheng's avatar
Lianmin Zheng committed
1654
1655
1656
            self.running_batch = ScheduleBatch(
                reqs=[], batch_is_full=self.running_batch.batch_is_full
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1657
1658
        else:
            new_batch.decoding_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1659
1660
1661

        return new_batch

Lianmin Zheng's avatar
Lianmin Zheng committed
1662
    def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
1663
        """Update the current running decoding batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1664
        initial_bs = batch.batch_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1665

1666
1667
        batch.filter_batch()
        if batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1668
1669
            batch.batch_is_full = False
            return batch
1670

Lianmin Zheng's avatar
Lianmin Zheng committed
1671
        # Check if decode out of memory
1672
        if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or (
1673
            TEST_RETRACT and batch.batch_size() > 10
1674
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1675
1676
            old_ratio = self.new_token_ratio

1677
            retracted_reqs, new_token_ratio = batch.retract_decode(self.server_args)
Lianmin Zheng's avatar
Lianmin Zheng committed
1678
            self.new_token_ratio = new_token_ratio
1679

Lianmin Zheng's avatar
Lianmin Zheng committed
1680
            logger.info(
1681
                "KV cache pool is full. Retract requests. "
Lianmin Zheng's avatar
Lianmin Zheng committed
1682
1683
1684
                f"#retracted_reqs: {len(retracted_reqs)}, "
                f"#new_token_ratio: {old_ratio:.4f} -> {self.new_token_ratio:.4f}"
            )
1685
            self._extend_requests_to_queue(retracted_reqs, is_retracted=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
1686
1687
        else:
            self.new_token_ratio = max(
1688
                self.new_token_ratio - self.new_token_ratio_decay,
Lianmin Zheng's avatar
Lianmin Zheng committed
1689
1690
1691
                self.min_new_token_ratio,
            )

Lianmin Zheng's avatar
Lianmin Zheng committed
1692
        if batch.batch_size() < initial_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1693
            batch.batch_is_full = False
Lianmin Zheng's avatar
Lianmin Zheng committed
1694
1695

        # Update batch tensors
1696
        batch.prepare_for_decode()
Lianmin Zheng's avatar
Lianmin Zheng committed
1697
        return batch
Lianmin Zheng's avatar
Lianmin Zheng committed
1698

1699
1700
1701
    def run_batch(
        self, batch: ScheduleBatch
    ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
1702
        """Run a batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1703
1704
        self.forward_ct += 1

1705
1706
        # Whether to run the profiler
        self._profile_batch_predicate(batch)
1707
1708
1709
1710
        if self.forward_sleep_time is not None:
            logger.info(f"Scheduler.run_batch sleep {self.forward_sleep_time}s")
            time.sleep(self.forward_sleep_time)

1711
        # Run forward
1712
        if self.is_generation:
1713
1714
            if self.spec_algorithm.is_none():
                model_worker_batch = batch.get_model_worker_batch()
1715
1716
1717
1718
1719

                # update the consumer index of hicache to the running batch
                self.tp_worker.set_hicache_consumer(
                    model_worker_batch.hicache_consumer_index
                )
1720
                if self.pp_group.is_last_rank:
1721
                    logits_output, next_token_ids, can_run_cuda_graph = (
1722
1723
1724
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
                else:
1725
                    pp_hidden_states_proxy_tensors, _, can_run_cuda_graph = (
1726
1727
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
1728
                bid = model_worker_batch.bid
Lianmin Zheng's avatar
Lianmin Zheng committed
1729
            else:
1730
1731
1732
                (
                    logits_output,
                    next_token_ids,
1733
                    bid,
1734
                    num_accepted_tokens,
1735
                    can_run_cuda_graph,
1736
                ) = self.draft_worker.forward_batch_speculative_generation(batch)
1737
1738
1739
                bs = batch.batch_size()
                self.spec_num_total_accepted_tokens += num_accepted_tokens + bs
                self.spec_num_total_forward_ct += bs
1740
                self.num_generated_tokens += num_accepted_tokens
1741
1742
1743

            if self.pp_group.is_last_rank:
                batch.output_ids = next_token_ids
1744

1745
1746
1747
            # These 2 values are needed for processing the output, but the values can be
            # modified by overlap schedule. So we have to copy them here so that
            # we can use the correct values in output processing.
1748
            if batch.return_logprob or self.spec_algorithm.is_eagle():
1749
                extend_input_len_per_req = [req.extend_input_len for req in batch.reqs]
1750
1751
1752
            else:
                extend_input_len_per_req = None
            if batch.return_logprob:
1753
1754
1755
1756
1757
1758
                extend_logprob_start_len_per_req = [
                    req.extend_logprob_start_len for req in batch.reqs
                ]
            else:
                extend_logprob_start_len_per_req = None

1759
            ret = GenerationBatchResult(
1760
1761
1762
1763
1764
1765
1766
                logits_output=logits_output if self.pp_group.is_last_rank else None,
                pp_hidden_states_proxy_tensors=(
                    pp_hidden_states_proxy_tensors
                    if not self.pp_group.is_last_rank
                    else None
                ),
                next_token_ids=next_token_ids if self.pp_group.is_last_rank else None,
1767
1768
                extend_input_len_per_req=extend_input_len_per_req,
                extend_logprob_start_len_per_req=extend_logprob_start_len_per_req,
1769
                bid=bid,
1770
                can_run_cuda_graph=can_run_cuda_graph,
1771
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1772
1773
1774
        else:  # embedding or reward model
            model_worker_batch = batch.get_model_worker_batch()
            embeddings = self.tp_worker.forward_batch_embedding(model_worker_batch)
1775
1776
1777
            ret = EmbeddingBatchResult(
                embeddings=embeddings, bid=model_worker_batch.bid
            )
1778
        return ret
Chayenne's avatar
Chayenne committed
1779

1780
1781
1782
1783
    def process_batch_result(
        self,
        batch: ScheduleBatch,
        result: Union[GenerationBatchResult, EmbeddingBatchResult],
1784
        launch_done: Optional[threading.Event] = None,
1785
    ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1786
        if batch.forward_mode.is_decode():
1787
            self.process_batch_result_decode(batch, result, launch_done)
1788
        elif batch.forward_mode.is_extend():
1789
            self.process_batch_result_prefill(batch, result, launch_done)
1790
1791
        elif batch.forward_mode.is_idle():
            if self.enable_overlap:
1792
                self.tp_worker.resolve_last_batch_result(launch_done)
1793
                self.set_next_batch_sampling_info_done(batch)
1794
        elif batch.forward_mode.is_dummy_first():
1795
            self.set_next_batch_sampling_info_done(batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1796

1797
1798
1799
1800
1801
1802
1803
        if self.return_health_check_ct:
            # Return some signal for the health check.
            # This is used to prevent the health check signal being blocked by long context prefill.
            # However, one minor issue is that this code path does not check the status of detokenizer manager.
            self.return_health_check_ct -= 1
            self.send_to_tokenizer.send_pyobj(HealthCheckOutput())

1804
1805
    def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
        return self.prepare_mlp_sync_batch_raw(
1806
1807
1808
1809
1810
1811
1812
1813
            local_batch,
            dp_size=self.server_args.dp_size,
            attn_tp_size=self.attn_tp_size,
            tp_cpu_group=self.tp_cpu_group,
            get_idle_batch=self.get_idle_batch,
            disable_cuda_graph=self.server_args.disable_cuda_graph,
            spec_algorithm=self.spec_algorithm,
            speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens,
1814
1815
1816
            enable_two_batch_overlap=self.server_args.enable_two_batch_overlap,
            enable_deepep_moe=self.server_args.enable_deepep_moe,
            deepep_mode=DeepEPMode[self.server_args.deepep_mode],
1817
            require_mlp_tp_gather=require_mlp_tp_gather(self.server_args),
1818
1819
1820
        )

    @staticmethod
1821
    def prepare_mlp_sync_batch_raw(
1822
1823
1824
1825
1826
1827
1828
1829
        local_batch: ScheduleBatch,
        dp_size,
        attn_tp_size: int,
        tp_cpu_group,
        get_idle_batch,
        disable_cuda_graph: bool,
        spec_algorithm,
        speculative_num_draft_tokens,
1830
1831
1832
        enable_two_batch_overlap: bool,
        enable_deepep_moe: bool,
        deepep_mode: DeepEPMode,
1833
        require_mlp_tp_gather: bool,
1834
    ):
1835
1836
1837
        # Check if other DP workers have running batches
        if local_batch is None:
            num_tokens = 0
1838
            num_tokens_for_logprob = 0
1839
1840
        elif local_batch.forward_mode.is_decode():
            num_tokens = local_batch.batch_size()
1841
            num_tokens_for_logprob = num_tokens
1842
1843
        else:
            num_tokens = local_batch.extend_num_tokens
1844
            num_tokens_for_logprob = sum(
Lianmin Zheng's avatar
Lianmin Zheng committed
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
                [
                    # We should have at least 1 token for sample in every case.
                    max(extend_len - logprob_start_len, 1)
                    for logprob_start_len, extend_len in zip(
                        local_batch.extend_logprob_start_lens, local_batch.extend_lens
                    )
                ]
            )

        if local_batch is None or local_batch.forward_mode.is_decode_or_idle():
            can_cuda_graph = 1
        else:
            can_cuda_graph = 0

        is_extend_in_batch = (
            local_batch.forward_mode.is_extend() if local_batch else False
        )
1862
1863
1864

        tbo_preparer = TboDPAttentionPreparer()

Lianmin Zheng's avatar
Lianmin Zheng committed
1865
1866
1867
1868
        local_info = torch.tensor(
            [
                num_tokens,
                can_cuda_graph,
1869
                num_tokens_for_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1870
                is_extend_in_batch,
1871
1872
1873
1874
1875
1876
                *tbo_preparer.prepare_all_gather(
                    local_batch,
                    deepep_mode,
                    enable_deepep_moe,
                    enable_two_batch_overlap,
                ),
Lianmin Zheng's avatar
Lianmin Zheng committed
1877
1878
1879
1880
            ],
            dtype=torch.int64,
        )
        global_info = torch.empty(
1881
            (dp_size, attn_tp_size, 6),
Lianmin Zheng's avatar
Lianmin Zheng committed
1882
1883
            dtype=torch.int64,
        )
1884
        torch.distributed.all_gather_into_tensor(
Lianmin Zheng's avatar
Lianmin Zheng committed
1885
1886
            global_info.flatten(),
            local_info,
1887
            group=tp_cpu_group,
1888
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1889
1890
1891
1892
        global_num_tokens = global_info[:, 0, 0].tolist()
        can_cuda_graph = min(global_info[:, 0, 1].tolist())
        global_num_tokens_for_logprob = global_info[:, 0, 2].tolist()
        is_extend_in_batch = global_info[:, 0, 3].tolist()
1893

1894
1895
1896
1897
        tbo_split_seq_index, global_forward_mode = tbo_preparer.compute_output(
            global_info[:, :, 4:6]
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1898
        if local_batch is None and max(global_num_tokens) > 0:
1899
            local_batch = get_idle_batch()
1900
1901

        if local_batch is not None:
1902
            # TODO: handle the case when moe_dense_tp_size != 1
1903
            if not require_mlp_tp_gather:
1904
1905
1906
1907
1908
1909
1910
                local_batch.global_num_tokens = [num_tokens]
                local_batch.global_num_tokens_for_logprob = [num_tokens_for_logprob]
            else:
                local_batch.global_num_tokens = global_num_tokens
                local_batch.global_num_tokens_for_logprob = (
                    global_num_tokens_for_logprob
                )
1911
            local_batch.is_extend_in_batch = any(is_extend_in_batch)
1912
1913
            local_batch.tbo_split_seq_index = tbo_split_seq_index
            local_batch.global_forward_mode = global_forward_mode
1914

1915
            # Check forward mode for cuda graph
1916
            if not disable_cuda_graph:
Lianmin Zheng's avatar
Lianmin Zheng committed
1917
                local_batch.can_run_dp_cuda_graph = can_cuda_graph
1918

1919
        # TODO(ch-wan): refactor: any(is_extend_in_batch) now is a part of local_batch. Remove it from here.
Lianmin Zheng's avatar
Lianmin Zheng committed
1920
        return local_batch, any(is_extend_in_batch)
1921
1922
1923
1924
1925

    def get_idle_batch(self):
        idle_batch = ScheduleBatch.init_new(
            [],
            self.req_to_token_pool,
1926
            self.token_to_kv_pool_allocator,
1927
1928
1929
            self.tree_cache,
            self.model_config,
            self.enable_overlap,
1930
            self.spec_algorithm,
1931
            self.server_args.enable_custom_logit_processor,
1932
1933
1934
1935
        )
        idle_batch.prepare_for_idle()
        return idle_batch

1936
1937
    def move_ready_grammar_requests(self):
        """Move requests whose grammar objects are ready from grammar_queue to waiting_queue."""
1938

1939
        num_ready_reqs = 0
1940
        num_timeout_reqs = 0
1941
1942
        for req in self.grammar_queue:
            try:
1943
1944
1945
                if req.finished():  # It is aborted by AbortReq
                    num_ready_reqs += 1
                    continue
1946
                req.grammar = req.grammar.result(timeout=0.03)
1947
1948
1949
1950
1951
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
1952
1953
                num_ready_reqs += 1
            except futures._base.TimeoutError:
1954
                req.grammar_wait_ct += 1
1955
1956
                # NOTE(lianmin): this timeout is the waiting time of the above line. It is
                # not the waiting time from it enters the grammar queue.
1957
                if req.grammar_wait_ct > GRAMMAR_TIMEOUT / 0.03:
1958
                    num_timeout_reqs = 1
1959
1960
                break

1961
        if self.server_args.enable_dp_attention:
1962
1963
            tp_size = self.attn_tp_size
            tp_group = self.attn_tp_cpu_group
1964
        else:
1965
1966
1967
1968
1969
            tp_size = self.tp_size
            tp_group = self.tp_cpu_group

        if tp_size > 1:
            # Sync across TP ranks to make sure they have the same number of ready requests
1970
            tensor = torch.tensor([num_ready_reqs, num_timeout_reqs], dtype=torch.int32)
1971
1972
1973
            torch.distributed.all_reduce(
                tensor, op=torch.distributed.ReduceOp.MAX, group=tp_group
            )
1974
            num_ready_reqs_max, num_timeout_reqs_max = tensor.tolist()
1975

1976
            for i in range(num_ready_reqs, num_ready_reqs_max):
1977
                req = self.grammar_queue[i]
1978
1979
                if req.finished():  # It is aborted by AbortReq
                    continue
1980
                req.grammar = req.grammar.result()
1981
1982
1983
1984
1985
1986
1987
1988
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
        else:
            num_ready_reqs_max = num_ready_reqs
            num_timeout_reqs_max = num_timeout_reqs
1989

1990
1991
1992
1993
1994
1995
1996
        for i in range(num_ready_reqs, num_ready_reqs + num_timeout_reqs_max):
            req = self.grammar_queue[i]
            req.grammar.cancel()
            error_msg = f"Grammar preprocessing timed out for {req.grammar_key=}"
            req.set_finish_with_abort(error_msg)
            self.grammar_backend.set_cache(req.grammar_key, INVALID_GRAMMAR_OBJ)
        num_ready_reqs = num_ready_reqs_max + num_timeout_reqs_max
1997

1998
        self._extend_requests_to_queue(self.grammar_queue[:num_ready_reqs])
1999
2000
        self.grammar_queue = self.grammar_queue[num_ready_reqs:]

2001
2002
2003
2004
2005
2006
2007
    def set_next_batch_sampling_info_done(self, batch: ScheduleBatch):
        if batch.next_batch_sampling_info:
            if batch.next_batch_sampling_info.grammars is not None:
                batch.next_batch_sampling_info.update_regex_vocab_mask()
                self.current_stream.synchronize()
            batch.next_batch_sampling_info.sampling_info_done.set()

2008
2009
2010
    def watchdog_thread(self):
        """A watch dog thread that will try to kill the server itself if one forward batch takes too long."""
        self.watchdog_last_forward_ct = 0
2011
        self.watchdog_last_time = time.perf_counter()
2012
2013

        while True:
2014
            current = time.perf_counter()
2015
2016
2017
2018
2019
2020
2021
2022
2023
            if self.cur_batch is not None:
                if self.watchdog_last_forward_ct == self.forward_ct:
                    if current > self.watchdog_last_time + self.watchdog_timeout:
                        break
                else:
                    self.watchdog_last_forward_ct = self.forward_ct
                    self.watchdog_last_time = current
            time.sleep(self.watchdog_timeout // 2)

Lianmin Zheng's avatar
Lianmin Zheng committed
2024
2025
2026
2027
2028
2029
2030
2031
2032
        if not disable_request_logging():
            # Print batch size and memory pool info to check whether there are de-sync issues.
            logger.error(
                f"{self.cur_batch.batch_size()=}, "
                f"{self.cur_batch.reqs=}, "
                f"{self.token_to_kv_pool_allocator.available_size()=}, "
                f"{self.tree_cache.evictable_size()=}, "
            )

2033
        pyspy_dump_schedulers()
Lianmin Zheng's avatar
Lianmin Zheng committed
2034
        logger.error(f"Watchdog timeout ({self.watchdog_timeout=})")
2035
2036
        print(file=sys.stderr, flush=True)
        print(file=sys.stdout, flush=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
2037
2038

        # Wait for some time so that the parent process can print the error.
2039
2040
2041
        time.sleep(5)
        self.parent_process.send_signal(signal.SIGQUIT)

2042
2043
2044
    def flush_cache_wrapped(self, recv_req: FlushCacheReqInput):
        success = self.flush_cache()
        return FlushCacheReqOutput(success=success)
2045

2046
    def flush_cache(self):
2047
        """Flush the memory pool and cache."""
2048
2049
2050
2051
2052
        if (
            len(self.waiting_queue) == 0
            and self.running_batch.is_empty()
            and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
        ):
2053
2054
            self.cur_batch = None
            self.last_batch = None
2055
            self.tree_cache.reset()
2056
            if self.grammar_backend:
Lianmin Zheng's avatar
Lianmin Zheng committed
2057
                self.grammar_backend.reset()
2058
            self.req_to_token_pool.clear()
2059
            self.token_to_kv_pool_allocator.clear()
2060
2061
2062

            if not self.spec_algorithm.is_none():
                self.draft_worker.model_runner.req_to_token_pool.clear()
2063
                self.draft_worker.model_runner.token_to_kv_pool_allocator.clear()
2064
2065
2066
2067
2068

            self.num_generated_tokens = 0
            self.forward_ct_decode = 0
            self.spec_num_total_accepted_tokens = 0
            self.spec_num_total_forward_ct = 0
2069
2070
            self.cum_spec_accept_length = 0
            self.cum_spec_accept_count = 0
2071
2072
2073
2074
2075
2076
2077
            torch.cuda.empty_cache()
            logger.info("Cache flushed successfully!")
            if_success = True
        else:
            logging.warning(
                f"Cache not flushed because there are pending requests. "
                f"#queue-req: {len(self.waiting_queue)}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2078
                f"#running-req: {len(self.running_batch.reqs)}"
2079
2080
2081
2082
            )
            if_success = False
        return if_success

Liangsheng Yin's avatar
Liangsheng Yin committed
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
    def get_load(self):
        # TODO(lsyin): use dynamically maintained num_waiting_tokens
        load = (
            self.max_total_num_tokens
            - self.token_to_kv_pool_allocator.available_size()
            - self.tree_cache.evictable_size()
        )
        load += sum(len(req.origin_input_ids) for req in self.waiting_queue)
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            load += sum(
                len(req.origin_input_ids)
                for req in self.disagg_prefill_bootstrap_queue.queue
            )
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            load += sum(
                len(req.req.origin_input_ids)
                for req in self.disagg_decode_prealloc_queue.queue
            )

        return load

2104
2105
2106
2107
2108
2109
2110
2111
2112
    def get_internal_state(self, recv_req: GetInternalStateReq):
        ret = dict(global_server_args_dict)
        ret["last_gen_throughput"] = self.last_gen_throughput
        if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
            ret["avg_spec_accept_length"] = (
                self.cum_spec_accept_length / self.cum_spec_accept_count
            )
        if RECORD_STEP_TIME:
            ret["step_time_dict"] = self.step_time_dict
Liangsheng Yin's avatar
Liangsheng Yin committed
2113
2114
2115
2116

        ret["load"] = self.get_load()

        return GetInternalStateReqOutput(internal_state=ret)
2117
2118
2119
2120
2121

    def set_internal_state(self, recv_req: SetInternalStateReq):
        server_args_dict = recv_req.server_args
        args_allow_update = set(
            [
2122
                "max_micro_batch_size",
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
                "speculative_accept_threshold_single",
                "speculative_accept_threshold_acc",
            ]
        )
        if_success = True
        for k, v in server_args_dict.items():
            if k not in args_allow_update:
                logging.warning(f"Updating {k} is not supported.")
                if_success = False
                break
2133
2134
2135
2136
2137
2138
2139
2140
            elif k == "max_micro_batch_size" and (
                v > self.max_running_requests // self.pp_size or v < 1
            ):
                logging.warning(
                    f"Updating {k} to {v} is rejected because it is out of the valid range [1, {self.max_running_requests // self.pp_size}]."
                )
                if_success = False
                break
2141
2142
2143
2144
2145
2146
2147
2148
2149
        if if_success:
            if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
                avg_spec_accept_length = (
                    self.cum_spec_accept_length / self.cum_spec_accept_count
                )
                logger.info(f"{avg_spec_accept_length=}")
            self.cum_spec_accept_length = self.cum_spec_accept_count = 0
            for k, v in server_args_dict.items():
                global_server_args_dict[k] = v
2150
            logger.info(f"Global server args updated! {global_server_args_dict=}")
2151
2152
2153
2154
2155
        return SetInternalStateReqOutput(
            updated=True,
            server_args=global_server_args_dict,
        )

2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
    def handle_rpc_request(self, recv_req: RpcReqInput):
        # Handle RPC requests
        logger.info(
            f"handle_rpc_request: {recv_req.method}, param: {recv_req.parameters}"
        )

        success = True
        exec = None
        try:
            func = getattr(self, recv_req.method)
            func(recv_req.parameters)
        except Exception as e:
            success = False
            exec = e
            logger.error(f"Failed to call rpc {recv_req.method}: {str(e)}")

        barrier()
        return RpcReqOutput(success, "" if not exec else str(exec))

    def save_remote_model(self, params):
        url = params["url"]

2178
        worker = self.tp_worker.worker
2179
2180
2181
2182

        worker.model_runner.save_remote_model(url)

    def save_sharded_model(self, params):
2183
        worker = self.tp_worker.worker
2184
2185
2186
2187
2188
2189
2190

        worker.model_runner.save_sharded_model(
            path=params["path"],
            pattern=params["pattern"],
            max_size=params["max_size"],
        )

2191
2192
    def abort_request(self, recv_req: AbortReq):
        # Delete requests in the waiting queue
Lianmin Zheng's avatar
Lianmin Zheng committed
2193
        to_del = []
2194
        for i, req in enumerate(self.waiting_queue):
Lianmin Zheng's avatar
Lianmin Zheng committed
2195
2196
            if req.rid.startswith(recv_req.rid):
                to_del.append(i)
2197

Lianmin Zheng's avatar
Lianmin Zheng committed
2198
        # Sort in reverse order to avoid index issues when deleting
Lianmin Zheng's avatar
Lianmin Zheng committed
2199
        for i in reversed(to_del):
2200
2201
2202
            # Abort method 1: directly pop from the queue
            # This only works for requests that have not started anything.
            # We still need to send something back to TokenizerManager to clean up the state.
Lianmin Zheng's avatar
Lianmin Zheng committed
2203
            req = self.waiting_queue.pop(i)
Lianmin Zheng's avatar
Lianmin Zheng committed
2204
            self.send_to_tokenizer.send_pyobj(AbortReq(req.rid))
2205
            logger.debug(f"Abort queued request. {req.rid=}")
2206

2207
2208
2209
2210
2211
2212
2213
        # Delete the requests in the grammar queue
        for req in self.grammar_queue:
            # Abort method 2: call `set_finish_with_abort`
            # The request will still run one prefill forward pass.
            # In this case, we change the input_ids to be only one token to make this prefill cheap.
            if req.rid.startswith(recv_req.rid):
                logger.debug(f"Abort grammar queue request. {req.rid=}")
2214
2215
                if req.grammar:
                    req.grammar.cancel()
2216
2217
                req.set_finish_with_abort("Aborted by AbortReq.")

2218
        # Delete requests in the running batch
Lianmin Zheng's avatar
Lianmin Zheng committed
2219
2220
2221
2222
2223
2224
        if self.cur_batch is self.running_batch or self.cur_batch is None:
            reqs = self.running_batch.reqs
        else:
            reqs = self.running_batch.reqs + self.cur_batch.reqs

        for req in reqs:
Lianmin Zheng's avatar
Lianmin Zheng committed
2225
            if req.rid.startswith(recv_req.rid) and not req.finished():
2226
2227
2228
                # Abort method 3: set `to_abort=True`
                # The request will still run one decode forward pass.
                # Then we reuse all existing code to clean up the KV cache allocation.
Lianmin Zheng's avatar
Lianmin Zheng committed
2229
2230
                logger.debug(f"Abort running request. {req.rid=}")
                req.to_abort = True
2231

2232
2233
2234
    def _pause_engine(self) -> Tuple[List[Req], int]:
        raise NotImplementedError()

Chayenne's avatar
Chayenne committed
2235
2236
2237
    def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
        """In-place update of the weights from disk."""
        success, message = self.tp_worker.update_weights_from_disk(recv_req)
2238
2239
2240
2241
2242
        if success:
            flash_cache_success = self.flush_cache()
            assert flash_cache_success, "Cache flush failed after updating weights"
        else:
            logger.error(message)
2243
        return UpdateWeightFromDiskReqOutput(success, message, 0)
2244

2245
2246
2247
    def init_weights_update_group(self, recv_req: InitWeightsUpdateGroupReqInput):
        """Initialize the online model parameter update group."""
        success, message = self.tp_worker.init_weights_update_group(recv_req)
2248
        return InitWeightsUpdateGroupReqOutput(success, message)
2249
2250

    def update_weights_from_distributed(
2251
2252
2253
        self,
        recv_req: UpdateWeightsFromDistributedReqInput,
    ) -> Tuple[bool, str]:
2254
2255
2256
2257
2258
2259
2260
        """Update the online model parameter."""
        success, message = self.tp_worker.update_weights_from_distributed(recv_req)
        if success:
            flash_cache_success = self.flush_cache()
            assert flash_cache_success, "Cache flush failed after updating weights"
        else:
            logger.error(message)
2261
        return UpdateWeightsFromDistributedReqOutput(success, message)
2262

2263
2264
2265
2266
2267
    def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
        """Update the online model parameter from tensors."""
        success, message = self.tp_worker.update_weights_from_tensor(recv_req)
        # TODO extract common code b/t update_weights_from_distributed and update_weights_from_tensor later
        if success:
2268
2269
2270
            if recv_req.flush_cache:
                flash_cache_success = self.flush_cache()
                assert flash_cache_success, "Cache flush failed after updating weights"
2271
2272
        else:
            logger.error(message)
2273
        barrier(group=self.tp_cpu_group)
2274
        return UpdateWeightsFromTensorReqOutput(success, message)
2275

2276
2277
    def get_weights_by_name(self, recv_req: GetWeightsByNameReqInput):
        parameter = self.tp_worker.get_weights_by_name(recv_req)
2278
        return GetWeightsByNameReqOutput(parameter)
2279

2280
    def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
        tags = recv_req.tags
        import subprocess

        if tags is None:
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_KV_CACHE)
            self.flush_cache()

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.stashed_model_static_state = _export_static_state(
                self.tp_worker.worker.model_runner.model
            )
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_WEIGHTS)

2297
        return ReleaseMemoryOccupationReqOutput()
2298

2299
    def resume_memory_occupation(self, recv_req: ResumeMemoryOccupationReqInput):
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
        tags = recv_req.tags
        if tags is None or len(tags) == 0:
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_WEIGHTS)
            _import_static_state(
                self.tp_worker.worker.model_runner.model,
                self.stashed_model_static_state,
            )
            del self.stashed_model_static_state

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_KV_CACHE)

2315
2316
        return ResumeMemoryOccupationReqOutput()

2317
2318
2319
2320
2321
2322
2323
    def slow_down(self, recv_req: SlowDownReqInput):
        t = recv_req.forward_sleep_time
        if t is not None and t <= 0:
            t = None
        self.forward_sleep_time = t
        return SlowDownReqOutput()

2324
    def profile(self, recv_req: ProfileReq):
2325
        if recv_req.type == ProfileReqType.START_PROFILE:
2326
2327
2328
2329
2330
2331
2332
2333
            if recv_req.profile_by_stage:
                return self.init_profile(
                    recv_req.output_dir,
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2334
                    recv_req.profile_id,
2335
2336
2337
2338
2339
2340
2341
2342
2343
                )
            else:
                self.init_profile(
                    recv_req.output_dir,
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2344
                    recv_req.profile_id,
2345
2346
                )
                return self.start_profile(True)
2347
        else:
2348
2349
            return self.stop_profile()

2350
    def init_profile(
2351
2352
2353
2354
        self,
        output_dir: Optional[str],
        num_steps: Optional[int],
        activities: Optional[List[str]],
2355
2356
        with_stack: Optional[bool],
        record_shapes: Optional[bool],
2357
        profile_by_stage: bool,
2358
        profile_id: str,
2359
2360
    ) -> ProfileReqOutput:
        if self.profile_in_progress:
2361
2362
2363
2364
2365
            return ProfileReqOutput(
                success=False,
                message="Profiling is already in progress. Call /stop_profile first.",
            )

2366
2367
        self.profile_by_stage = profile_by_stage

2368
2369
2370
2371
2372
2373
        if output_dir is None:
            output_dir = os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp")
        if activities is None:
            activities = ["CPU", "GPU"]

        self.torch_profiler_output_dir = output_dir
2374
2375
        self.torch_profiler_with_stack = with_stack
        self.torch_profiler_record_shapes = record_shapes
2376
        self.profiler_activities = activities
2377
        self.profile_id = profile_id
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397

        if num_steps:
            self.profile_steps = num_steps
            if self.profile_by_stage:
                self.profiler_target_prefill_ct = num_steps
                self.profiler_target_decode_ct = num_steps
                self.profiler_prefill_ct = 0
                self.profiler_decode_ct = 0
            else:
                self.profiler_target_forward_ct = self.forward_ct + num_steps
            # The caller will be notified when reaching profiler_target_forward_ct
        else:
            self.profiler_target_forward_ct = None

        return ProfileReqOutput(success=True, message="Succeeded")

    def start_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        stage_str = f" for {stage.__str__()}" if stage else ""
2398
        logger.info(
2399
            f"Profiling starts{stage_str}. Traces will be saved to: {self.torch_profiler_output_dir} (with profile id: {self.profile_id})",
2400
2401
        )

2402
2403
2404
2405
        activities = self.profiler_activities
        with_stack = self.torch_profiler_with_stack
        record_shapes = self.torch_profiler_record_shapes

2406
2407
2408
2409
2410
2411
2412
2413
        activity_map = {
            "CPU": torch.profiler.ProfilerActivity.CPU,
            "GPU": torch.profiler.ProfilerActivity.CUDA,
        }
        torchprof_activities = [
            activity_map[a] for a in activities if a in activity_map
        ]

2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
        if "RPD" in activities:
            from rpdTracerControl import rpdTracerControl

            rpdTracerControl.skipCreate()

            self.rpd_profile_path = os.path.join(
                self.torch_profiler_output_dir,
                "rpd-" + str(time.time()) + f"-TP-{self.tp_rank}" + ".trace.json.gz",
            )

            if self.tp_rank == 0:
                import sqlite3

                from rocpd.schema import RocpdSchema

                if os.path.exists("trace.rpd"):
                    os.unlink("trace.rpd")
                schema = RocpdSchema()
                connection = sqlite3.connect("trace.rpd")
                schema.writeSchema(connection)
                connection.commit()
                del connection
            torch.distributed.barrier(self.tp_cpu_group)

            self.rpd_profiler = rpdTracerControl()
            self.rpd_profiler.setPythonTrace(True)
            self.rpd_profiler.start()
            self.rpd_profiler.rangePush("", "rpd profile range", "")
            self.profile_in_progress = True
        elif torchprof_activities:
2444
2445
            self.torch_profiler = torch.profiler.profile(
                activities=torchprof_activities,
2446
2447
                with_stack=with_stack if with_stack is not None else True,
                record_shapes=record_shapes if record_shapes is not None else False,
2448
2449
            )
            self.torch_profiler.start()
2450
            self.profile_in_progress = True
2451
2452
2453

        if "MEM" in activities:
            torch.cuda.memory._record_memory_history(max_entries=100000)
2454
            self.profile_in_progress = True
2455

2456
2457
2458
        if "CUDA_PROFILER" in activities:
            torch.cuda.cudart().cudaProfilerStart()

2459
        return ProfileReqOutput(success=True, message="Succeeded")
2460

2461
2462
2463
2464
    def stop_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        if not self.profile_in_progress:
2465
2466
2467
2468
            return ProfileReqOutput(
                success=False,
                message="Profiling is not in progress. Call /start_profile first.",
            )
2469

2470
2471
2472
        if not Path(self.torch_profiler_output_dir).exists():
            Path(self.torch_profiler_output_dir).mkdir(parents=True, exist_ok=True)

2473
2474
        stage_suffix = f"-{stage.__str__()}" if stage else ""
        logger.info("Stop profiling" + stage_suffix + "...")
2475
2476
2477
2478
2479
        if self.torch_profiler is not None:
            self.torch_profiler.stop()
            self.torch_profiler.export_chrome_trace(
                os.path.join(
                    self.torch_profiler_output_dir,
2480
                    self.profile_id
2481
2482
2483
                    + f"-TP-{self.tp_rank}"
                    + stage_suffix
                    + ".trace.json.gz",
2484
2485
                )
            )
2486
2487
2488
2489
2490
2491
            torch.distributed.barrier(self.tp_cpu_group)

        if self.rpd_profiler is not None:
            self.rpd_profiler.rangePop()
            self.rpd_profiler.stop()
            self.rpd_profiler.flush()
2492

2493
2494
2495
2496
2497
2498
2499
2500
2501
            torch.distributed.barrier(self.tp_cpu_group)
            if self.tp_rank == 0:
                from sglang.srt.utils import rpd_to_chrome_trace

                rpd_to_chrome_trace("trace.rpd", self.rpd_profile_path)
            self.rpd_profiler = None
            self.rpd_profiler_path = None

        if self.profiler_activities is not None and "MEM" in self.profiler_activities:
2502
            memory_profile_path = os.path.join(
2503
                self.torch_profiler_output_dir,
2504
2505
2506
2507
                str(time.time())
                + f"-TP-{self.tp_rank}-memory"
                + stage_suffix
                + ".pickle",
2508
2509
2510
2511
            )
            torch.cuda.memory._dump_snapshot(memory_profile_path)
            torch.cuda.memory._record_memory_history(enabled=None)

2512
2513
2514
        if "CUDA_PROFILER" in self.profiler_activities:
            torch.cuda.cudart().cudaProfilerStop()

2515
2516
2517
        logger.info(
            "Profiling done. Traces are saved to: %s",
            self.torch_profiler_output_dir,
2518
        )
2519
        self.torch_profiler = None
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
        self.profile_in_progress = False

        return ProfileReqOutput(success=True, message="Succeeded.")

    def _profile_batch_predicate(self, batch):
        if self.profile_by_stage:
            if batch.forward_mode.is_prefill():
                if self.profiler_prefill_ct == 0:
                    self.start_profile(batch.forward_mode)
                self.profiler_prefill_ct += 1
                if self.profiler_prefill_ct > self.profiler_target_prefill_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.EXTEND)
            elif batch.forward_mode.is_decode():
                if self.profiler_decode_ct == 0:
                    if self.profile_in_progress:
                        # force trace flush
                        self.stop_profile(ForwardMode.EXTEND)
                    self.start_profile(batch.forward_mode)
                self.profiler_decode_ct += 1
                if self.profiler_decode_ct > self.profiler_target_decode_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.DECODE)
2543
2544
            elif batch.forward_mode.is_idle():
                pass
2545
            else:
2546
                raise RuntimeError(f"unsupported profile stage: {batch.forward_mode}")
2547
2548
2549
2550
2551
2552
2553
        else:
            # Check profiler
            if (
                self.profiler_target_forward_ct
                and self.profiler_target_forward_ct <= self.forward_ct
            ):
                self.stop_profile()
2554

2555
2556
    def expert_distribution_handle(self, recv_req: ExpertDistributionReq):
        if recv_req == ExpertDistributionReq.START_RECORD:
2557
            get_global_expert_distribution_recorder().start_record()
2558
        elif recv_req == ExpertDistributionReq.STOP_RECORD:
2559
            get_global_expert_distribution_recorder().stop_record()
2560
        elif recv_req == ExpertDistributionReq.DUMP_RECORD:
2561
            get_global_expert_distribution_recorder().dump_record()
2562
2563
        else:
            raise ValueError("Unrecognized ExpertDistributionReq value")
2564
        return ExpertDistributionReqOutput()
2565

2566
    def open_session(self, recv_req: OpenSessionReqInput):
2567
2568
2569
2570
        # handle error
        session_id = recv_req.session_id
        if session_id in self.sessions:
            logger.warning(f"session id {session_id} already exist, cannot open.")
2571
            return OpenSessionReqOutput(session_id, False)
2572
        elif session_id is None:
2573
            logger.warning("session id is None, cannot open.")
2574
            return OpenSessionReqOutput(session_id, False)
2575
2576
2577
2578
        else:
            self.sessions[session_id] = Session(
                recv_req.capacity_of_str_len, session_id
            )
2579
            return OpenSessionReqOutput(session_id, True)
2580
2581
2582
2583
2584
2585
2586
2587
2588

    def close_session(self, recv_req: CloseSessionReqInput):
        # handle error
        session_id = recv_req.session_id
        if session_id not in self.sessions:
            logger.warning(f"session id {session_id} does not exist, cannot delete.")
        else:
            del self.sessions[session_id]

2589
2590
    def get_print_prefix(self):
        prefix = ""
2591
2592
        if self.attn_dp_rank is not None:
            prefix += f" DP{self.attn_dp_rank}"
2593
2594
2595
2596
2597
2598
        if self.server_args.tp_size > 1:
            prefix += f" TP{self.tp_rank}"
        if self.pp_size > 1:
            prefix += f" PP{self.pp_rank}"
        return prefix

2599
2600
2601
2602
2603
2604
2605
    def _publish_kv_events(self):
        if self.enable_kv_cache_events:
            events = self.tree_cache.take_events()
            if events:
                batch = KVEventBatch(ts=time.time(), events=events)
                self.kv_event_publisher.publish(batch)

2606

2607
2608
2609
2610
def is_health_check_generate_req(recv_req):
    return getattr(recv_req, "rid", "").startswith("HEALTH_CHECK")


2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
def _export_static_state(model):
    return dict(
        buffers=[
            (name, buffer.detach().clone()) for name, buffer in model.named_buffers()
        ]
    )


def _import_static_state(model, static_params):
    self_named_buffers = dict(model.named_buffers())
    for name, tensor in static_params["buffers"]:
        self_named_buffers[name][...] = tensor


2625
2626
2627
2628
2629
def run_scheduler_process(
    server_args: ServerArgs,
    port_args: PortArgs,
    gpu_id: int,
    tp_rank: int,
2630
    pp_rank: int,
2631
    dp_rank: Optional[int],
2632
    pipe_writer,
2633
):
2634
    # Generate the prefix
2635
2636
2637
2638
2639
2640
2641
    prefix = ""
    if dp_rank is not None:
        prefix += f" DP{dp_rank}"
    if server_args.tp_size > 1:
        prefix += f" TP{tp_rank}"
    if server_args.pp_size > 1:
        prefix += f" PP{pp_rank}"
2642

2643
    # Config the process
2644
    kill_itself_when_parent_died()
2645
    setproctitle.setproctitle(f"sglang::scheduler{prefix.replace(' ', '_')}")
2646
    faulthandler.enable()
2647
    parent_process = psutil.Process().parent()
2648

2649
2650
2651
    # [For Router] if env var "SGLANG_DP_RANK" exist, set dp_rank to the value of the env var
    if dp_rank is None and "SGLANG_DP_RANK" in os.environ:
        dp_rank = int(os.environ["SGLANG_DP_RANK"])
2652

Wang Ran (汪然)'s avatar
Wang Ran (汪然) committed
2653
    # Configure the logger
2654
    configure_logger(server_args, prefix=prefix)
2655
    suppress_other_loggers()
2656

2657
    # Set cpu affinity to this gpu process
2658
2659
2660
    if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
        set_gpu_proc_affinity(server_args.tp_size, server_args.nnodes, gpu_id)

2661
2662
2663
2664
    embedding_cache_size = 100
    if "SGLANG_VLM_CACHE_SIZE_MB" in os.environ:
        embedding_cache_size = int(os.environ["SGLANG_VLM_CACHE_SIZE_MB"])
    init_embedding_cache(embedding_cache_size * 1024 * 1024)
2665
    # Create a scheduler and run the event loop
2666
    try:
2667
        scheduler = Scheduler(server_args, port_args, gpu_id, tp_rank, pp_rank, dp_rank)
2668
        pipe_writer.send(
Mick's avatar
Mick committed
2669
2670
2671
2672
2673
            {
                "status": "ready",
                "max_total_num_tokens": scheduler.max_total_num_tokens,
                "max_req_input_len": scheduler.max_req_input_len,
            }
2674
        )
Byron Hsu's avatar
Byron Hsu committed
2675
2676
2677
        disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode

        if disaggregation_mode == DisaggregationMode.NULL:
2678
2679
2680
            if server_args.pp_size > 1:
                scheduler.event_loop_pp()
            elif scheduler.enable_overlap:
Byron Hsu's avatar
Byron Hsu committed
2681
2682
2683
2684
                scheduler.event_loop_overlap()
            else:
                scheduler.event_loop_normal()
        elif disaggregation_mode == DisaggregationMode.PREFILL:
2685
2686
2687
2688
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_prefill()
            else:
                scheduler.event_loop_normal_disagg_prefill()
2689

Byron Hsu's avatar
Byron Hsu committed
2690
        elif disaggregation_mode == DisaggregationMode.DECODE:
2691
2692
2693
2694
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_decode()
            else:
                scheduler.event_loop_normal_disagg_decode()
Byron Hsu's avatar
Byron Hsu committed
2695

2696
    except Exception:
2697
2698
2699
        traceback = get_exception_traceback()
        logger.error(f"Scheduler hit an exception: {traceback}")
        parent_process.send_signal(signal.SIGQUIT)