scheduler.py 110 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
14
15
"""A scheduler that manages a tensor parallel GPU worker."""

16
import datetime
17
import faulthandler
18
import logging
19
import os
20
import signal
21
import sys
Lianmin Zheng's avatar
Lianmin Zheng committed
22
import threading
23
import time
24
from collections import defaultdict, deque
Lianmin Zheng's avatar
Lianmin Zheng committed
25
from concurrent import futures
26
from dataclasses import dataclass
27
from pathlib import Path
28
from types import SimpleNamespace
29
from typing import Dict, List, Optional, Tuple, Union
30

31
import psutil
32
import setproctitle
33
import torch
34
import zmq
35
from torch.distributed import barrier
36

37
from sglang.global_config import global_config
Lianmin Zheng's avatar
Lianmin Zheng committed
38
from sglang.srt.configs.model_config import ModelConfig
39
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE, GPU_MEMORY_TYPE_WEIGHTS
40
41
42
43
from sglang.srt.constrained.base_grammar_backend import (
    INVALID_GRAMMAR_OBJ,
    create_grammar_backend,
)
Byron Hsu's avatar
Byron Hsu committed
44
45
46
47
48
from sglang.srt.disaggregation.decode import (
    DecodePreallocQueue,
    DecodeTransferQueue,
    SchedulerDisaggregationDecodeMixin,
)
49
from sglang.srt.disaggregation.kv_events import EventPublisherFactory, KVEventBatch
Byron Hsu's avatar
Byron Hsu committed
50
51
52
53
54
55
from sglang.srt.disaggregation.prefill import (
    PrefillBootstrapQueue,
    SchedulerDisaggregationPrefillMixin,
)
from sglang.srt.disaggregation.utils import (
    DisaggregationMode,
56
    MetadataBuffers,
Byron Hsu's avatar
Byron Hsu committed
57
    ReqToMetadataIdxAllocator,
58
    TransferBackend,
59
    prepare_abort,
Byron Hsu's avatar
Byron Hsu committed
60
)
61
from sglang.srt.distributed import get_pp_group, get_world_group
fzyzcjy's avatar
fzyzcjy committed
62
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
xm:D's avatar
xm:D committed
63
64
65
66
67
from sglang.srt.hf_transformers_utils import (
    get_processor,
    get_tokenizer,
    get_tokenizer_from_processor,
)
68
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
69
70
71
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.io_struct import (
    AbortReq,
72
    CloseSessionReqInput,
73
    ExpertDistributionReq,
74
    ExpertDistributionReqOutput,
75
76
    FlushCacheReqInput,
    FlushCacheReqOutput,
77
78
    GetInternalStateReq,
    GetInternalStateReqOutput,
79
80
    GetWeightsByNameReqInput,
    GetWeightsByNameReqOutput,
81
    HealthCheckOutput,
82
83
    InitWeightsUpdateGroupReqInput,
    InitWeightsUpdateGroupReqOutput,
84
85
    LoadLoRAAdapterReqInput,
    LoadLoRAAdapterReqOutput,
86
87
    OpenSessionReqInput,
    OpenSessionReqOutput,
88
    ProfileReq,
89
90
    ProfileReqOutput,
    ProfileReqType,
91
92
93
94
    ReleaseMemoryOccupationReqInput,
    ReleaseMemoryOccupationReqOutput,
    ResumeMemoryOccupationReqInput,
    ResumeMemoryOccupationReqOutput,
95
96
    RpcReqInput,
    RpcReqOutput,
97
98
    SetInternalStateReq,
    SetInternalStateReqOutput,
99
100
    SlowDownReqInput,
    SlowDownReqOutput,
101
102
    TokenizedEmbeddingReqInput,
    TokenizedGenerateReqInput,
103
104
    UnloadLoRAAdapterReqInput,
    UnloadLoRAAdapterReqOutput,
Chayenne's avatar
Chayenne committed
105
106
    UpdateWeightFromDiskReqInput,
    UpdateWeightFromDiskReqOutput,
107
108
    UpdateWeightsFromDistributedReqInput,
    UpdateWeightsFromDistributedReqOutput,
109
110
    UpdateWeightsFromTensorReqInput,
    UpdateWeightsFromTensorReqOutput,
111
)
112
from sglang.srt.managers.mm_utils import init_embedding_cache
113
114
from sglang.srt.managers.schedule_batch import (
    FINISH_ABORT,
Mick's avatar
Mick committed
115
    MultimodalInputs,
116
117
    Req,
    ScheduleBatch,
118
    global_server_args_dict,
119
)
120
121
122
123
124
from sglang.srt.managers.schedule_policy import (
    AddReqResult,
    PrefillAdder,
    SchedulePolicy,
)
125
126
127
from sglang.srt.managers.scheduler_output_processor_mixin import (
    SchedulerOutputProcessorMixin,
)
128
from sglang.srt.managers.session_controller import Session
129
from sglang.srt.managers.tp_worker import TpModelWorker
130
from sglang.srt.managers.tp_worker_overlap_thread import TpModelWorkerClient
131
from sglang.srt.managers.utils import validate_input_length
tarinkk's avatar
tarinkk committed
132
133
from sglang.srt.mem_cache.allocator import SWATokenToKVPoolAllocator
from sglang.srt.mem_cache.chunk_cache import ChunkCache, SWAChunkCache
134
from sglang.srt.mem_cache.hiradix_cache import HiRadixCache
135
from sglang.srt.mem_cache.radix_cache import RadixCache
136
from sglang.srt.metrics.collector import SchedulerMetricsCollector, SchedulerStats
Lianmin Zheng's avatar
Lianmin Zheng committed
137
from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors
138
from sglang.srt.reasoning_parser import ReasoningParser
139
from sglang.srt.server_args import PortArgs, ServerArgs
140
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
141
from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter
142
from sglang.srt.two_batch_overlap import TboDPAttentionPreparer
143
from sglang.srt.utils import (
144
    DeepEPMode,
145
    DynamicGradMode,
146
    broadcast_pyobj,
fzyzcjy's avatar
fzyzcjy committed
147
    configure_gc_logger,
148
    configure_logger,
Lianmin Zheng's avatar
Lianmin Zheng committed
149
    disable_request_logging,
150
    get_available_gpu_memory,
151
    get_bool_env_var,
152
    get_zmq_socket,
153
    is_cpu,
Lianmin Zheng's avatar
Lianmin Zheng committed
154
    kill_itself_when_parent_died,
155
    point_to_point_pyobj,
156
    pyspy_dump_schedulers,
157
158
    require_mlp_sync,
    require_mlp_tp_gather,
159
    set_gpu_proc_affinity,
160
161
162
    set_random_seed,
    suppress_other_loggers,
)
163
from sglang.utils import TypeBasedDispatcher, get_exception_traceback
164
165
166

logger = logging.getLogger(__name__)

167
# Test retract decode for debugging purposes
168
169
TEST_RETRACT = get_bool_env_var("SGLANG_TEST_RETRACT")
RECORD_STEP_TIME = get_bool_env_var("SGLANG_RECORD_STEP_TIME")
170
GRAMMAR_TIMEOUT = float(os.environ.get("SGLANG_GRAMMAR_TIMEOUT", 300))
171

172
173
_is_cpu = is_cpu()

174

175
176
@dataclass
class GenerationBatchResult:
177
178
179
    logits_output: Optional[LogitsProcessorOutput]
    pp_hidden_states_proxy_tensors: Optional[torch.Tensor]
    next_token_ids: Optional[List[int]]
180
181
    extend_input_len_per_req: List[int]
    extend_logprob_start_len_per_req: List[int]
182
    bid: int
183
    can_run_cuda_graph: bool
184
185
186
187
188
189
190
191


@dataclass
class EmbeddingBatchResult:
    embeddings: torch.Tensor
    bid: int


192
193
194
195
196
197
198
199
200
201
202
203
class KvMetrics:
    def __init__(self):
        self.request_active_slots = None
        self.request_total_slots = None
        self.kv_active_blocks = None
        self.kv_total_blocks = None
        self.num_requests_waiting = None
        self.gpu_cache_usage_perc = None
        self.gpu_prefix_cache_hit_rate = None
        self.data_parallel_rank = None


204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
class IdleSleeper:
    """
    In setups which have long inactivity periods it is desirable to reduce
    system power consumption when sglang does nothing. This would lead not only
    to power savings, but also to more CPU thermal headroom when a request
    eventually comes. This is important in cases when multiple GPUs are connected
    as each GPU would otherwise pin one thread at 100% CPU usage.

    The simplest solution is to use zmq.Poller on all sockets that may receive
    data that needs handling immediately.
    """

    def __init__(self, sockets):
        self.poller = zmq.Poller()
        for s in sockets:
            self.poller.register(s, zmq.POLLIN)

    def maybe_sleep(self):
        self.poller.poll(1000)


Byron Hsu's avatar
Byron Hsu committed
225
226
227
228
229
class Scheduler(
    SchedulerOutputProcessorMixin,
    SchedulerDisaggregationDecodeMixin,
    SchedulerDisaggregationPrefillMixin,
):
230
231
232
233
234
235
236
237
    """A scheduler that manages a tensor parallel GPU worker."""

    def __init__(
        self,
        server_args: ServerArgs,
        port_args: PortArgs,
        gpu_id: int,
        tp_rank: int,
238
        pp_rank: int,
239
        dp_rank: Optional[int],
240
241
    ):
        # Parse args
242
        self.server_args = server_args
243
        self.tp_rank = tp_rank
244
        self.pp_rank = pp_rank
245
        self.dp_rank = dp_rank
246
        self.tp_size = server_args.tp_size
247
248
        self.pp_size = server_args.pp_size
        self.dp_size = server_args.dp_size
249
250
251
        self.schedule_policy = server_args.schedule_policy
        self.lora_paths = server_args.lora_paths
        self.max_loras_per_batch = server_args.max_loras_per_batch
252
        self.enable_overlap = not server_args.disable_overlap_schedule
253
        self.skip_tokenizer_init = server_args.skip_tokenizer_init
254
        self.enable_metrics = server_args.enable_metrics
255
        self.enable_kv_cache_events = server_args.kv_events_config is not None
256
        self.stream_interval = server_args.stream_interval
257
258
259
        self.spec_algorithm = SpeculativeAlgorithm.from_string(
            server_args.speculative_algorithm
        )
260
261
        self.gpu_id = gpu_id
        self.enable_hierarchical_cache = server_args.enable_hierarchical_cache
Lianmin Zheng's avatar
Lianmin Zheng committed
262
        self.page_size = server_args.page_size
263
264
        self.dp_size = server_args.dp_size
        self.attn_tp_rank, self.attn_tp_size, self.attn_dp_rank = (
265
266
267
268
269
270
271
272
            compute_dp_attention_world_info(
                server_args.enable_dp_attention,
                self.tp_rank,
                self.tp_size,
                self.dp_size,
            )
        )

273
274
        # Init inter-process communication
        context = zmq.Context(2)
275
276
        self.idle_sleeper = None

277
        if self.pp_rank == 0 and self.attn_tp_rank == 0:
278
            self.recv_from_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
279
                context, zmq.PULL, port_args.scheduler_input_ipc_name, False
280
            )
281
            self.send_to_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
282
                context, zmq.PUSH, port_args.tokenizer_ipc_name, False
283
            )
284
285
286
            self.send_metrics_from_scheduler = get_zmq_socket(
                context, zmq.PUSH, port_args.metrics_ipc_name, False
            )
287

288
            if server_args.skip_tokenizer_init:
289
                # Directly send to the TokenizerManager
290
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
291
                    context, zmq.PUSH, port_args.tokenizer_ipc_name, False
292
293
                )
            else:
294
                # Send to the DetokenizerManager
295
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
296
                    context, zmq.PUSH, port_args.detokenizer_ipc_name, False
297
                )
298
299
300
301

            self.recv_from_rpc = get_zmq_socket(
                context, zmq.DEALER, port_args.rpc_ipc_name, False
            )
302
303
304
305
306
307
308
            if self.server_args.sleep_on_idle:
                self.idle_sleeper = IdleSleeper(
                    [
                        self.recv_from_tokenizer,
                        self.recv_from_rpc,
                    ]
                )
309
        else:
310
            self.recv_from_tokenizer = None
311
            self.recv_from_rpc = None
312
            self.send_metrics_from_scheduler = None
313
314
            self.send_to_tokenizer = SimpleNamespace(send_pyobj=lambda x: None)
            self.send_to_detokenizer = SimpleNamespace(send_pyobj=lambda x: None)
315
316

        # Init tokenizer
317
        self.init_tokenizer()
318

319
320
321
322
323
324
325
326
327
        # Set reasoning_parser and think_end_id if --reasoning_parser is enabled
        if self.server_args.reasoning_parser and self.tokenizer:
            reasoning_parser = ReasoningParser(
                model_type=self.server_args.reasoning_parser, stream_reasoning=False
            )
            self.tokenizer.think_end_id = self.tokenizer.encode(
                reasoning_parser.detector.think_end_token, add_special_tokens=False
            )[0]

328
329
330
331
        # Check whether overlap can be enabled
        if not self.is_generation:
            self.enable_overlap = False
            logger.info("Overlap scheduler is disabled for embedding models.")
332

333
        # Launch a tensor parallel worker
334
        if self.enable_overlap:
335
            TpWorkerClass = TpModelWorkerClient
336
337
        else:
            TpWorkerClass = TpModelWorker
338

339
        self.tp_worker = TpWorkerClass(
340
            server_args=server_args,
341
342
            gpu_id=gpu_id,
            tp_rank=tp_rank,
343
            pp_rank=pp_rank,
344
            dp_rank=dp_rank,
345
            nccl_port=port_args.nccl_port,
346
        )
347

348
        # Launch a draft worker for speculative decoding
349
350
351
352
353
354
355
356
357
358
359
360
361
362
        if self.spec_algorithm.is_eagle():
            from sglang.srt.speculative.eagle_worker import EAGLEWorker

            self.draft_worker = EAGLEWorker(
                gpu_id=gpu_id,
                tp_rank=tp_rank,
                server_args=server_args,
                nccl_port=port_args.nccl_port,
                target_worker=self.tp_worker,
                dp_rank=dp_rank,
            )
        else:
            self.draft_worker = None

363
        # Get token and memory info from the model worker
364
365
366
367
        (
            self.max_total_num_tokens,
            self.max_prefill_tokens,
            self.max_running_requests,
368
            self.max_req_len,
369
370
            self.max_req_input_len,
            self.random_seed,
371
            self.device,
372
373
374
375
376
            worker_global_server_args_dict,
            _,
            _,
            _,
        ) = self.tp_worker.get_worker_info()
377
378
379
380
381
382
383
384
        if global_server_args_dict["max_micro_batch_size"] is None:
            global_server_args_dict["max_micro_batch_size"] = max(
                self.max_running_requests // server_args.pp_size, 1
            )

        self.tp_group = self.tp_worker.get_tp_group()
        self.tp_cpu_group = self.tp_group.cpu_group
        self.attn_tp_group = self.tp_worker.get_attention_tp_group()
385
        self.attn_tp_cpu_group = self.tp_worker.get_attention_tp_cpu_group()
386
387
388
        self.pp_group = get_pp_group()
        self.world_group = get_world_group()

389
        self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
390
        global_server_args_dict.update(worker_global_server_args_dict)
391
        set_random_seed(self.random_seed)
392

393
        # Print debug info
394
        if tp_rank == 0:
395
396
397
            avail_mem = get_available_gpu_memory(
                self.device, self.gpu_id, empty_cache=False
            )
398
399
400
401
402
            logger.info(
                f"max_total_num_tokens={self.max_total_num_tokens}, "
                f"chunked_prefill_size={server_args.chunked_prefill_size}, "
                f"max_prefill_tokens={self.max_prefill_tokens}, "
                f"max_running_requests={self.max_running_requests}, "
403
404
                f"context_len={self.model_config.context_len}, "
                f"available_gpu_mem={avail_mem:.2f} GB"
405
            )
406

Lianmin Zheng's avatar
Lianmin Zheng committed
407
        # Init memory pool and cache
408
        self.init_memory_pool_and_cache()
409
410
411

        # Init running status
        self.waiting_queue: List[Req] = []
412
        # The running decoding batch for continuous batching
Lianmin Zheng's avatar
Lianmin Zheng committed
413
        self.running_batch: ScheduleBatch = ScheduleBatch(reqs=[], batch_is_full=False)
414
        # The current forward batch
Lianmin Zheng's avatar
Lianmin Zheng committed
415
        self.cur_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
416
        # The last forward batch
417
        self.last_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
418
419
        self.forward_ct = 0
        self.forward_ct_decode = 0
420
        self.num_generated_tokens = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
421
        self.last_prefill_tokens = 0
422
423
        self.last_decode_stats_tic = time.perf_counter()
        self.last_prefill_stats_tic = time.perf_counter()
424
        self.return_health_check_ct = 0
425
426
427
428
429
        self.num_retracted_reqs: int = 0
        self.num_paused_reqs: int = 0
        self.kv_transfer_speed_gb_s: float = 0.0
        self.kv_transfer_latency_ms: float = 0.0
        self.sessions: Dict[str, Session] = {}
430
        self.current_stream = torch.get_device_module(self.device).current_stream()
431
432
        if self.device == "cpu":
            self.current_stream.synchronize = lambda: None  # No-op for CPU
433
        self.forward_sleep_time = None
434

435
436
        # Init chunked prefill
        self.chunked_prefill_size = server_args.chunked_prefill_size
437
438
        if self.chunked_prefill_size <= 0:  # -1 means disable
            self.chunked_prefill_size = None
439
        self.chunked_req = None
440
441
442
443
        self.is_mixed_chunk = (
            self.chunked_prefill_size is not None and server_args.enable_mixed_chunk
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
444
        # Init the grammar backend for constrained generation
445
        self.grammar_queue: List[Req] = []
446
        if not server_args.skip_tokenizer_init:
447
448
449
            self.grammar_backend = create_grammar_backend(
                server_args, self.tokenizer, self.model_config.vocab_size
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
450
451
        else:
            self.grammar_backend = None
452

453
        # Init schedule policy and new token estimation
454
        self.policy = SchedulePolicy(
Lianmin Zheng's avatar
Lianmin Zheng committed
455
456
457
            self.schedule_policy,
            self.tree_cache,
            self.enable_hierarchical_cache,
458
        )
459
460
461
        assert (
            server_args.schedule_conservativeness >= 0
        ), "Invalid schedule_conservativeness"
462
463
        self.init_new_token_ratio = min(
            global_config.default_init_new_token_ratio
464
465
            * server_args.schedule_conservativeness,
            1.0,
466
        )
467
468
469
470
471
472
473
474
475
476
        self.min_new_token_ratio = min(
            self.init_new_token_ratio
            * global_config.default_min_new_token_ratio_factor,
            1.0,
        )
        self.new_token_ratio_decay = (
            self.init_new_token_ratio - self.min_new_token_ratio
        ) / global_config.default_new_token_ratio_decay_steps
        self.new_token_ratio = self.init_new_token_ratio

Lianmin Zheng's avatar
Lianmin Zheng committed
477
478
479
480
        # Init watchdog thread
        self.watchdog_timeout = server_args.watchdog_timeout
        t = threading.Thread(target=self.watchdog_thread, daemon=True)
        t.start()
481
        self.parent_process = psutil.Process().parent()
482
483

        # Init memory saver, profiler and metric stats
484
485
486
        self.memory_saver_adapter = TorchMemorySaverAdapter.create(
            enable=server_args.enable_memory_saver
        )
487
        self.init_profier()
488
        self.init_metrics()
489
        self.init_kv_events(server_args.kv_events_config)
490

491
492
        # Init request dispatcher
        self._request_dispatcher = TypeBasedDispatcher(
493
494
495
            [
                (TokenizedGenerateReqInput, self.handle_generate_request),
                (TokenizedEmbeddingReqInput, self.handle_embedding_request),
496
                (FlushCacheReqInput, self.flush_cache_wrapped),
497
                (AbortReq, self.abort_request),
498
499
                (OpenSessionReqInput, self.open_session),
                (CloseSessionReqInput, self.close_session),
500
501
502
503
504
505
506
507
                (UpdateWeightFromDiskReqInput, self.update_weights_from_disk),
                (InitWeightsUpdateGroupReqInput, self.init_weights_update_group),
                (
                    UpdateWeightsFromDistributedReqInput,
                    self.update_weights_from_distributed,
                ),
                (UpdateWeightsFromTensorReqInput, self.update_weights_from_tensor),
                (GetWeightsByNameReqInput, self.get_weights_by_name),
508
509
                (ReleaseMemoryOccupationReqInput, self.release_memory_occupation),
                (ResumeMemoryOccupationReqInput, self.resume_memory_occupation),
510
                (SlowDownReqInput, self.slow_down),
511
                (ProfileReq, self.profile),
512
                (GetInternalStateReq, self.get_internal_state),
513
                (SetInternalStateReq, self.set_internal_state),
514
                (RpcReqInput, self.handle_rpc_request),
515
                (ExpertDistributionReq, self.expert_distribution_handle),
516
517
                (LoadLoRAAdapterReqInput, self.load_lora_adapter),
                (UnloadLoRAAdapterReqInput, self.unload_lora_adapter),
518
519
520
            ]
        )

521
        # Init disaggregation
Byron Hsu's avatar
Byron Hsu committed
522
523
524
525
526
        self.disaggregation_mode = DisaggregationMode(
            self.server_args.disaggregation_mode
        )
        self.init_disaggregation()

fzyzcjy's avatar
fzyzcjy committed
527
528
529
        if get_bool_env_var("SGLANG_GC_LOG"):
            configure_gc_logger()

530
531
532
533
    def maybe_sleep_on_idle(self):
        if self.idle_sleeper is not None:
            self.idle_sleeper.maybe_sleep()

534
535
    def init_tokenizer(self):
        server_args = self.server_args
Lianmin Zheng's avatar
Lianmin Zheng committed
536

537
        self.model_config = ModelConfig.from_server_args(server_args)
538
        self.is_generation = self.model_config.is_generation
539

540
541
542
543
544
545
546
547
548
        if server_args.skip_tokenizer_init:
            self.tokenizer = self.processor = None
        else:
            if self.model_config.is_multimodal:
                self.processor = get_processor(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
549
                    use_fast=not server_args.disable_fast_image_processor,
550
                )
xm:D's avatar
xm:D committed
551
                self.tokenizer = get_tokenizer_from_processor(self.processor)
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
            else:
                self.tokenizer = get_tokenizer(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
                )

    def init_memory_pool_and_cache(self):
        server_args = self.server_args

        self.req_to_token_pool, self.token_to_kv_pool_allocator = (
            self.tp_worker.get_memory_pool()
        )

        if (
            server_args.chunked_prefill_size is not None
            and server_args.disable_radix_cache
        ):
tarinkk's avatar
tarinkk committed
571
572
573
574
575
            if self.model_config.is_hybrid:
                ChunkCacheClass = SWAChunkCache
            else:
                ChunkCacheClass = ChunkCache
            self.tree_cache = ChunkCacheClass(
576
577
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
578
                page_size=self.page_size,
579
580
581
582
583
584
            )
        else:
            if self.enable_hierarchical_cache:
                self.tree_cache = HiRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
585
586
587
588
589
                    tp_cache_group=(
                        self.attn_tp_cpu_group
                        if self.server_args.enable_dp_attention
                        else self.tp_cpu_group
                    ),
590
                    page_size=self.page_size,
591
                    hicache_ratio=server_args.hicache_ratio,
Zhiqiang Xie's avatar
Zhiqiang Xie committed
592
593
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
594
595
596
597
598
599
                    hicache_io_backend=(
                        "direct"
                        if server_args.attention_backend
                        == "fa3"  # hot fix for incompatibility
                        else server_args.hicache_io_backend
                    ),
600
                )
601
602
603
604
                self.tp_worker.register_hicache_layer_transfer_counter(
                    self.tree_cache.cache_controller.layer_done_counter
                )

605
606
607
608
            else:
                self.tree_cache = RadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Lianmin Zheng's avatar
Lianmin Zheng committed
609
                    page_size=self.page_size,
610
                    disable=server_args.disable_radix_cache,
611
                    enable_kv_cache_events=self.enable_kv_cache_events,
612
613
614
615
616
617
618
619
620
621
622
623
                )

        self.decode_mem_cache_buf_multiplier = (
            1
            if self.spec_algorithm.is_none()
            else (
                server_args.speculative_num_draft_tokens
                + (
                    server_args.speculative_eagle_topk
                    * server_args.speculative_num_steps
                )
            )
624
        )
625

626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
    def init_profier(self):
        self.torch_profiler = None
        self.torch_profiler_output_dir: Optional[str] = None
        self.profiler_activities: Optional[List[str]] = None
        self.profile_id: Optional[str] = None
        self.profiler_target_forward_ct: Optional[int] = None
        self.profiler_target_prefill_ct: Optional[int] = None
        self.profiler_target_decode_ct: Optional[int] = None
        self.profiler_prefill_ct: Optional[int] = None
        self.profiler_decode_ct: Optional[int] = None
        self.profile_by_stage: bool = False
        self.profile_steps: Optional[int] = None
        self.profile_in_progress: bool = False
        self.rpd_profiler = None

641
642
    def init_metrics(self):
        self.last_gen_throughput: float = 0.0
Lianmin Zheng's avatar
Lianmin Zheng committed
643
        self.last_input_throughput: float = 0.0
644
645
646
647
648
649
650
651
652
653
654
655
656
657
        self.step_time_dict = defaultdict(list)  # Dict[batch size -> step time]
        self.spec_num_total_accepted_tokens = 0
        self.spec_num_total_forward_ct = 0
        self.cum_spec_accept_length = 0
        self.cum_spec_accept_count = 0
        self.stats = SchedulerStats()
        if self.enable_metrics:
            engine_type = "unified"
            self.metrics_collector = SchedulerMetricsCollector(
                labels={
                    "model_name": self.server_args.served_model_name,
                    "engine_type": engine_type,
                },
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
658

659
660
    def init_kv_events(self, kv_events_config: Optional[str]):
        if self.enable_kv_cache_events:
661
662
663
            self.kv_event_publisher = EventPublisherFactory.create(
                kv_events_config, self.attn_dp_rank
            )
664

Byron Hsu's avatar
Byron Hsu committed
665
    def init_disaggregation(self):
666
667
668
669
        self.transfer_backend = TransferBackend(
            self.server_args.disaggregation_transfer_backend
        )

Byron Hsu's avatar
Byron Hsu committed
670
671
672
673
        if (
            self.disaggregation_mode == DisaggregationMode.DECODE
        ):  # *2 for the headroom.
            buffer_size = (self.req_to_token_pool.size) * 2
Byron Hsu's avatar
Byron Hsu committed
674
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
675
676
                buffer_size
            )
677
678
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
679
680
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
681
682
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
683
684
685

            # The decode requests polling kv cache
            self.disagg_decode_transfer_queue = DecodeTransferQueue(
686
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
687
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
688
                tp_rank=self.tp_rank,
689
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
690
691
                scheduler=self,
                tree_cache=self.tree_cache,
Byron Hsu's avatar
Byron Hsu committed
692
693
694
695
696
697
            )

            # The decode requests pending for pre-allocation
            self.disagg_decode_prealloc_queue = DecodePreallocQueue(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Byron Hsu's avatar
Byron Hsu committed
698
699
700
701
702
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
703
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
704
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
705
706
707
                scheduler=self,
                transfer_queue=self.disagg_decode_transfer_queue,
                tree_cache=self.tree_cache,
708
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
709
710
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
711
712
                dp_size=self.server_args.dp_size,
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
713
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
714
715
                max_total_num_tokens=self.max_total_num_tokens,
                prefill_pp_size=self.server_args.disaggregation_prefill_pp,
716
                num_reserved_decode_tokens=self.server_args.num_reserved_decode_tokens,
717
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
718
            )
Liangsheng Yin's avatar
Liangsheng Yin committed
719

Byron Hsu's avatar
Byron Hsu committed
720
721
722
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            # *2 for the headroom.
            buffer_size = self.max_running_requests * 2
Byron Hsu's avatar
Byron Hsu committed
723
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
724
725
                buffer_size
            )
726
727
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
728
729
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
730
731
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
732

Liangsheng Yin's avatar
Liangsheng Yin committed
733
            self.disagg_prefill_bootstrap_queue = PrefillBootstrapQueue(
Byron Hsu's avatar
Byron Hsu committed
734
                token_to_kv_pool=self.token_to_kv_pool_allocator.get_kvcache(),
Byron Hsu's avatar
Byron Hsu committed
735
736
737
738
739
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
740
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
741
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
742
743
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
Byron Hsu's avatar
Byron Hsu committed
744
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
745
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
746
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
747
748
749
                max_total_num_tokens=self.max_total_num_tokens,
                decode_tp_size=self.server_args.disaggregation_decode_tp,
                decode_dp_size=self.server_args.disaggregation_decode_dp,
750
                scheduler=self,
Byron Hsu's avatar
Byron Hsu committed
751
752
753
                pp_rank=self.pp_rank,
                pp_size=self.pp_size,
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
754
755
            )
            # The prefill requests that are in the middle of kv sending
756
            self.disagg_prefill_inflight_queue: List[Req] = []
Byron Hsu's avatar
Byron Hsu committed
757

758
    @DynamicGradMode()
759
    def event_loop_normal(self):
760
        """A normal scheduler loop."""
761
        while True:
Lianmin Zheng's avatar
Lianmin Zheng committed
762
763
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)
764

765
            batch = self.get_next_batch_to_run()
Lianmin Zheng's avatar
Lianmin Zheng committed
766
            self.cur_batch = batch
767
768
769
770

            if batch:
                result = self.run_batch(batch)
                self.process_batch_result(batch, result)
Lianmin Zheng's avatar
Lianmin Zheng committed
771
            else:
Lianmin Zheng's avatar
Lianmin Zheng committed
772
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
773
                self.check_memory()
774
                self.new_token_ratio = self.init_new_token_ratio
775
                self.maybe_sleep_on_idle()
776
777

            self.last_batch = batch
778

779
    @DynamicGradMode()
Lianmin Zheng's avatar
Lianmin Zheng committed
780
    def event_loop_overlap(self):
781
        """A scheduler loop that overlaps the CPU processing and GPU computation."""
782
        self.result_queue = deque()
Lianmin Zheng's avatar
Lianmin Zheng committed
783
784
785
786
787
788
789

        while True:
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)

            batch = self.get_next_batch_to_run()
            self.cur_batch = batch
790

Lianmin Zheng's avatar
Lianmin Zheng committed
791
            if batch:
792
                batch.launch_done = threading.Event()
Lianmin Zheng's avatar
Lianmin Zheng committed
793
                result = self.run_batch(batch)
794
                self.result_queue.append((batch.copy(), result))
Lianmin Zheng's avatar
Lianmin Zheng committed
795

796
                if self.last_batch is None:
797
                    # Create a dummy first batch to start the pipeline for overlap schedule.
798
799
800
801
802
803
                    # It is now used for triggering the sampling_info_done event.
                    tmp_batch = ScheduleBatch(
                        reqs=None,
                        forward_mode=ForwardMode.DUMMY_FIRST,
                        next_batch_sampling_info=self.tp_worker.cur_sampling_info,
                    )
804
                    self.process_batch_result(tmp_batch, None, batch.launch_done)
805

Lianmin Zheng's avatar
Lianmin Zheng committed
806
            if self.last_batch:
807
                # Process the results of the last batch
808
                tmp_batch, tmp_result = self.result_queue.popleft()
809
810
811
                tmp_batch.next_batch_sampling_info = (
                    self.tp_worker.cur_sampling_info if batch else None
                )
812
813
814
815
                # NOTE: we should use current launched batch's launch_done event Instead of the last batch's
                self.process_batch_result(
                    tmp_batch, tmp_result, batch.launch_done if batch else None
                )
Lianmin Zheng's avatar
Lianmin Zheng committed
816
            elif batch is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
817
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
818
                self.check_memory()
819
                self.new_token_ratio = self.init_new_token_ratio
820
                self.maybe_sleep_on_idle()
Lianmin Zheng's avatar
Lianmin Zheng committed
821
822
823

            self.last_batch = batch

824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
    @DynamicGradMode()
    def event_loop_pp(self):
        """A non-overlap scheduler loop for pipeline parallelism."""
        mbs = [None] * self.pp_size
        last_mbs = [None] * self.pp_size
        self.running_mbs = [
            ScheduleBatch(reqs=[], batch_is_full=False) for _ in range(self.pp_size)
        ]
        bids = [None] * self.pp_size
        pp_outputs: Optional[PPProxyTensors] = None
        while True:
            server_is_idle = True
            for mb_id in range(self.pp_size):
                self.running_batch = self.running_mbs[mb_id]
                self.last_batch = last_mbs[mb_id]

                recv_reqs = self.recv_requests()
                self.process_input_requests(recv_reqs)
                mbs[mb_id] = self.get_next_batch_to_run()
                self.running_mbs[mb_id] = self.running_batch

                self.cur_batch = mbs[mb_id]
                if self.cur_batch:
                    server_is_idle = False
                    result = self.run_batch(self.cur_batch)

850
                # (last rank) send the outputs to the next step
851
852
853
854
855
856
                if self.pp_group.is_last_rank:
                    if self.cur_batch:
                        next_token_ids, bids[mb_id] = (
                            result.next_token_ids,
                            result.bid,
                        )
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
                        if self.cur_batch.return_logprob:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                    "extend_input_len_per_req": result.extend_input_len_per_req,
                                    "extend_logprob_start_len_per_req": result.extend_logprob_start_len_per_req,
                                }
                                | (
                                    {
                                        f"logits_output.{k}": v
                                        for k, v in result.logits_output.__dict__.items()
                                    }
                                    if result.logits_output is not None
                                    else {}
                                )
                            )
                        else:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                }
                            )
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
                        # send the output from the last round to let the next stage worker run post processing
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                # receive outputs and post-process (filter finished reqs) the coming microbatch
                next_mb_id = (mb_id + 1) % self.pp_size
                next_pp_outputs = None
                if mbs[next_mb_id] is not None:
                    next_pp_outputs: Optional[PPProxyTensors] = PPProxyTensors(
                        self.pp_group.recv_tensor_dict(
                            all_gather_group=self.attn_tp_group
                        )
                    )
                    mbs[next_mb_id].output_ids = next_pp_outputs["next_token_ids"]
895
896
897
898
899
900
901
902
903
                    logits_output_args = {
                        k[len("logits_output.") :]: v
                        for k, v in next_pp_outputs.tensors.items()
                        if k.startswith("logits_output.")
                    }
                    if len(logits_output_args) > 0:
                        logits_output = LogitsProcessorOutput(**logits_output_args)
                    else:
                        logits_output = None
904
                    output_result = GenerationBatchResult(
905
                        logits_output=logits_output,
906
907
                        pp_hidden_states_proxy_tensors=None,
                        next_token_ids=next_pp_outputs["next_token_ids"],
908
909
910
911
912
913
                        extend_input_len_per_req=next_pp_outputs.tensors.get(
                            "extend_input_len_per_req", None
                        ),
                        extend_logprob_start_len_per_req=next_pp_outputs.tensors.get(
                            "extend_logprob_start_len_per_req", None
                        ),
914
                        bid=bids[next_mb_id],
915
                        can_run_cuda_graph=result.can_run_cuda_graph,
916
917
918
919
                    )
                    self.process_batch_result(mbs[next_mb_id], output_result)
                    last_mbs[next_mb_id] = mbs[next_mb_id]

920
                # (not last rank)
921
922
923
                if not self.pp_group.is_last_rank:
                    if self.cur_batch:
                        bids[mb_id] = result.bid
924
925
                    # carry the outputs to the next stage
                    # send the outputs from the last round to let the next stage worker run post processing
926
927
928
929
930
931
932
                    if pp_outputs:
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                    # send out reqs to the next stage
933
                    dp_offset = self.attn_dp_rank * self.attn_tp_size
934
935
936
937
                    if self.attn_tp_rank == 0:
                        point_to_point_pyobj(
                            recv_reqs,
                            self.pp_rank * self.tp_size + dp_offset,
938
                            self.world_group.device_group,
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
                            self.pp_rank * self.tp_size + dp_offset,
                            (self.pp_rank + 1) * self.tp_size + dp_offset,
                        )

                    # send out proxy tensors to the next stage
                    if self.cur_batch:
                        self.pp_group.send_tensor_dict(
                            result.pp_hidden_states_proxy_tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                pp_outputs = next_pp_outputs

            # When the server is idle, self-check and re-init some states
            if server_is_idle:
                self.check_memory()
                self.new_token_ratio = self.init_new_token_ratio
956
                self.maybe_sleep_on_idle()
957

958
959
    def recv_requests(self) -> List[Req]:
        """Receive results at tp_rank = 0 and broadcast it to all other TP ranks."""
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
        if self.pp_rank == 0:
            if self.attn_tp_rank == 0:
                recv_reqs = []

                while True:
                    try:
                        recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_req)

                while True:
                    try:
                        recv_rpc = self.recv_from_rpc.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_rpc)
            else:
                recv_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
979
        else:
980
            if self.attn_tp_rank == 0:
981
                dp_offset = self.attn_dp_rank * self.attn_tp_size
982
983
984
                recv_reqs = point_to_point_pyobj(
                    [],
                    self.pp_rank * self.tp_size + dp_offset,
985
                    self.world_group.device_group,
986
987
988
989
990
                    (self.pp_rank - 1) * self.tp_size + dp_offset,
                    self.pp_rank * self.tp_size + dp_offset,
                )
            else:
                recv_reqs = None
991

992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
        if self.server_args.enable_dp_attention:
            if self.attn_tp_rank == 0:
                work_reqs = [
                    req
                    for req in recv_reqs
                    if isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
                control_reqs = [
                    req
                    for req in recv_reqs
                    if not isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
            else:
                work_reqs = None
                control_reqs = None

            if self.attn_tp_size != 1:
                work_reqs = broadcast_pyobj(
                    work_reqs,
1015
                    self.attn_tp_group.rank,
1016
                    self.attn_tp_cpu_group,
1017
                    src=self.attn_tp_group.ranks[0],
1018
1019
1020
                )
            if self.tp_size != 1:
                control_reqs = broadcast_pyobj(
1021
1022
1023
1024
                    control_reqs,
                    self.tp_group.rank,
                    self.tp_cpu_group,
                    src=self.tp_group.ranks[0],
1025
1026
1027
                )
            recv_reqs = work_reqs + control_reqs
        elif self.tp_size != 1:
1028
1029
1030
1031
1032
1033
            recv_reqs = broadcast_pyobj(
                recv_reqs,
                self.tp_group.rank,
                self.tp_cpu_group,
                src=self.tp_group.ranks[0],
            )
1034
1035
        return recv_reqs

Lianmin Zheng's avatar
Lianmin Zheng committed
1036
    def process_input_requests(self, recv_reqs: List):
1037
        for recv_req in recv_reqs:
1038
1039
            # If it is a health check generation request and there are running requests, ignore it.
            if is_health_check_generate_req(recv_req) and (
Lianmin Zheng's avatar
Lianmin Zheng committed
1040
                self.chunked_req is not None or not self.running_batch.is_empty()
1041
1042
1043
1044
            ):
                self.return_health_check_ct += 1
                continue

1045
            output = self._request_dispatcher(recv_req)
1046
            if output is not None:
1047
1048
1049
1050
1051
                if isinstance(output, RpcReqOutput):
                    if self.recv_from_rpc is not None:
                        self.recv_from_rpc.send_pyobj(output)
                else:
                    self.send_to_tokenizer.send_pyobj(output)
1052
1053
1054
1055
1056

    def handle_generate_request(
        self,
        recv_req: TokenizedGenerateReqInput,
    ):
1057
        # Create a new request
1058
1059
1060
1061
1062
        if (
            recv_req.session_params is None
            or recv_req.session_params.id is None
            or recv_req.session_params.id not in self.sessions
        ):
Rin Intachuen's avatar
Rin Intachuen committed
1063
1064
1065
1066
1067
1068
            if recv_req.input_embeds is not None:
                # Generate fake input_ids based on the length of input_embeds
                seq_length = len(recv_req.input_embeds)
                fake_input_ids = [1] * seq_length
                recv_req.input_ids = fake_input_ids

1069
1070
1071
1072
            if recv_req.bootstrap_port is None:
                # Use default bootstrap port
                recv_req.bootstrap_port = self.server_args.disaggregation_bootstrap_port

1073
1074
1075
1076
1077
            req = Req(
                recv_req.rid,
                recv_req.input_text,
                recv_req.input_ids,
                recv_req.sampling_params,
Lianmin Zheng's avatar
Lianmin Zheng committed
1078
1079
                return_logprob=recv_req.return_logprob,
                top_logprobs_num=recv_req.top_logprobs_num,
1080
                token_ids_logprob=recv_req.token_ids_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1081
                stream=recv_req.stream,
1082
                lora_path=recv_req.lora_path,
Rin Intachuen's avatar
Rin Intachuen committed
1083
                input_embeds=recv_req.input_embeds,
Lianmin Zheng's avatar
Lianmin Zheng committed
1084
                custom_logit_processor=recv_req.custom_logit_processor,
1085
                return_hidden_states=recv_req.return_hidden_states,
1086
                eos_token_ids=self.model_config.hf_eos_token_id,
1087
                bootstrap_host=recv_req.bootstrap_host,
1088
                bootstrap_port=recv_req.bootstrap_port,
1089
                bootstrap_room=recv_req.bootstrap_room,
1090
                data_parallel_rank=recv_req.data_parallel_rank,
1091
1092
            )
            req.tokenizer = self.tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
1093

1094
1095
1096
            if self.disaggregation_mode != DisaggregationMode.NULL:
                # Invalid request for disaggregated mode
                if recv_req.bootstrap_room is None:
1097
                    error_msg = (
1098
1099
1100
                        f"Invalid request: Disaggregated request received without "
                        f"boostrap room id. {req.rid=}"
                    )
1101
1102
                    logger.error(error_msg)
                    prepare_abort(req, error_msg)
1103
1104
1105
                    self.stream_output([req], req.return_logprob)
                    return

1106
1107
1108
1109
            if (
                recv_req.session_params is not None
                and recv_req.session_params.id is not None
            ):
1110
                req.set_finish_with_abort(
1111
                    f"Invalid request: session id {recv_req.session_params.id} does not exist"
1112
                )
1113
                self._add_request_to_queue(req)
1114
1115
                return
        else:
1116
1117
            # Create a new request from a previous session
            session = self.sessions[recv_req.session_params.id]
1118
            req = session.create_req(recv_req, self.tokenizer)
1119
            if isinstance(req.finished_reason, FINISH_ABORT):
1120
                self._add_request_to_queue(req)
1121
                return
1122

1123
        # Handle multimodal inputs
Mick's avatar
Mick committed
1124
1125
        if recv_req.mm_inputs is not None:
            image_inputs = MultimodalInputs.from_dict(recv_req.mm_inputs)
1126
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
1127
            req.origin_input_ids = self.pad_input_ids_func(
1128
                req.origin_input_ids, image_inputs
1129
            )
1130
            req.extend_image_inputs(image_inputs)
1131

1132
            if len(req.origin_input_ids) >= self.max_req_input_len:
1133
1134
1135
1136
1137
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1138
                )
1139
                self._add_request_to_queue(req)
1140
1141
                return

1142
        # Validate prompt length
1143
1144
1145
1146
1147
1148
        error_msg = validate_input_length(
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
        if error_msg:
1149
            req.set_finish_with_abort(error_msg)
1150
            self._add_request_to_queue(req)
1151
            return
1152

1153
        # Copy more attributes
1154
        if recv_req.logprob_start_len == -1 or not recv_req.return_logprob:
1155
1156
1157
1158
1159
            # By default, only return the logprobs for output tokens
            req.logprob_start_len = len(req.origin_input_ids) - 1
        else:
            req.logprob_start_len = recv_req.logprob_start_len

1160
        if req.logprob_start_len >= len(req.origin_input_ids):
1161
            error_msg = f"{req.logprob_start_len=} is higher than the number of input tokens {len(req.origin_input_ids)=}. Please use a smaller logprob_start_len."
1162
            req.logprob_start_len = len(req.origin_input_ids) - 1
1163
            req.set_finish_with_abort(error_msg)
1164
1165
1166
            self._add_request_to_queue(req)
            return

1167
1168
1169
1170
1171
1172
        req.sampling_params.max_new_tokens = min(
            (
                req.sampling_params.max_new_tokens
                if req.sampling_params.max_new_tokens is not None
                else 1 << 30
            ),
1173
            self.max_req_len - len(req.origin_input_ids) - 1,
1174
1175
        )

1176
1177
1178
1179
1180
        # Init grammar cache for this request
        add_to_grammar_queue = False
        if (
            req.sampling_params.json_schema is not None
            or req.sampling_params.regex is not None
1181
            or req.sampling_params.ebnf is not None
1182
            or req.sampling_params.structural_tag is not None
1183
1184
1185
1186
1187
1188
        ):
            assert self.grammar_backend is not None
            if req.sampling_params.json_schema is not None:
                key = ("json", req.sampling_params.json_schema)
            elif req.sampling_params.regex is not None:
                key = ("regex", req.sampling_params.regex)
1189
1190
            elif req.sampling_params.ebnf is not None:
                key = ("ebnf", req.sampling_params.ebnf)
1191
1192
            elif req.sampling_params.structural_tag:
                key = ("structural_tag", req.sampling_params.structural_tag)
1193

1194
1195
1196
1197
1198
            value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
            req.grammar = value

            if not cache_hit:
                req.grammar_key = key
1199
                add_to_grammar_queue = True
1200
1201
1202
1203
            else:
                if value is INVALID_GRAMMAR_OBJ:  # We hit a cached invalid grammar.
                    error_msg = f"Invalid grammar request with cache hit: {key=}"
                    req.set_finish_with_abort(error_msg)
1204
1205

        if add_to_grammar_queue:
1206
            req.queue_time_start = time.perf_counter()
1207
1208
            self.grammar_queue.append(req)
        else:
1209
1210
1211
            self._add_request_to_queue(req)

    def _add_request_to_queue(self, req: Req):
1212
        req.queue_time_start = time.perf_counter()
Byron Hsu's avatar
Byron Hsu committed
1213
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1214
1215
1216
            self.disagg_prefill_bootstrap_queue.add(
                req, self.model_config.num_key_value_heads
            )
Byron Hsu's avatar
Byron Hsu committed
1217
1218
1219
1220
1221
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            self.disagg_decode_prealloc_queue.add(req)
        else:
            self.waiting_queue.append(req)

1222
    def _extend_requests_to_queue(self, reqs: List[Req], is_retracted: bool = False):
1223
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1224
1225
1226
            self.disagg_prefill_bootstrap_queue.extend(
                reqs, self.model_config.num_key_value_heads
            )
1227
1228
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            # If this is a decode server, we put the request to the decode pending prealloc queue
1229
            self.disagg_decode_prealloc_queue.extend(reqs, is_retracted)
Byron Hsu's avatar
Byron Hsu committed
1230
1231
        else:
            self.waiting_queue.extend(reqs)
1232
1233
1234

    def handle_embedding_request(
        self,
1235
        recv_req: TokenizedEmbeddingReqInput,
1236
1237
1238
1239
1240
1241
    ):
        req = Req(
            recv_req.rid,
            recv_req.input_text,
            recv_req.input_ids,
            recv_req.sampling_params,
woodx's avatar
woodx committed
1242
            token_type_ids=recv_req.token_type_ids,
1243
1244
1245
        )
        req.tokenizer = self.tokenizer

1246
1247
        # Handle multimodal inputs
        if recv_req.image_inputs is not None:
Mick's avatar
Mick committed
1248
            image_inputs = MultimodalInputs.from_dict(recv_req.image_inputs)
1249
1250
1251
1252
1253
1254
1255
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
            req.origin_input_ids = self.pad_input_ids_func(
                req.origin_input_ids, image_inputs
            )
            req.extend_image_inputs(image_inputs)

            if len(req.origin_input_ids) >= self.max_req_input_len:
1256
1257
1258
1259
1260
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1261
                )
1262
                self._add_request_to_queue(req)
1263
1264
                return

1265
        # Validate prompts length
1266
        error_msg = validate_input_length(
1267
1268
1269
1270
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
1271
        if error_msg:
1272
            self._add_request_to_queue(req)
1273
            return
1274

1275
1276
        # Copy more attributes
        req.logprob_start_len = len(req.origin_input_ids) - 1
1277
        self._add_request_to_queue(req)
1278

1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
    def _emit_kv_metrics(self):
        kv_metrics = KvMetrics()
        kv_metrics.request_active_slots = self.stats.num_running_reqs
        kv_metrics.request_total_slots = self.max_running_requests
        kv_metrics.kv_active_blocks = int(
            self.stats.token_usage * self.max_total_num_tokens
        )
        kv_metrics.kv_total_blocks = self.max_total_num_tokens
        kv_metrics.num_requests_waiting = self.stats.num_queue_reqs
        kv_metrics.gpu_cache_usage_perc = self.stats.token_usage
        kv_metrics.gpu_prefix_cache_hit_rate = self.stats.cache_hit_rate
        kv_metrics.data_parallel_rank = self.dp_rank if self.dp_rank is not None else 0

        if not self.send_metrics_from_scheduler.closed:
            self.send_metrics_from_scheduler.send_pyobj(kv_metrics)

1295
1296
1297
1298
    def log_prefill_stats(
        self,
        adder: PrefillAdder,
        can_run_list: List[Req],
1299
        running_bs: int,
1300
    ):
1301
1302
        gap_latency = time.perf_counter() - self.last_prefill_stats_tic
        self.last_prefill_stats_tic = time.perf_counter()
Liangsheng Yin's avatar
Liangsheng Yin committed
1303
1304
        self.last_input_throughput = self.last_prefill_tokens / gap_latency
        self.last_prefill_tokens = adder.log_input_tokens
Lianmin Zheng's avatar
Lianmin Zheng committed
1305

tarinkk's avatar
tarinkk committed
1306
1307
        usage_msg, num_used = self.token_to_kv_pool_allocator.log_usage(
            self.tree_cache.evictable_size()
1308
1309
        )

1310
        num_new_seq = len(can_run_list)
1311
        f = (
1312
            f"Prefill batch. "
1313
            f"#new-seq: {num_new_seq}, "
1314
1315
            f"#new-token: {adder.log_input_tokens}, "
            f"#cached-token: {adder.log_hit_tokens}, "
tarinkk's avatar
tarinkk committed
1316
            f"{usage_msg}"
1317
        )
Liangsheng Yin's avatar
Liangsheng Yin committed
1318
1319
1320
1321

        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            f += f"#unbootstrapped-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, "
            f += f"#queue-req: {len(self.waiting_queue)}, "
fzyzcjy's avatar
fzyzcjy committed
1322
            f += f"#transferring-req: {len(self.disagg_prefill_inflight_queue)}, "
1323
            f += f"input throughput (token/s): {self.last_input_throughput:.2f}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1324
        else:
Liangsheng Yin's avatar
Liangsheng Yin committed
1325
            f += f"#running-req: {running_bs}, "
1326
1327
1328
            f += f"#queue-req: {len(self.waiting_queue)}, "

        f += f"timestamp: {datetime.datetime.now().isoformat()}"
Liangsheng Yin's avatar
Liangsheng Yin committed
1329

1330
        logger.info(f)
1331
1332

        if self.enable_metrics:
1333
1334
1335
            cache_hit_rate = adder.log_hit_tokens / (
                adder.log_input_tokens + adder.log_hit_tokens
            )
1336
1337
1338
            self.stats.num_running_reqs = running_bs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = round(num_used / self.max_total_num_tokens, 2)
1339
1340
            self.stats.num_queue_reqs = len(self.waiting_queue)
            self.stats.cache_hit_rate = cache_hit_rate
1341
1342
1343
1344
1345
1346

            total_queue_latency = 0
            for req in can_run_list:
                total_queue_latency += req.queue_time_end - req.queue_time_start
            self.stats.avg_request_queue_latency = total_queue_latency / num_new_seq

1347
            self.metrics_collector.log_stats(self.stats)
1348
            self._emit_kv_metrics()
1349
        self._publish_kv_events()
1350

1351
1352
1353
    def log_decode_stats(
        self, can_run_cuda_graph: bool, running_batch: ScheduleBatch = None
    ):
1354
1355
        batch = running_batch or self.running_batch

1356
1357
        gap_latency = time.perf_counter() - self.last_decode_stats_tic
        self.last_decode_stats_tic = time.perf_counter()
1358
1359
        self.last_gen_throughput = self.num_generated_tokens / gap_latency
        self.num_generated_tokens = 0
1360
        num_running_reqs = len(batch.reqs)
tarinkk's avatar
tarinkk committed
1361
1362
        usage_msg, num_used = self.token_to_kv_pool_allocator.log_usage(
            self.tree_cache.evictable_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1363
        )
1364
1365
1366
1367
1368

        if RECORD_STEP_TIME:
            self.step_time_dict[num_running_reqs].append(
                gap_latency / self.server_args.decode_log_interval
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1369

tarinkk's avatar
tarinkk committed
1370
        msg = f"Decode batch. " f"#running-req: {num_running_reqs}, " f"{usage_msg}"
Liangsheng Yin's avatar
Liangsheng Yin committed
1371

1372
        if self.spec_algorithm.is_none():
1373
            spec_accept_length = 0
1374
        else:
1375
            spec_accept_length = (
1376
1377
                self.spec_num_total_accepted_tokens / self.spec_num_total_forward_ct
            )
1378
1379
            self.cum_spec_accept_length += self.spec_num_total_accepted_tokens
            self.cum_spec_accept_count += self.spec_num_total_forward_ct
1380
            self.spec_num_total_accepted_tokens = self.spec_num_total_forward_ct = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
1381
1382
1383
            msg += f"accept len: {spec_accept_length:.2f}, "

        if self.disaggregation_mode == DisaggregationMode.DECODE:
1384
            msg += f"pre-allocated usage: {self.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.max_total_num_tokens:.2f}, "
1385
            msg += f"#retracted-req: {len(self.disagg_decode_prealloc_queue.retracted_queue)}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1386
1387

        msg += (
1388
            f"cuda graph: {can_run_cuda_graph}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1389
            f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
1390
1391
            f"#queue-req: {len(self.waiting_queue)}, "
            f"timestamp: {datetime.datetime.now().isoformat()}"
Liangsheng Yin's avatar
Liangsheng Yin committed
1392
        )
1393
1394

        logger.info(msg)
1395
1396
1397
1398
        if self.enable_metrics:
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = num_used / self.max_total_num_tokens
1399
1400
            self.stats.cache_hit_rate = 0.0
            self.stats.gen_throughput = self.last_gen_throughput
1401
            self.stats.num_queue_reqs = len(self.waiting_queue)
1402
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1403
            self.stats.spec_accept_length = spec_accept_length
1404
            self.metrics_collector.log_stats(self.stats)
1405
            self._emit_kv_metrics()
1406
        self._publish_kv_events()
1407

Lianmin Zheng's avatar
Lianmin Zheng committed
1408
    def check_memory(self):
tarinkk's avatar
tarinkk committed
1409
1410
1411
1412
1413
        if isinstance(self.token_to_kv_pool_allocator, SWATokenToKVPoolAllocator):
            available_token_size = self.token_to_kv_pool_allocator.full_available_size()
        else:
            available_token_size = self.token_to_kv_pool_allocator.available_size()
        available_size = available_token_size + self.tree_cache.evictable_size()
1414
1415
1416
1417
1418
1419
1420
        protected_size = self.tree_cache.protected_size()
        memory_leak = available_size != (
            self.max_total_num_tokens
            if not self.enable_hierarchical_cache
            else self.max_total_num_tokens - protected_size
        )
        if memory_leak:
1421
            msg = (
1422
                "token_to_kv_pool_allocator memory leak detected! "
1423
                f"{available_size=}, {protected_size=}, {self.max_total_num_tokens=}\n"
tarinkk's avatar
tarinkk committed
1424
                f"{available_token_size=}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1425
                f"{self.tree_cache.evictable_size()=}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1426
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1427
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1428

1429
1430
1431
1432
1433
1434
1435
1436
        if self.disaggregation_mode == DisaggregationMode.DECODE:
            req_total_size = (
                self.req_to_token_pool.size + self.req_to_token_pool.pre_alloc_size
            )
        else:
            req_total_size = self.req_to_token_pool.size

        if len(self.req_to_token_pool.free_slots) != req_total_size:
1437
            msg = (
1438
                "req_to_token_pool memory leak detected!"
1439
1440
                f"available_size={len(self.req_to_token_pool.free_slots)}, "
                f"total_size={self.req_to_token_pool.size}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1441
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1442
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1443

1444
1445
1446
        if (
            self.enable_metrics
            and self.attn_tp_rank == 0
1447
            and time.perf_counter() > self.metrics_collector.last_log_time + 30
1448
1449
1450
        ):
            # During idle time, also collect metrics every 30 seconds.
            num_used = self.max_total_num_tokens - (
1451
                self.token_to_kv_pool_allocator.available_size()
1452
1453
                + self.tree_cache.evictable_size()
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1454
            num_running_reqs = len(self.running_batch.reqs)
1455
1456
1457
1458
1459
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = num_used / self.max_total_num_tokens
            self.stats.gen_throughput = 0
            self.stats.num_queue_reqs = len(self.waiting_queue)
1460
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1461
            self.metrics_collector.log_stats(self.stats)
1462
        self._publish_kv_events()
1463

1464
    def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
1465
        # Merge the prefill batch into the running batch
1466
1467
1468
1469
1470
1471
1472
1473
        chunked_req_to_exclude = set()
        if self.chunked_req:
            # Move the chunked request out of the batch so that we can merge
            # only finished requests to running_batch.
            chunked_req_to_exclude.add(self.chunked_req)
            self.tree_cache.cache_unfinished_req(self.chunked_req)
            # chunked request keeps its rid but will get a new req_pool_idx
            self.req_to_token_pool.free(self.chunked_req.req_pool_idx)
Lianmin Zheng's avatar
Lianmin Zheng committed
1474
        if self.last_batch and self.last_batch.forward_mode.is_extend():
1475
1476
1477
1478
            if self.last_batch.chunked_req is not None:
                # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
                # We need to discard it.
                chunked_req_to_exclude.add(self.last_batch.chunked_req)
Lianmin Zheng's avatar
Lianmin Zheng committed
1479

1480
            # Filter batch
1481
            last_bs = self.last_batch.batch_size()
1482
1483
1484
            self.last_batch.filter_batch(
                chunked_req_to_exclude=list(chunked_req_to_exclude)
            )
1485
            if self.last_batch.batch_size() < last_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1486
                self.running_batch.batch_is_full = False
1487

1488
            # Merge the new batch into the running batch
1489
            if not self.last_batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1490
                if self.running_batch.is_empty():
1491
1492
                    self.running_batch = self.last_batch
                else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1493
                    # Merge running_batch with prefill batch
1494
                    self.running_batch.merge_batch(self.last_batch)
1495

1496
        new_batch = self.get_new_batch_prefill()
1497

1498
1499
1500
1501
1502
        need_dp_attn_preparation = require_mlp_sync(self.server_args)

        if need_dp_attn_preparation and not self.spec_algorithm.is_none():
            # In speculative decoding, prefill batches and decode batches cannot be processed in the same DP attention group.
            # We prepare idle batches in advance to skip preparing decode batches when there are prefill batches in the group.
1503
            new_batch = self.prepare_mlp_sync_batch(new_batch)
1504
1505
1506
            need_dp_attn_preparation = new_batch is None

        if new_batch is not None:
1507
1508
1509
1510
            # Run prefill first if possible
            ret = new_batch
        else:
            # Run decode
Lianmin Zheng's avatar
Lianmin Zheng committed
1511
            if not self.running_batch.is_empty():
1512
                self.running_batch = self.update_running_batch(self.running_batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1513
1514
1515
                ret = self.running_batch if not self.running_batch.is_empty() else None
            else:
                ret = None
1516

1517
1518
        # Handle DP attention
        if need_dp_attn_preparation:
1519
            ret = self.prepare_mlp_sync_batch(ret)
1520
1521

        return ret
1522

1523
1524
1525
1526
1527
1528
    def get_num_allocatable_reqs(self, running_bs):
        res = global_server_args_dict["max_micro_batch_size"] - running_bs
        if self.pp_size > 1:
            res = min(res, self.req_to_token_pool.available_size())
        return res

Lianmin Zheng's avatar
Lianmin Zheng committed
1529
    def get_new_batch_prefill(self) -> Optional[ScheduleBatch]:
Lianmin Zheng's avatar
Lianmin Zheng committed
1530
        # Check if the grammar is ready in the grammar queue
1531
        if self.grammar_queue:
1532
            self.move_ready_grammar_requests()
1533

Lianmin Zheng's avatar
Lianmin Zheng committed
1534
1535
        # Handle the cases where prefill is not allowed
        if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1536
            self.running_batch.batch_is_full or len(self.waiting_queue) == 0
1537
        ) and self.chunked_req is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
1538
1539
            return None

Lianmin Zheng's avatar
Lianmin Zheng committed
1540
        running_bs = len(self.running_batch.reqs)
1541
        # Ignore the check if self.chunked_req is not None.
1542
1543
1544
1545
1546
        # In the non-PP case, when self.chunked_req is not None, num_allocatable_reqs should always be greater than 0,
        # as the space for the chunked request has just been released.
        # In PP case, a chunked req can start in one microbatch and end in another microbatch, so the max_running_requests per microbatch should not be strict.
        # Instead, we should always allow chunked request to be added, otherwise, there will be a memory leak.
        if self.get_num_allocatable_reqs(running_bs) <= 0 and not self.chunked_req:
Lianmin Zheng's avatar
Lianmin Zheng committed
1547
            self.running_batch.batch_is_full = True
1548
1549
            return None

1550
        if self.enable_hierarchical_cache:
1551
            self.tree_cache.check_hicache_events()
1552

1553
        # Get priority queue
1554
        self.policy.calc_priority(self.waiting_queue)
1555

Lianmin Zheng's avatar
Lianmin Zheng committed
1556
        # Prefill policy
1557
        adder = PrefillAdder(
1558
            self.page_size,
1559
            self.tree_cache,
1560
            self.token_to_kv_pool_allocator,
1561
1562
1563
1564
            self.running_batch,
            self.new_token_ratio,
            self.max_prefill_tokens,
            self.chunked_prefill_size,
1565
            running_bs if self.is_mixed_chunk else 0,
1566
1567
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1568
        if self.chunked_req is not None:
1569
1570
            self.chunked_req.init_next_round_input()
            self.chunked_req = adder.add_chunked_req(self.chunked_req)
1571

Lianmin Zheng's avatar
Lianmin Zheng committed
1572
        if self.lora_paths:
Lianmin Zheng's avatar
Lianmin Zheng committed
1573
1574
            lora_set = set([req.lora_path for req in self.running_batch.reqs])

1575
        # Get requests from the waiting queue to a new prefill batch
1576
1577
        for req in self.waiting_queue:
            if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1578
                self.lora_paths
1579
1580
1581
1582
1583
1584
1585
                and len(
                    lora_set
                    | set([req.lora_path for req in adder.can_run_list])
                    | set([req.lora_path])
                )
                > self.max_loras_per_batch
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1586
                self.running_batch.batch_is_full = True
1587
1588
                break

1589
            if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):
Lianmin Zheng's avatar
Lianmin Zheng committed
1590
                self.running_batch.batch_is_full = True
1591
                break
1592

Byron Hsu's avatar
Byron Hsu committed
1593
1594
1595
1596
1597
1598
1599
            if self.disaggregation_mode == DisaggregationMode.PREFILL:
                # In prefill mode, prealloc queue and transfer queue can also take memory,
                # so we need to check if the available size for the actual available size.
                if len(adder.can_run_list) >= self.req_to_token_pool.available_size():
                    self.running_batch.batch_is_full = True
                    break

1600
1601
            req.init_next_round_input(self.tree_cache)
            res = adder.add_one_req(req, has_chunked_req=(self.chunked_req is not None))
1602

1603
1604
            if res != AddReqResult.CONTINUE:
                if res == AddReqResult.NO_TOKEN:
1605
1606
                    if self.enable_hierarchical_cache:
                        # Set batch_is_full after making sure there are requests that can be served
Lianmin Zheng's avatar
Lianmin Zheng committed
1607
1608
                        self.running_batch.batch_is_full = len(
                            adder.can_run_list
1609
                        ) > 0 or (not self.running_batch.is_empty())
1610
                    else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1611
                        self.running_batch.batch_is_full = True
1612
1613
                break

Lianmin Zheng's avatar
Lianmin Zheng committed
1614
        # Update waiting queue
1615
        can_run_list: List[Req] = adder.can_run_list
Lianmin Zheng's avatar
Lianmin Zheng committed
1616
1617
        if len(can_run_list) == 0:
            return None
1618
1619
1620
1621

        if self.enable_metrics:
            # only record queue time when enable_metrics is True to avoid overhead
            for req in can_run_list:
1622
                req.queue_time_end = time.perf_counter()
1623

Lianmin Zheng's avatar
Lianmin Zheng committed
1624
1625
1626
        self.waiting_queue = [
            x for x in self.waiting_queue if x not in set(can_run_list)
        ]
1627

1628
1629
1630
        if adder.new_chunked_req is not None:
            assert self.chunked_req is None
            self.chunked_req = adder.new_chunked_req
1631

1632
1633
        if self.chunked_req:
            self.chunked_req.is_chunked += 1
Lianmin Zheng's avatar
Lianmin Zheng committed
1634

1635
        # Print stats
1636
        if self.attn_tp_rank == 0:
1637
            self.log_prefill_stats(adder, can_run_list, running_bs)
1638

Lianmin Zheng's avatar
Lianmin Zheng committed
1639
        # Create a new batch
1640
1641
1642
        new_batch = ScheduleBatch.init_new(
            can_run_list,
            self.req_to_token_pool,
1643
            self.token_to_kv_pool_allocator,
1644
            self.tree_cache,
1645
            self.model_config,
1646
            self.enable_overlap,
1647
            self.spec_algorithm,
1648
            self.server_args.enable_custom_logit_processor,
1649
            chunked_req=self.chunked_req,
1650
        )
1651
1652
        if self.enable_hierarchical_cache:
            # todo (zhiqiang): disable cuda graph execution if hicache loading triggered
1653
1654
1655
            new_batch.hicache_consumer_index = (
                self.tree_cache.ready_to_load_host_cache()
            )
1656

1657
        new_batch.prepare_for_extend()
1658

Lianmin Zheng's avatar
Lianmin Zheng committed
1659
        # Mixed-style chunked prefill
1660
1661
        if (
            self.is_mixed_chunk
Lianmin Zheng's avatar
Lianmin Zheng committed
1662
            and not self.running_batch.is_empty()
1663
1664
1665
            and not (new_batch.return_logprob or self.running_batch.return_logprob)
        ):
            # TODO (lianmin): support return_logprob + mixed chunked prefill
1666
1667
            self.running_batch.filter_batch()
            if not self.running_batch.is_empty():
1668
                self.running_batch.prepare_for_decode()
1669
1670
                new_batch.mix_with_running(self.running_batch)
                new_batch.decoding_reqs = self.running_batch.reqs
Lianmin Zheng's avatar
Lianmin Zheng committed
1671
1672
1673
            self.running_batch = ScheduleBatch(
                reqs=[], batch_is_full=self.running_batch.batch_is_full
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1674
1675
        else:
            new_batch.decoding_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1676
1677
1678

        return new_batch

Lianmin Zheng's avatar
Lianmin Zheng committed
1679
    def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
1680
        """Update the current running decoding batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1681
        initial_bs = batch.batch_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1682

1683
1684
        batch.filter_batch()
        if batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1685
1686
            batch.batch_is_full = False
            return batch
1687

Lianmin Zheng's avatar
Lianmin Zheng committed
1688
        # Check if decode out of memory
1689
        if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or (
1690
            TEST_RETRACT and batch.batch_size() > 10
1691
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1692
1693
            old_ratio = self.new_token_ratio

1694
            retracted_reqs, new_token_ratio = batch.retract_decode(self.server_args)
Lianmin Zheng's avatar
Lianmin Zheng committed
1695
            self.new_token_ratio = new_token_ratio
1696

Lianmin Zheng's avatar
Lianmin Zheng committed
1697
            logger.info(
1698
                "KV cache pool is full. Retract requests. "
Lianmin Zheng's avatar
Lianmin Zheng committed
1699
1700
1701
                f"#retracted_reqs: {len(retracted_reqs)}, "
                f"#new_token_ratio: {old_ratio:.4f} -> {self.new_token_ratio:.4f}"
            )
1702
            self._extend_requests_to_queue(retracted_reqs, is_retracted=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
1703
1704
        else:
            self.new_token_ratio = max(
1705
                self.new_token_ratio - self.new_token_ratio_decay,
Lianmin Zheng's avatar
Lianmin Zheng committed
1706
1707
1708
                self.min_new_token_ratio,
            )

Lianmin Zheng's avatar
Lianmin Zheng committed
1709
        if batch.batch_size() < initial_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1710
            batch.batch_is_full = False
Lianmin Zheng's avatar
Lianmin Zheng committed
1711
1712

        # Update batch tensors
1713
        batch.prepare_for_decode()
Lianmin Zheng's avatar
Lianmin Zheng committed
1714
        return batch
Lianmin Zheng's avatar
Lianmin Zheng committed
1715

1716
1717
1718
    def run_batch(
        self, batch: ScheduleBatch
    ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
1719
        """Run a batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1720
1721
        self.forward_ct += 1

1722
1723
        # Whether to run the profiler
        self._profile_batch_predicate(batch)
1724
1725
1726
1727
        if self.forward_sleep_time is not None:
            logger.info(f"Scheduler.run_batch sleep {self.forward_sleep_time}s")
            time.sleep(self.forward_sleep_time)

1728
        # Run forward
1729
        if self.is_generation:
1730
1731
            if self.spec_algorithm.is_none():
                model_worker_batch = batch.get_model_worker_batch()
1732
1733
1734
1735
1736

                # update the consumer index of hicache to the running batch
                self.tp_worker.set_hicache_consumer(
                    model_worker_batch.hicache_consumer_index
                )
1737
                if self.pp_group.is_last_rank:
1738
                    logits_output, next_token_ids, can_run_cuda_graph = (
1739
1740
1741
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
                else:
1742
                    pp_hidden_states_proxy_tensors, _, can_run_cuda_graph = (
1743
1744
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
1745
                bid = model_worker_batch.bid
Lianmin Zheng's avatar
Lianmin Zheng committed
1746
            else:
1747
1748
1749
                (
                    logits_output,
                    next_token_ids,
1750
                    bid,
1751
                    num_accepted_tokens,
1752
                    can_run_cuda_graph,
1753
                ) = self.draft_worker.forward_batch_speculative_generation(batch)
1754
1755
1756
                bs = batch.batch_size()
                self.spec_num_total_accepted_tokens += num_accepted_tokens + bs
                self.spec_num_total_forward_ct += bs
1757
                self.num_generated_tokens += num_accepted_tokens
1758
1759
1760

            if self.pp_group.is_last_rank:
                batch.output_ids = next_token_ids
1761

1762
1763
1764
            # These 2 values are needed for processing the output, but the values can be
            # modified by overlap schedule. So we have to copy them here so that
            # we can use the correct values in output processing.
1765
            if batch.return_logprob or self.spec_algorithm.is_eagle():
1766
                extend_input_len_per_req = [req.extend_input_len for req in batch.reqs]
1767
1768
1769
            else:
                extend_input_len_per_req = None
            if batch.return_logprob:
1770
1771
1772
1773
1774
1775
                extend_logprob_start_len_per_req = [
                    req.extend_logprob_start_len for req in batch.reqs
                ]
            else:
                extend_logprob_start_len_per_req = None

1776
            ret = GenerationBatchResult(
1777
1778
1779
1780
1781
1782
1783
                logits_output=logits_output if self.pp_group.is_last_rank else None,
                pp_hidden_states_proxy_tensors=(
                    pp_hidden_states_proxy_tensors
                    if not self.pp_group.is_last_rank
                    else None
                ),
                next_token_ids=next_token_ids if self.pp_group.is_last_rank else None,
1784
1785
                extend_input_len_per_req=extend_input_len_per_req,
                extend_logprob_start_len_per_req=extend_logprob_start_len_per_req,
1786
                bid=bid,
1787
                can_run_cuda_graph=can_run_cuda_graph,
1788
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1789
1790
1791
        else:  # embedding or reward model
            model_worker_batch = batch.get_model_worker_batch()
            embeddings = self.tp_worker.forward_batch_embedding(model_worker_batch)
1792
1793
1794
            ret = EmbeddingBatchResult(
                embeddings=embeddings, bid=model_worker_batch.bid
            )
1795
        return ret
Chayenne's avatar
Chayenne committed
1796

1797
1798
1799
1800
    def process_batch_result(
        self,
        batch: ScheduleBatch,
        result: Union[GenerationBatchResult, EmbeddingBatchResult],
1801
        launch_done: Optional[threading.Event] = None,
1802
    ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1803
        if batch.forward_mode.is_decode():
1804
            self.process_batch_result_decode(batch, result, launch_done)
1805
        elif batch.forward_mode.is_extend():
1806
            self.process_batch_result_prefill(batch, result, launch_done)
1807
1808
        elif batch.forward_mode.is_idle():
            if self.enable_overlap:
1809
                self.tp_worker.resolve_last_batch_result(launch_done)
1810
                self.set_next_batch_sampling_info_done(batch)
1811
        elif batch.forward_mode.is_dummy_first():
1812
            self.set_next_batch_sampling_info_done(batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1813

1814
1815
1816
1817
1818
1819
1820
        if self.return_health_check_ct:
            # Return some signal for the health check.
            # This is used to prevent the health check signal being blocked by long context prefill.
            # However, one minor issue is that this code path does not check the status of detokenizer manager.
            self.return_health_check_ct -= 1
            self.send_to_tokenizer.send_pyobj(HealthCheckOutput())

1821
1822
    def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
        return self.prepare_mlp_sync_batch_raw(
1823
1824
1825
1826
1827
1828
1829
1830
            local_batch,
            dp_size=self.server_args.dp_size,
            attn_tp_size=self.attn_tp_size,
            tp_cpu_group=self.tp_cpu_group,
            get_idle_batch=self.get_idle_batch,
            disable_cuda_graph=self.server_args.disable_cuda_graph,
            spec_algorithm=self.spec_algorithm,
            speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens,
1831
1832
1833
            enable_two_batch_overlap=self.server_args.enable_two_batch_overlap,
            enable_deepep_moe=self.server_args.enable_deepep_moe,
            deepep_mode=DeepEPMode[self.server_args.deepep_mode],
1834
            require_mlp_tp_gather=require_mlp_tp_gather(self.server_args),
1835
1836
1837
        )

    @staticmethod
1838
    def prepare_mlp_sync_batch_raw(
1839
1840
1841
1842
1843
1844
1845
1846
        local_batch: ScheduleBatch,
        dp_size,
        attn_tp_size: int,
        tp_cpu_group,
        get_idle_batch,
        disable_cuda_graph: bool,
        spec_algorithm,
        speculative_num_draft_tokens,
1847
1848
1849
        enable_two_batch_overlap: bool,
        enable_deepep_moe: bool,
        deepep_mode: DeepEPMode,
1850
        require_mlp_tp_gather: bool,
1851
    ):
1852
1853
1854
        # Check if other DP workers have running batches
        if local_batch is None:
            num_tokens = 0
1855
            num_tokens_for_logprob = 0
1856
1857
        elif local_batch.forward_mode.is_decode():
            num_tokens = local_batch.batch_size()
1858
            num_tokens_for_logprob = num_tokens
1859
1860
        else:
            num_tokens = local_batch.extend_num_tokens
1861
            num_tokens_for_logprob = sum(
Lianmin Zheng's avatar
Lianmin Zheng committed
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
                [
                    # We should have at least 1 token for sample in every case.
                    max(extend_len - logprob_start_len, 1)
                    for logprob_start_len, extend_len in zip(
                        local_batch.extend_logprob_start_lens, local_batch.extend_lens
                    )
                ]
            )

        if local_batch is None or local_batch.forward_mode.is_decode_or_idle():
            can_cuda_graph = 1
        else:
            can_cuda_graph = 0

        is_extend_in_batch = (
            local_batch.forward_mode.is_extend() if local_batch else False
        )
1879
1880
1881

        tbo_preparer = TboDPAttentionPreparer()

Lianmin Zheng's avatar
Lianmin Zheng committed
1882
1883
1884
1885
        local_info = torch.tensor(
            [
                num_tokens,
                can_cuda_graph,
1886
                num_tokens_for_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1887
                is_extend_in_batch,
1888
1889
1890
1891
1892
1893
                *tbo_preparer.prepare_all_gather(
                    local_batch,
                    deepep_mode,
                    enable_deepep_moe,
                    enable_two_batch_overlap,
                ),
Lianmin Zheng's avatar
Lianmin Zheng committed
1894
1895
1896
1897
            ],
            dtype=torch.int64,
        )
        global_info = torch.empty(
1898
            (dp_size, attn_tp_size, 6),
Lianmin Zheng's avatar
Lianmin Zheng committed
1899
1900
            dtype=torch.int64,
        )
1901
        torch.distributed.all_gather_into_tensor(
Lianmin Zheng's avatar
Lianmin Zheng committed
1902
1903
            global_info.flatten(),
            local_info,
1904
            group=tp_cpu_group,
1905
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1906
1907
1908
1909
        global_num_tokens = global_info[:, 0, 0].tolist()
        can_cuda_graph = min(global_info[:, 0, 1].tolist())
        global_num_tokens_for_logprob = global_info[:, 0, 2].tolist()
        is_extend_in_batch = global_info[:, 0, 3].tolist()
1910

1911
1912
1913
1914
        tbo_split_seq_index, global_forward_mode = tbo_preparer.compute_output(
            global_info[:, :, 4:6]
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1915
        if local_batch is None and max(global_num_tokens) > 0:
1916
            local_batch = get_idle_batch()
1917
1918

        if local_batch is not None:
1919
            # TODO: handle the case when moe_dense_tp_size != 1
1920
            if not require_mlp_tp_gather:
1921
1922
1923
1924
1925
1926
1927
                local_batch.global_num_tokens = [num_tokens]
                local_batch.global_num_tokens_for_logprob = [num_tokens_for_logprob]
            else:
                local_batch.global_num_tokens = global_num_tokens
                local_batch.global_num_tokens_for_logprob = (
                    global_num_tokens_for_logprob
                )
1928
            local_batch.is_extend_in_batch = any(is_extend_in_batch)
1929
1930
            local_batch.tbo_split_seq_index = tbo_split_seq_index
            local_batch.global_forward_mode = global_forward_mode
1931

1932
            # Check forward mode for cuda graph
1933
            if not disable_cuda_graph:
Lianmin Zheng's avatar
Lianmin Zheng committed
1934
                local_batch.can_run_dp_cuda_graph = can_cuda_graph
1935

1936
        return local_batch
1937
1938
1939
1940
1941

    def get_idle_batch(self):
        idle_batch = ScheduleBatch.init_new(
            [],
            self.req_to_token_pool,
1942
            self.token_to_kv_pool_allocator,
1943
1944
1945
            self.tree_cache,
            self.model_config,
            self.enable_overlap,
1946
            self.spec_algorithm,
1947
            self.server_args.enable_custom_logit_processor,
1948
1949
1950
1951
        )
        idle_batch.prepare_for_idle()
        return idle_batch

1952
1953
    def move_ready_grammar_requests(self):
        """Move requests whose grammar objects are ready from grammar_queue to waiting_queue."""
1954

1955
        num_ready_reqs = 0
1956
        num_timeout_reqs = 0
1957
1958
        for req in self.grammar_queue:
            try:
1959
1960
1961
                if req.finished():  # It is aborted by AbortReq
                    num_ready_reqs += 1
                    continue
1962
                req.grammar = req.grammar.result(timeout=0.03)
1963
1964
1965
1966
1967
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
1968
1969
                num_ready_reqs += 1
            except futures._base.TimeoutError:
1970
                req.grammar_wait_ct += 1
1971
1972
                # NOTE(lianmin): this timeout is the waiting time of the above line. It is
                # not the waiting time from it enters the grammar queue.
1973
                if req.grammar_wait_ct > GRAMMAR_TIMEOUT / 0.03:
1974
                    num_timeout_reqs = 1
1975
1976
                break

1977
        if self.server_args.enable_dp_attention:
1978
1979
            tp_size = self.attn_tp_size
            tp_group = self.attn_tp_cpu_group
1980
        else:
1981
1982
1983
1984
1985
            tp_size = self.tp_size
            tp_group = self.tp_cpu_group

        if tp_size > 1:
            # Sync across TP ranks to make sure they have the same number of ready requests
1986
            tensor = torch.tensor([num_ready_reqs, num_timeout_reqs], dtype=torch.int32)
1987
1988
1989
            torch.distributed.all_reduce(
                tensor, op=torch.distributed.ReduceOp.MAX, group=tp_group
            )
1990
            num_ready_reqs_max, num_timeout_reqs_max = tensor.tolist()
1991

1992
            for i in range(num_ready_reqs, num_ready_reqs_max):
1993
                req = self.grammar_queue[i]
1994
1995
                if req.finished():  # It is aborted by AbortReq
                    continue
1996
                req.grammar = req.grammar.result()
1997
1998
1999
2000
2001
2002
2003
2004
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
        else:
            num_ready_reqs_max = num_ready_reqs
            num_timeout_reqs_max = num_timeout_reqs
2005

2006
2007
2008
2009
2010
2011
2012
        for i in range(num_ready_reqs, num_ready_reqs + num_timeout_reqs_max):
            req = self.grammar_queue[i]
            req.grammar.cancel()
            error_msg = f"Grammar preprocessing timed out for {req.grammar_key=}"
            req.set_finish_with_abort(error_msg)
            self.grammar_backend.set_cache(req.grammar_key, INVALID_GRAMMAR_OBJ)
        num_ready_reqs = num_ready_reqs_max + num_timeout_reqs_max
2013

2014
        self._extend_requests_to_queue(self.grammar_queue[:num_ready_reqs])
2015
2016
        self.grammar_queue = self.grammar_queue[num_ready_reqs:]

2017
2018
2019
2020
2021
2022
2023
    def set_next_batch_sampling_info_done(self, batch: ScheduleBatch):
        if batch.next_batch_sampling_info:
            if batch.next_batch_sampling_info.grammars is not None:
                batch.next_batch_sampling_info.update_regex_vocab_mask()
                self.current_stream.synchronize()
            batch.next_batch_sampling_info.sampling_info_done.set()

2024
2025
2026
    def watchdog_thread(self):
        """A watch dog thread that will try to kill the server itself if one forward batch takes too long."""
        self.watchdog_last_forward_ct = 0
2027
        self.watchdog_last_time = time.perf_counter()
2028
2029

        while True:
2030
            current = time.perf_counter()
2031
2032
2033
2034
2035
2036
2037
2038
2039
            if self.cur_batch is not None:
                if self.watchdog_last_forward_ct == self.forward_ct:
                    if current > self.watchdog_last_time + self.watchdog_timeout:
                        break
                else:
                    self.watchdog_last_forward_ct = self.forward_ct
                    self.watchdog_last_time = current
            time.sleep(self.watchdog_timeout // 2)

Lianmin Zheng's avatar
Lianmin Zheng committed
2040
2041
2042
2043
2044
2045
2046
2047
2048
        if not disable_request_logging():
            # Print batch size and memory pool info to check whether there are de-sync issues.
            logger.error(
                f"{self.cur_batch.batch_size()=}, "
                f"{self.cur_batch.reqs=}, "
                f"{self.token_to_kv_pool_allocator.available_size()=}, "
                f"{self.tree_cache.evictable_size()=}, "
            )

2049
        pyspy_dump_schedulers()
Lianmin Zheng's avatar
Lianmin Zheng committed
2050
        logger.error(f"Watchdog timeout ({self.watchdog_timeout=})")
2051
2052
        print(file=sys.stderr, flush=True)
        print(file=sys.stdout, flush=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
2053
2054

        # Wait for some time so that the parent process can print the error.
2055
2056
2057
        time.sleep(5)
        self.parent_process.send_signal(signal.SIGQUIT)

2058
2059
2060
    def flush_cache_wrapped(self, recv_req: FlushCacheReqInput):
        success = self.flush_cache()
        return FlushCacheReqOutput(success=success)
2061

2062
    def flush_cache(self):
2063
        """Flush the memory pool and cache."""
2064
2065
2066
2067
2068
        if (
            len(self.waiting_queue) == 0
            and self.running_batch.is_empty()
            and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
        ):
2069
2070
            self.cur_batch = None
            self.last_batch = None
2071
            self.tree_cache.reset()
2072
            if self.grammar_backend:
Lianmin Zheng's avatar
Lianmin Zheng committed
2073
                self.grammar_backend.reset()
2074
            self.req_to_token_pool.clear()
2075
            self.token_to_kv_pool_allocator.clear()
2076
2077
2078

            if not self.spec_algorithm.is_none():
                self.draft_worker.model_runner.req_to_token_pool.clear()
2079
                self.draft_worker.model_runner.token_to_kv_pool_allocator.clear()
2080
2081
2082
2083
2084

            self.num_generated_tokens = 0
            self.forward_ct_decode = 0
            self.spec_num_total_accepted_tokens = 0
            self.spec_num_total_forward_ct = 0
2085
2086
            self.cum_spec_accept_length = 0
            self.cum_spec_accept_count = 0
2087
2088
2089
2090
2091
2092
2093
            torch.cuda.empty_cache()
            logger.info("Cache flushed successfully!")
            if_success = True
        else:
            logging.warning(
                f"Cache not flushed because there are pending requests. "
                f"#queue-req: {len(self.waiting_queue)}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2094
                f"#running-req: {len(self.running_batch.reqs)}"
2095
2096
2097
2098
            )
            if_success = False
        return if_success

Liangsheng Yin's avatar
Liangsheng Yin committed
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
    def get_load(self):
        # TODO(lsyin): use dynamically maintained num_waiting_tokens
        load = (
            self.max_total_num_tokens
            - self.token_to_kv_pool_allocator.available_size()
            - self.tree_cache.evictable_size()
        )
        load += sum(len(req.origin_input_ids) for req in self.waiting_queue)
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            load += sum(
                len(req.origin_input_ids)
                for req in self.disagg_prefill_bootstrap_queue.queue
            )
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            load += sum(
                len(req.req.origin_input_ids)
                for req in self.disagg_decode_prealloc_queue.queue
            )

        return load

2120
2121
2122
    def get_internal_state(self, recv_req: GetInternalStateReq):
        ret = dict(global_server_args_dict)
        ret["last_gen_throughput"] = self.last_gen_throughput
2123
2124
2125
2126
2127
2128
2129
2130
2131
        ret["memory_usage"] = {
            "weight": round(
                self.tp_worker.worker.model_runner.weight_load_mem_usage, 2
            ),
            "kvcache": round(
                self.token_to_kv_pool_allocator.get_kvcache().mem_usage, 2
            ),
            "token_capacity": int(self.max_total_num_tokens),
        }
2132
2133
2134
2135
2136
2137

        if not _is_cpu:
            ret["memory_usage"]["cuda_graph"] = round(
                self.tp_worker.worker.model_runner.cuda_graph_mem_usage, 2
            )

2138
2139
2140
2141
2142
2143
        if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
            ret["avg_spec_accept_length"] = (
                self.cum_spec_accept_length / self.cum_spec_accept_count
            )
        if RECORD_STEP_TIME:
            ret["step_time_dict"] = self.step_time_dict
Liangsheng Yin's avatar
Liangsheng Yin committed
2144
2145
2146
2147

        ret["load"] = self.get_load()

        return GetInternalStateReqOutput(internal_state=ret)
2148
2149
2150
2151
2152

    def set_internal_state(self, recv_req: SetInternalStateReq):
        server_args_dict = recv_req.server_args
        args_allow_update = set(
            [
2153
                "max_micro_batch_size",
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
                "speculative_accept_threshold_single",
                "speculative_accept_threshold_acc",
            ]
        )
        if_success = True
        for k, v in server_args_dict.items():
            if k not in args_allow_update:
                logging.warning(f"Updating {k} is not supported.")
                if_success = False
                break
2164
2165
2166
2167
2168
2169
2170
2171
            elif k == "max_micro_batch_size" and (
                v > self.max_running_requests // self.pp_size or v < 1
            ):
                logging.warning(
                    f"Updating {k} to {v} is rejected because it is out of the valid range [1, {self.max_running_requests // self.pp_size}]."
                )
                if_success = False
                break
2172
2173
2174
2175
2176
2177
2178
2179
2180
        if if_success:
            if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
                avg_spec_accept_length = (
                    self.cum_spec_accept_length / self.cum_spec_accept_count
                )
                logger.info(f"{avg_spec_accept_length=}")
            self.cum_spec_accept_length = self.cum_spec_accept_count = 0
            for k, v in server_args_dict.items():
                global_server_args_dict[k] = v
2181
            logger.info(f"Global server args updated! {global_server_args_dict=}")
2182
2183
2184
2185
2186
        return SetInternalStateReqOutput(
            updated=True,
            server_args=global_server_args_dict,
        )

2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
    def handle_rpc_request(self, recv_req: RpcReqInput):
        # Handle RPC requests
        logger.info(
            f"handle_rpc_request: {recv_req.method}, param: {recv_req.parameters}"
        )

        success = True
        exec = None
        try:
            func = getattr(self, recv_req.method)
            func(recv_req.parameters)
        except Exception as e:
            success = False
            exec = e
            logger.error(f"Failed to call rpc {recv_req.method}: {str(e)}")

        barrier()
        return RpcReqOutput(success, "" if not exec else str(exec))

    def save_remote_model(self, params):
        url = params["url"]

2209
        worker = self.tp_worker.worker
2210
2211
2212
2213

        worker.model_runner.save_remote_model(url)

    def save_sharded_model(self, params):
2214
        worker = self.tp_worker.worker
2215
2216
2217
2218
2219
2220
2221

        worker.model_runner.save_sharded_model(
            path=params["path"],
            pattern=params["pattern"],
            max_size=params["max_size"],
        )

2222
2223
    def abort_request(self, recv_req: AbortReq):
        # Delete requests in the waiting queue
Lianmin Zheng's avatar
Lianmin Zheng committed
2224
        to_del = []
2225
        for i, req in enumerate(self.waiting_queue):
2226
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
Lianmin Zheng's avatar
Lianmin Zheng committed
2227
                to_del.append(i)
2228

Lianmin Zheng's avatar
Lianmin Zheng committed
2229
        # Sort in reverse order to avoid index issues when deleting
Lianmin Zheng's avatar
Lianmin Zheng committed
2230
        for i in reversed(to_del):
2231
2232
2233
            # Abort method 1: directly pop from the queue
            # This only works for requests that have not started anything.
            # We still need to send something back to TokenizerManager to clean up the state.
Lianmin Zheng's avatar
Lianmin Zheng committed
2234
            req = self.waiting_queue.pop(i)
Lianmin Zheng's avatar
Lianmin Zheng committed
2235
            self.send_to_tokenizer.send_pyobj(AbortReq(req.rid))
2236
            logger.debug(f"Abort queued request. {req.rid=}")
2237

2238
2239
2240
2241
2242
        # Delete the requests in the grammar queue
        for req in self.grammar_queue:
            # Abort method 2: call `set_finish_with_abort`
            # The request will still run one prefill forward pass.
            # In this case, we change the input_ids to be only one token to make this prefill cheap.
2243
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2244
                logger.debug(f"Abort grammar queue request. {req.rid=}")
2245
2246
                if req.grammar:
                    req.grammar.cancel()
2247
2248
                req.set_finish_with_abort("Aborted by AbortReq.")

2249
        # Delete requests in the running batch
Lianmin Zheng's avatar
Lianmin Zheng committed
2250
2251
2252
2253
2254
2255
        if self.cur_batch is self.running_batch or self.cur_batch is None:
            reqs = self.running_batch.reqs
        else:
            reqs = self.running_batch.reqs + self.cur_batch.reqs

        for req in reqs:
2256
2257
2258
            if not req.finished() and (
                recv_req.abort_all or req.rid.startswith(recv_req.rid)
            ):
2259
2260
2261
                # Abort method 3: set `to_abort=True`
                # The request will still run one decode forward pass.
                # Then we reuse all existing code to clean up the KV cache allocation.
Lianmin Zheng's avatar
Lianmin Zheng committed
2262
2263
                logger.debug(f"Abort running request. {req.rid=}")
                req.to_abort = True
2264

2265
2266
2267
    def _pause_engine(self) -> Tuple[List[Req], int]:
        raise NotImplementedError()

Chayenne's avatar
Chayenne committed
2268
2269
2270
    def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
        """In-place update of the weights from disk."""
        success, message = self.tp_worker.update_weights_from_disk(recv_req)
2271
        if success:
Stefan He's avatar
Stefan He committed
2272
2273
            flush_cache_success = self.flush_cache()
            assert flush_cache_success, "Cache flush failed after updating weights"
2274
2275
        else:
            logger.error(message)
2276
        return UpdateWeightFromDiskReqOutput(success, message, 0)
2277

2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
    def load_lora_adapter(
        self, recv_req: LoadLoRAAdapterReqInput
    ) -> LoadLoRAAdapterReqOutput:
        """In-place loading a new lora adapter from disk or huggingface."""

        result = self.tp_worker.load_lora_adapter(recv_req)

        if result.success:
            flush_cache_success = self.flush_cache()
            assert flush_cache_success, "Cache flush failed after loading lora adapter."
        else:
            logger.error(result.error_message)
        return result

    def unload_lora_adapter(
        self, recv_req: UnloadLoRAAdapterReqInput
    ) -> UnloadLoRAAdapterReqOutput:
        """Unload the lora adapter."""

        result = self.tp_worker.unload_lora_adapter(recv_req)

        if result.success:
            flush_cache_success = self.flush_cache()
            assert (
                flush_cache_success
            ), "Cache flush failed after unloading LoRA weights"
        else:
            logger.error(result.error_message)
        return result

2308
2309
2310
    def init_weights_update_group(self, recv_req: InitWeightsUpdateGroupReqInput):
        """Initialize the online model parameter update group."""
        success, message = self.tp_worker.init_weights_update_group(recv_req)
2311
        return InitWeightsUpdateGroupReqOutput(success, message)
2312
2313

    def update_weights_from_distributed(
2314
2315
2316
        self,
        recv_req: UpdateWeightsFromDistributedReqInput,
    ) -> Tuple[bool, str]:
2317
2318
2319
        """Update the online model parameter."""
        success, message = self.tp_worker.update_weights_from_distributed(recv_req)
        if success:
2320
2321
2322
            if recv_req.flush_cache:
                flush_cache_success = self.flush_cache()
                assert flush_cache_success, "Cache flush failed after updating weights"
2323
2324
        else:
            logger.error(message)
2325
        return UpdateWeightsFromDistributedReqOutput(success, message)
2326

2327
2328
2329
2330
2331
    def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
        """Update the online model parameter from tensors."""
        success, message = self.tp_worker.update_weights_from_tensor(recv_req)
        # TODO extract common code b/t update_weights_from_distributed and update_weights_from_tensor later
        if success:
2332
            if recv_req.flush_cache:
Stefan He's avatar
Stefan He committed
2333
2334
                flush_cache_success = self.flush_cache()
                assert flush_cache_success, "Cache flush failed after updating weights"
2335
2336
        else:
            logger.error(message)
2337
        barrier(group=self.tp_cpu_group)
2338
        return UpdateWeightsFromTensorReqOutput(success, message)
2339

2340
2341
    def get_weights_by_name(self, recv_req: GetWeightsByNameReqInput):
        parameter = self.tp_worker.get_weights_by_name(recv_req)
2342
        return GetWeightsByNameReqOutput(parameter)
2343

2344
    def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
2345
2346
        tags = recv_req.tags

2347
        if tags is None or len(tags) == 0:
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_KV_CACHE)
            self.flush_cache()

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.stashed_model_static_state = _export_static_state(
                self.tp_worker.worker.model_runner.model
            )
2358
            torch.distributed.barrier(self.tp_cpu_group)
2359
2360
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_WEIGHTS)

2361
        return ReleaseMemoryOccupationReqOutput()
2362

2363
    def resume_memory_occupation(self, recv_req: ResumeMemoryOccupationReqInput):
2364
        tags = recv_req.tags
2365

2366
2367
2368
2369
2370
        if tags is None or len(tags) == 0:
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_WEIGHTS)
2371
            torch.distributed.barrier(self.tp_cpu_group)
2372
2373
2374
2375
2376
2377
2378
2379
2380
            _import_static_state(
                self.tp_worker.worker.model_runner.model,
                self.stashed_model_static_state,
            )
            del self.stashed_model_static_state

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_KV_CACHE)

2381
2382
        return ResumeMemoryOccupationReqOutput()

2383
2384
2385
2386
2387
2388
2389
    def slow_down(self, recv_req: SlowDownReqInput):
        t = recv_req.forward_sleep_time
        if t is not None and t <= 0:
            t = None
        self.forward_sleep_time = t
        return SlowDownReqOutput()

2390
    def profile(self, recv_req: ProfileReq):
2391
        if recv_req.type == ProfileReqType.START_PROFILE:
2392
2393
2394
2395
2396
2397
2398
2399
            if recv_req.profile_by_stage:
                return self.init_profile(
                    recv_req.output_dir,
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2400
                    recv_req.profile_id,
2401
2402
2403
2404
2405
2406
2407
2408
2409
                )
            else:
                self.init_profile(
                    recv_req.output_dir,
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2410
                    recv_req.profile_id,
2411
2412
                )
                return self.start_profile(True)
2413
        else:
2414
2415
            return self.stop_profile()

2416
    def init_profile(
2417
2418
2419
2420
        self,
        output_dir: Optional[str],
        num_steps: Optional[int],
        activities: Optional[List[str]],
2421
2422
        with_stack: Optional[bool],
        record_shapes: Optional[bool],
2423
        profile_by_stage: bool,
2424
        profile_id: str,
2425
2426
    ) -> ProfileReqOutput:
        if self.profile_in_progress:
2427
2428
2429
2430
2431
            return ProfileReqOutput(
                success=False,
                message="Profiling is already in progress. Call /stop_profile first.",
            )

2432
2433
        self.profile_by_stage = profile_by_stage

2434
2435
2436
2437
2438
2439
        if output_dir is None:
            output_dir = os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp")
        if activities is None:
            activities = ["CPU", "GPU"]

        self.torch_profiler_output_dir = output_dir
2440
2441
        self.torch_profiler_with_stack = with_stack
        self.torch_profiler_record_shapes = record_shapes
2442
        self.profiler_activities = activities
2443
        self.profile_id = profile_id
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463

        if num_steps:
            self.profile_steps = num_steps
            if self.profile_by_stage:
                self.profiler_target_prefill_ct = num_steps
                self.profiler_target_decode_ct = num_steps
                self.profiler_prefill_ct = 0
                self.profiler_decode_ct = 0
            else:
                self.profiler_target_forward_ct = self.forward_ct + num_steps
            # The caller will be notified when reaching profiler_target_forward_ct
        else:
            self.profiler_target_forward_ct = None

        return ProfileReqOutput(success=True, message="Succeeded")

    def start_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        stage_str = f" for {stage.__str__()}" if stage else ""
2464
        logger.info(
2465
            f"Profiling starts{stage_str}. Traces will be saved to: {self.torch_profiler_output_dir} (with profile id: {self.profile_id})",
2466
2467
        )

2468
2469
2470
2471
        activities = self.profiler_activities
        with_stack = self.torch_profiler_with_stack
        record_shapes = self.torch_profiler_record_shapes

2472
2473
2474
2475
2476
2477
2478
2479
        activity_map = {
            "CPU": torch.profiler.ProfilerActivity.CPU,
            "GPU": torch.profiler.ProfilerActivity.CUDA,
        }
        torchprof_activities = [
            activity_map[a] for a in activities if a in activity_map
        ]

2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
        if "RPD" in activities:
            from rpdTracerControl import rpdTracerControl

            rpdTracerControl.skipCreate()

            self.rpd_profile_path = os.path.join(
                self.torch_profiler_output_dir,
                "rpd-" + str(time.time()) + f"-TP-{self.tp_rank}" + ".trace.json.gz",
            )

            if self.tp_rank == 0:
                import sqlite3

                from rocpd.schema import RocpdSchema

                if os.path.exists("trace.rpd"):
                    os.unlink("trace.rpd")
                schema = RocpdSchema()
                connection = sqlite3.connect("trace.rpd")
                schema.writeSchema(connection)
                connection.commit()
                del connection
            torch.distributed.barrier(self.tp_cpu_group)

            self.rpd_profiler = rpdTracerControl()
            self.rpd_profiler.setPythonTrace(True)
            self.rpd_profiler.start()
            self.rpd_profiler.rangePush("", "rpd profile range", "")
            self.profile_in_progress = True
        elif torchprof_activities:
2510
2511
            self.torch_profiler = torch.profiler.profile(
                activities=torchprof_activities,
2512
2513
                with_stack=with_stack if with_stack is not None else True,
                record_shapes=record_shapes if record_shapes is not None else False,
2514
2515
            )
            self.torch_profiler.start()
2516
            self.profile_in_progress = True
2517
2518
2519

        if "MEM" in activities:
            torch.cuda.memory._record_memory_history(max_entries=100000)
2520
            self.profile_in_progress = True
2521

2522
2523
2524
        if "CUDA_PROFILER" in activities:
            torch.cuda.cudart().cudaProfilerStart()

2525
        return ProfileReqOutput(success=True, message="Succeeded")
2526

2527
2528
2529
2530
    def stop_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        if not self.profile_in_progress:
2531
2532
2533
2534
            return ProfileReqOutput(
                success=False,
                message="Profiling is not in progress. Call /start_profile first.",
            )
2535

2536
2537
2538
        if not Path(self.torch_profiler_output_dir).exists():
            Path(self.torch_profiler_output_dir).mkdir(parents=True, exist_ok=True)

2539
2540
        stage_suffix = f"-{stage.__str__()}" if stage else ""
        logger.info("Stop profiling" + stage_suffix + "...")
2541
2542
2543
2544
2545
        if self.torch_profiler is not None:
            self.torch_profiler.stop()
            self.torch_profiler.export_chrome_trace(
                os.path.join(
                    self.torch_profiler_output_dir,
2546
                    self.profile_id
2547
2548
2549
                    + f"-TP-{self.tp_rank}"
                    + stage_suffix
                    + ".trace.json.gz",
2550
2551
                )
            )
2552
2553
2554
2555
2556
2557
            torch.distributed.barrier(self.tp_cpu_group)

        if self.rpd_profiler is not None:
            self.rpd_profiler.rangePop()
            self.rpd_profiler.stop()
            self.rpd_profiler.flush()
2558

2559
2560
2561
2562
2563
2564
2565
2566
2567
            torch.distributed.barrier(self.tp_cpu_group)
            if self.tp_rank == 0:
                from sglang.srt.utils import rpd_to_chrome_trace

                rpd_to_chrome_trace("trace.rpd", self.rpd_profile_path)
            self.rpd_profiler = None
            self.rpd_profiler_path = None

        if self.profiler_activities is not None and "MEM" in self.profiler_activities:
2568
            memory_profile_path = os.path.join(
2569
                self.torch_profiler_output_dir,
2570
2571
2572
2573
                str(time.time())
                + f"-TP-{self.tp_rank}-memory"
                + stage_suffix
                + ".pickle",
2574
2575
2576
2577
            )
            torch.cuda.memory._dump_snapshot(memory_profile_path)
            torch.cuda.memory._record_memory_history(enabled=None)

2578
2579
2580
        if "CUDA_PROFILER" in self.profiler_activities:
            torch.cuda.cudart().cudaProfilerStop()

2581
2582
2583
        logger.info(
            "Profiling done. Traces are saved to: %s",
            self.torch_profiler_output_dir,
2584
        )
2585
        self.torch_profiler = None
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
        self.profile_in_progress = False

        return ProfileReqOutput(success=True, message="Succeeded.")

    def _profile_batch_predicate(self, batch):
        if self.profile_by_stage:
            if batch.forward_mode.is_prefill():
                if self.profiler_prefill_ct == 0:
                    self.start_profile(batch.forward_mode)
                self.profiler_prefill_ct += 1
                if self.profiler_prefill_ct > self.profiler_target_prefill_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.EXTEND)
            elif batch.forward_mode.is_decode():
                if self.profiler_decode_ct == 0:
                    if self.profile_in_progress:
                        # force trace flush
                        self.stop_profile(ForwardMode.EXTEND)
                    self.start_profile(batch.forward_mode)
                self.profiler_decode_ct += 1
                if self.profiler_decode_ct > self.profiler_target_decode_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.DECODE)
2609
2610
            elif batch.forward_mode.is_idle():
                pass
2611
            else:
2612
                raise RuntimeError(f"unsupported profile stage: {batch.forward_mode}")
2613
2614
2615
2616
2617
2618
2619
        else:
            # Check profiler
            if (
                self.profiler_target_forward_ct
                and self.profiler_target_forward_ct <= self.forward_ct
            ):
                self.stop_profile()
2620

2621
2622
    def expert_distribution_handle(self, recv_req: ExpertDistributionReq):
        if recv_req == ExpertDistributionReq.START_RECORD:
2623
            get_global_expert_distribution_recorder().start_record()
2624
        elif recv_req == ExpertDistributionReq.STOP_RECORD:
2625
            get_global_expert_distribution_recorder().stop_record()
2626
        elif recv_req == ExpertDistributionReq.DUMP_RECORD:
2627
            get_global_expert_distribution_recorder().dump_record()
2628
2629
        else:
            raise ValueError("Unrecognized ExpertDistributionReq value")
2630
        return ExpertDistributionReqOutput()
2631

2632
    def open_session(self, recv_req: OpenSessionReqInput):
2633
2634
2635
2636
        # handle error
        session_id = recv_req.session_id
        if session_id in self.sessions:
            logger.warning(f"session id {session_id} already exist, cannot open.")
2637
            return OpenSessionReqOutput(session_id, False)
2638
        elif session_id is None:
2639
            logger.warning("session id is None, cannot open.")
2640
            return OpenSessionReqOutput(session_id, False)
2641
2642
2643
2644
        else:
            self.sessions[session_id] = Session(
                recv_req.capacity_of_str_len, session_id
            )
2645
            return OpenSessionReqOutput(session_id, True)
2646
2647
2648
2649
2650
2651
2652
2653
2654

    def close_session(self, recv_req: CloseSessionReqInput):
        # handle error
        session_id = recv_req.session_id
        if session_id not in self.sessions:
            logger.warning(f"session id {session_id} does not exist, cannot delete.")
        else:
            del self.sessions[session_id]

2655
2656
    def get_print_prefix(self):
        prefix = ""
2657
2658
        if self.attn_dp_rank is not None:
            prefix += f" DP{self.attn_dp_rank}"
2659
2660
2661
2662
2663
2664
        if self.server_args.tp_size > 1:
            prefix += f" TP{self.tp_rank}"
        if self.pp_size > 1:
            prefix += f" PP{self.pp_rank}"
        return prefix

2665
2666
2667
2668
2669
2670
2671
    def _publish_kv_events(self):
        if self.enable_kv_cache_events:
            events = self.tree_cache.take_events()
            if events:
                batch = KVEventBatch(ts=time.time(), events=events)
                self.kv_event_publisher.publish(batch)

2672

2673
2674
2675
2676
def is_health_check_generate_req(recv_req):
    return getattr(recv_req, "rid", "").startswith("HEALTH_CHECK")


2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
def _export_static_state(model):
    return dict(
        buffers=[
            (name, buffer.detach().clone()) for name, buffer in model.named_buffers()
        ]
    )


def _import_static_state(model, static_params):
    self_named_buffers = dict(model.named_buffers())
    for name, tensor in static_params["buffers"]:
        self_named_buffers[name][...] = tensor


2691
2692
2693
2694
2695
def run_scheduler_process(
    server_args: ServerArgs,
    port_args: PortArgs,
    gpu_id: int,
    tp_rank: int,
2696
    pp_rank: int,
2697
    dp_rank: Optional[int],
2698
    pipe_writer,
2699
):
2700
    # Generate the prefix
2701
2702
2703
2704
2705
2706
2707
    prefix = ""
    if dp_rank is not None:
        prefix += f" DP{dp_rank}"
    if server_args.tp_size > 1:
        prefix += f" TP{tp_rank}"
    if server_args.pp_size > 1:
        prefix += f" PP{pp_rank}"
2708

2709
    # Config the process
2710
    kill_itself_when_parent_died()
2711
    setproctitle.setproctitle(f"sglang::scheduler{prefix.replace(' ', '_')}")
2712
    faulthandler.enable()
2713
    parent_process = psutil.Process().parent()
2714

2715
2716
2717
    # [For Router] if env var "SGLANG_DP_RANK" exist, set dp_rank to the value of the env var
    if dp_rank is None and "SGLANG_DP_RANK" in os.environ:
        dp_rank = int(os.environ["SGLANG_DP_RANK"])
2718

Wang Ran (汪然)'s avatar
Wang Ran (汪然) committed
2719
    # Configure the logger
2720
    configure_logger(server_args, prefix=prefix)
2721
    suppress_other_loggers()
2722

2723
    # Set cpu affinity to this gpu process
2724
2725
2726
    if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
        set_gpu_proc_affinity(server_args.tp_size, server_args.nnodes, gpu_id)

2727
2728
2729
2730
    embedding_cache_size = 100
    if "SGLANG_VLM_CACHE_SIZE_MB" in os.environ:
        embedding_cache_size = int(os.environ["SGLANG_VLM_CACHE_SIZE_MB"])
    init_embedding_cache(embedding_cache_size * 1024 * 1024)
2731
    # Create a scheduler and run the event loop
2732
    try:
2733
        scheduler = Scheduler(server_args, port_args, gpu_id, tp_rank, pp_rank, dp_rank)
2734
        pipe_writer.send(
Mick's avatar
Mick committed
2735
2736
2737
2738
2739
            {
                "status": "ready",
                "max_total_num_tokens": scheduler.max_total_num_tokens,
                "max_req_input_len": scheduler.max_req_input_len,
            }
2740
        )
Byron Hsu's avatar
Byron Hsu committed
2741
2742
2743
        disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode

        if disaggregation_mode == DisaggregationMode.NULL:
2744
2745
2746
            if server_args.pp_size > 1:
                scheduler.event_loop_pp()
            elif scheduler.enable_overlap:
Byron Hsu's avatar
Byron Hsu committed
2747
2748
2749
2750
                scheduler.event_loop_overlap()
            else:
                scheduler.event_loop_normal()
        elif disaggregation_mode == DisaggregationMode.PREFILL:
2751
2752
2753
2754
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_prefill()
            else:
                scheduler.event_loop_normal_disagg_prefill()
2755

Byron Hsu's avatar
Byron Hsu committed
2756
        elif disaggregation_mode == DisaggregationMode.DECODE:
2757
2758
2759
2760
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_decode()
            else:
                scheduler.event_loop_normal_disagg_decode()
Byron Hsu's avatar
Byron Hsu committed
2761

2762
    except Exception:
2763
2764
2765
        traceback = get_exception_traceback()
        logger.error(f"Scheduler hit an exception: {traceback}")
        parent_process.send_signal(signal.SIGQUIT)