scheduler.py 109 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
14
15
"""A scheduler that manages a tensor parallel GPU worker."""

16
import faulthandler
17
import logging
18
import os
19
import signal
20
import sys
Lianmin Zheng's avatar
Lianmin Zheng committed
21
import threading
22
import time
23
from collections import defaultdict, deque
Lianmin Zheng's avatar
Lianmin Zheng committed
24
from concurrent import futures
25
from dataclasses import dataclass
26
from pathlib import Path
27
from types import SimpleNamespace
28
from typing import Dict, List, Optional, Tuple, Union
29

30
import psutil
31
import setproctitle
32
import torch
33
import zmq
34
from torch.distributed import barrier
35

36
from sglang.global_config import global_config
Lianmin Zheng's avatar
Lianmin Zheng committed
37
from sglang.srt.configs.model_config import ModelConfig
38
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE, GPU_MEMORY_TYPE_WEIGHTS
39
40
41
42
from sglang.srt.constrained.base_grammar_backend import (
    INVALID_GRAMMAR_OBJ,
    create_grammar_backend,
)
Byron Hsu's avatar
Byron Hsu committed
43
44
45
46
47
from sglang.srt.disaggregation.decode import (
    DecodePreallocQueue,
    DecodeTransferQueue,
    SchedulerDisaggregationDecodeMixin,
)
48
from sglang.srt.disaggregation.kv_events import EventPublisherFactory, KVEventBatch
Byron Hsu's avatar
Byron Hsu committed
49
50
51
52
53
54
from sglang.srt.disaggregation.prefill import (
    PrefillBootstrapQueue,
    SchedulerDisaggregationPrefillMixin,
)
from sglang.srt.disaggregation.utils import (
    DisaggregationMode,
55
    MetadataBuffers,
Byron Hsu's avatar
Byron Hsu committed
56
    ReqToMetadataIdxAllocator,
57
    TransferBackend,
58
    prepare_abort,
Byron Hsu's avatar
Byron Hsu committed
59
)
60
from sglang.srt.distributed import get_pp_group, get_world_group
xm:D's avatar
xm:D committed
61
62
63
64
65
from sglang.srt.hf_transformers_utils import (
    get_processor,
    get_tokenizer,
    get_tokenizer_from_processor,
)
66
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
67
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
68
69
70
from sglang.srt.managers.expert_distribution import (
    get_global_expert_distribution_recorder,
)
71
72
from sglang.srt.managers.io_struct import (
    AbortReq,
73
    CloseSessionReqInput,
74
    ExpertDistributionReq,
75
    ExpertDistributionReqOutput,
76
77
    FlushCacheReqInput,
    FlushCacheReqOutput,
78
79
    GetInternalStateReq,
    GetInternalStateReqOutput,
80
81
    GetWeightsByNameReqInput,
    GetWeightsByNameReqOutput,
82
    HealthCheckOutput,
83
84
    InitWeightsUpdateGroupReqInput,
    InitWeightsUpdateGroupReqOutput,
85
86
    LoadLoRAAdapterReqInput,
    LoadLoRAAdapterReqOutput,
87
88
    OpenSessionReqInput,
    OpenSessionReqOutput,
89
    ProfileReq,
90
91
    ProfileReqOutput,
    ProfileReqType,
92
93
94
95
    ReleaseMemoryOccupationReqInput,
    ReleaseMemoryOccupationReqOutput,
    ResumeMemoryOccupationReqInput,
    ResumeMemoryOccupationReqOutput,
96
97
    RpcReqInput,
    RpcReqOutput,
98
99
    SetInternalStateReq,
    SetInternalStateReqOutput,
100
101
    SlowDownReqInput,
    SlowDownReqOutput,
102
103
    TokenizedEmbeddingReqInput,
    TokenizedGenerateReqInput,
104
105
    UnloadLoRAAdapterReqInput,
    UnloadLoRAAdapterReqOutput,
Chayenne's avatar
Chayenne committed
106
107
    UpdateWeightFromDiskReqInput,
    UpdateWeightFromDiskReqOutput,
108
109
    UpdateWeightsFromDistributedReqInput,
    UpdateWeightsFromDistributedReqOutput,
110
111
    UpdateWeightsFromTensorReqInput,
    UpdateWeightsFromTensorReqOutput,
112
)
113
from sglang.srt.managers.mm_utils import init_embedding_cache
114
115
from sglang.srt.managers.schedule_batch import (
    FINISH_ABORT,
Mick's avatar
Mick committed
116
    MultimodalInputs,
117
118
    Req,
    ScheduleBatch,
119
    global_server_args_dict,
120
)
121
122
123
124
125
from sglang.srt.managers.schedule_policy import (
    AddReqResult,
    PrefillAdder,
    SchedulePolicy,
)
126
127
128
from sglang.srt.managers.scheduler_output_processor_mixin import (
    SchedulerOutputProcessorMixin,
)
129
from sglang.srt.managers.session_controller import Session
130
from sglang.srt.managers.tp_worker import TpModelWorker
131
from sglang.srt.managers.tp_worker_overlap_thread import TpModelWorkerClient
132
from sglang.srt.managers.utils import validate_input_length
tarinkk's avatar
tarinkk committed
133
134
from sglang.srt.mem_cache.allocator import SWATokenToKVPoolAllocator
from sglang.srt.mem_cache.chunk_cache import ChunkCache, SWAChunkCache
135
from sglang.srt.mem_cache.hiradix_cache import HiRadixCache
136
from sglang.srt.mem_cache.radix_cache import RadixCache
137
from sglang.srt.metrics.collector import SchedulerMetricsCollector, SchedulerStats
Lianmin Zheng's avatar
Lianmin Zheng committed
138
from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors
139
from sglang.srt.reasoning_parser import ReasoningParser
140
from sglang.srt.server_args import PortArgs, ServerArgs
141
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
142
from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter
143
from sglang.srt.two_batch_overlap import TboDPAttentionPreparer
144
from sglang.srt.utils import (
145
    DeepEPMode,
146
    DynamicGradMode,
147
    broadcast_pyobj,
fzyzcjy's avatar
fzyzcjy committed
148
    configure_gc_logger,
149
    configure_logger,
Lianmin Zheng's avatar
Lianmin Zheng committed
150
    disable_request_logging,
151
    get_available_gpu_memory,
152
    get_bool_env_var,
153
    get_zmq_socket,
Lianmin Zheng's avatar
Lianmin Zheng committed
154
    kill_itself_when_parent_died,
155
    point_to_point_pyobj,
156
    pyspy_dump_schedulers,
157
158
    require_mlp_sync,
    require_mlp_tp_gather,
159
    set_gpu_proc_affinity,
160
161
162
    set_random_seed,
    suppress_other_loggers,
)
163
from sglang.utils import TypeBasedDispatcher, get_exception_traceback
164
165
166

logger = logging.getLogger(__name__)

167
# Test retract decode for debugging purposes
168
169
TEST_RETRACT = get_bool_env_var("SGLANG_TEST_RETRACT")
RECORD_STEP_TIME = get_bool_env_var("SGLANG_RECORD_STEP_TIME")
170
GRAMMAR_TIMEOUT = float(os.environ.get("SGLANG_GRAMMAR_TIMEOUT", 300))
171

172

173
174
@dataclass
class GenerationBatchResult:
175
176
177
    logits_output: Optional[LogitsProcessorOutput]
    pp_hidden_states_proxy_tensors: Optional[torch.Tensor]
    next_token_ids: Optional[List[int]]
178
179
    extend_input_len_per_req: List[int]
    extend_logprob_start_len_per_req: List[int]
180
    bid: int
181
    can_run_cuda_graph: bool
182
183
184
185
186
187
188
189


@dataclass
class EmbeddingBatchResult:
    embeddings: torch.Tensor
    bid: int


190
191
192
193
194
195
196
197
198
199
200
201
class KvMetrics:
    def __init__(self):
        self.request_active_slots = None
        self.request_total_slots = None
        self.kv_active_blocks = None
        self.kv_total_blocks = None
        self.num_requests_waiting = None
        self.gpu_cache_usage_perc = None
        self.gpu_prefix_cache_hit_rate = None
        self.data_parallel_rank = None


202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
class IdleSleeper:
    """
    In setups which have long inactivity periods it is desirable to reduce
    system power consumption when sglang does nothing. This would lead not only
    to power savings, but also to more CPU thermal headroom when a request
    eventually comes. This is important in cases when multiple GPUs are connected
    as each GPU would otherwise pin one thread at 100% CPU usage.

    The simplest solution is to use zmq.Poller on all sockets that may receive
    data that needs handling immediately.
    """

    def __init__(self, sockets):
        self.poller = zmq.Poller()
        for s in sockets:
            self.poller.register(s, zmq.POLLIN)

    def maybe_sleep(self):
        self.poller.poll(1000)


Byron Hsu's avatar
Byron Hsu committed
223
224
225
226
227
class Scheduler(
    SchedulerOutputProcessorMixin,
    SchedulerDisaggregationDecodeMixin,
    SchedulerDisaggregationPrefillMixin,
):
228
229
230
231
232
233
234
235
    """A scheduler that manages a tensor parallel GPU worker."""

    def __init__(
        self,
        server_args: ServerArgs,
        port_args: PortArgs,
        gpu_id: int,
        tp_rank: int,
236
        pp_rank: int,
237
        dp_rank: Optional[int],
238
239
    ):
        # Parse args
240
        self.server_args = server_args
241
        self.tp_rank = tp_rank
242
        self.pp_rank = pp_rank
243
        self.dp_rank = dp_rank
244
        self.tp_size = server_args.tp_size
245
246
        self.pp_size = server_args.pp_size
        self.dp_size = server_args.dp_size
247
248
249
        self.schedule_policy = server_args.schedule_policy
        self.lora_paths = server_args.lora_paths
        self.max_loras_per_batch = server_args.max_loras_per_batch
250
        self.enable_overlap = not server_args.disable_overlap_schedule
251
        self.skip_tokenizer_init = server_args.skip_tokenizer_init
252
        self.enable_metrics = server_args.enable_metrics
253
        self.enable_kv_cache_events = server_args.kv_events_config is not None
254
        self.stream_interval = server_args.stream_interval
255
256
257
        self.spec_algorithm = SpeculativeAlgorithm.from_string(
            server_args.speculative_algorithm
        )
258
259
        self.gpu_id = gpu_id
        self.enable_hierarchical_cache = server_args.enable_hierarchical_cache
Lianmin Zheng's avatar
Lianmin Zheng committed
260
        self.page_size = server_args.page_size
261
262
        self.dp_size = server_args.dp_size
        self.attn_tp_rank, self.attn_tp_size, self.attn_dp_rank = (
263
264
265
266
267
268
269
270
            compute_dp_attention_world_info(
                server_args.enable_dp_attention,
                self.tp_rank,
                self.tp_size,
                self.dp_size,
            )
        )

271
272
        # Init inter-process communication
        context = zmq.Context(2)
273
274
        self.idle_sleeper = None

275
        if self.pp_rank == 0 and self.attn_tp_rank == 0:
276
            self.recv_from_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
277
                context, zmq.PULL, port_args.scheduler_input_ipc_name, False
278
            )
279
            self.send_to_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
280
                context, zmq.PUSH, port_args.tokenizer_ipc_name, False
281
            )
282
283
284
            self.send_metrics_from_scheduler = get_zmq_socket(
                context, zmq.PUSH, port_args.metrics_ipc_name, False
            )
285

286
            if server_args.skip_tokenizer_init:
287
                # Directly send to the TokenizerManager
288
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
289
                    context, zmq.PUSH, port_args.tokenizer_ipc_name, False
290
291
                )
            else:
292
                # Send to the DetokenizerManager
293
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
294
                    context, zmq.PUSH, port_args.detokenizer_ipc_name, False
295
                )
296
297
298
299

            self.recv_from_rpc = get_zmq_socket(
                context, zmq.DEALER, port_args.rpc_ipc_name, False
            )
300
301
302
303
304
305
306
            if self.server_args.sleep_on_idle:
                self.idle_sleeper = IdleSleeper(
                    [
                        self.recv_from_tokenizer,
                        self.recv_from_rpc,
                    ]
                )
307
        else:
308
            self.recv_from_tokenizer = None
309
            self.recv_from_rpc = None
310
            self.send_metrics_from_scheduler = None
311
312
            self.send_to_tokenizer = SimpleNamespace(send_pyobj=lambda x: None)
            self.send_to_detokenizer = SimpleNamespace(send_pyobj=lambda x: None)
313
314

        # Init tokenizer
315
        self.init_tokenizer()
316

317
318
319
320
321
322
323
324
325
        # Set reasoning_parser and think_end_id if --reasoning_parser is enabled
        if self.server_args.reasoning_parser and self.tokenizer:
            reasoning_parser = ReasoningParser(
                model_type=self.server_args.reasoning_parser, stream_reasoning=False
            )
            self.tokenizer.think_end_id = self.tokenizer.encode(
                reasoning_parser.detector.think_end_token, add_special_tokens=False
            )[0]

326
327
328
329
        # Check whether overlap can be enabled
        if not self.is_generation:
            self.enable_overlap = False
            logger.info("Overlap scheduler is disabled for embedding models.")
330

331
        # Launch a tensor parallel worker
332
        if self.enable_overlap:
333
            TpWorkerClass = TpModelWorkerClient
334
335
        else:
            TpWorkerClass = TpModelWorker
336

337
        self.tp_worker = TpWorkerClass(
338
            server_args=server_args,
339
340
            gpu_id=gpu_id,
            tp_rank=tp_rank,
341
            pp_rank=pp_rank,
342
            dp_rank=dp_rank,
343
            nccl_port=port_args.nccl_port,
344
        )
345

346
        # Launch a draft worker for speculative decoding
347
348
349
350
351
352
353
354
355
356
357
358
359
360
        if self.spec_algorithm.is_eagle():
            from sglang.srt.speculative.eagle_worker import EAGLEWorker

            self.draft_worker = EAGLEWorker(
                gpu_id=gpu_id,
                tp_rank=tp_rank,
                server_args=server_args,
                nccl_port=port_args.nccl_port,
                target_worker=self.tp_worker,
                dp_rank=dp_rank,
            )
        else:
            self.draft_worker = None

361
        # Get token and memory info from the model worker
362
363
364
365
        (
            self.max_total_num_tokens,
            self.max_prefill_tokens,
            self.max_running_requests,
366
            self.max_req_len,
367
368
            self.max_req_input_len,
            self.random_seed,
369
            self.device,
370
371
372
373
374
            worker_global_server_args_dict,
            _,
            _,
            _,
        ) = self.tp_worker.get_worker_info()
375
376
377
378
379
380
381
382
        if global_server_args_dict["max_micro_batch_size"] is None:
            global_server_args_dict["max_micro_batch_size"] = max(
                self.max_running_requests // server_args.pp_size, 1
            )

        self.tp_group = self.tp_worker.get_tp_group()
        self.tp_cpu_group = self.tp_group.cpu_group
        self.attn_tp_group = self.tp_worker.get_attention_tp_group()
383
        self.attn_tp_cpu_group = self.tp_worker.get_attention_tp_cpu_group()
384
385
386
        self.pp_group = get_pp_group()
        self.world_group = get_world_group()

387
        self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
388
        global_server_args_dict.update(worker_global_server_args_dict)
389
        set_random_seed(self.random_seed)
390

391
        # Print debug info
392
        if tp_rank == 0:
393
394
395
            avail_mem = get_available_gpu_memory(
                self.device, self.gpu_id, empty_cache=False
            )
396
397
398
399
400
            logger.info(
                f"max_total_num_tokens={self.max_total_num_tokens}, "
                f"chunked_prefill_size={server_args.chunked_prefill_size}, "
                f"max_prefill_tokens={self.max_prefill_tokens}, "
                f"max_running_requests={self.max_running_requests}, "
401
402
                f"context_len={self.model_config.context_len}, "
                f"available_gpu_mem={avail_mem:.2f} GB"
403
            )
404

Lianmin Zheng's avatar
Lianmin Zheng committed
405
        # Init memory pool and cache
406
        self.init_memory_pool_and_cache()
407
408
409

        # Init running status
        self.waiting_queue: List[Req] = []
410
        # The running decoding batch for continuous batching
Lianmin Zheng's avatar
Lianmin Zheng committed
411
        self.running_batch: ScheduleBatch = ScheduleBatch(reqs=[], batch_is_full=False)
412
        # The current forward batch
Lianmin Zheng's avatar
Lianmin Zheng committed
413
        self.cur_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
414
        # The last forward batch
415
        self.last_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
416
417
        self.forward_ct = 0
        self.forward_ct_decode = 0
418
        self.num_generated_tokens = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
419
        self.last_prefill_tokens = 0
420
421
        self.last_decode_stats_tic = time.perf_counter()
        self.last_prefill_stats_tic = time.perf_counter()
422
        self.return_health_check_ct = 0
423
        self.current_stream = torch.get_device_module(self.device).current_stream()
424
425
        if self.device == "cpu":
            self.current_stream.synchronize = lambda: None  # No-op for CPU
426
        self.forward_sleep_time = None
427

428
        # Init session info
429
        self.sessions: Dict[str, Session] = {}
430
431
432

        # Init chunked prefill
        self.chunked_prefill_size = server_args.chunked_prefill_size
433
434
        if self.chunked_prefill_size <= 0:  # -1 means disable
            self.chunked_prefill_size = None
435
        self.chunked_req = None
436
437
438
439
        self.is_mixed_chunk = (
            self.chunked_prefill_size is not None and server_args.enable_mixed_chunk
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
440
        # Init the grammar backend for constrained generation
441
        self.grammar_queue: List[Req] = []
442
        if not server_args.skip_tokenizer_init:
443
444
445
            self.grammar_backend = create_grammar_backend(
                server_args, self.tokenizer, self.model_config.vocab_size
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
446
447
        else:
            self.grammar_backend = None
448

449
        # Init schedule policy and new token estimation
450
        self.policy = SchedulePolicy(
Lianmin Zheng's avatar
Lianmin Zheng committed
451
452
453
            self.schedule_policy,
            self.tree_cache,
            self.enable_hierarchical_cache,
454
        )
455
456
457
        assert (
            server_args.schedule_conservativeness >= 0
        ), "Invalid schedule_conservativeness"
458
459
        self.init_new_token_ratio = min(
            global_config.default_init_new_token_ratio
460
461
            * server_args.schedule_conservativeness,
            1.0,
462
        )
463
464
465
466
467
468
469
470
471
472
        self.min_new_token_ratio = min(
            self.init_new_token_ratio
            * global_config.default_min_new_token_ratio_factor,
            1.0,
        )
        self.new_token_ratio_decay = (
            self.init_new_token_ratio - self.min_new_token_ratio
        ) / global_config.default_new_token_ratio_decay_steps
        self.new_token_ratio = self.init_new_token_ratio

Lianmin Zheng's avatar
Lianmin Zheng committed
473
474
475
476
        # Init watchdog thread
        self.watchdog_timeout = server_args.watchdog_timeout
        t = threading.Thread(target=self.watchdog_thread, daemon=True)
        t.start()
477
        self.parent_process = psutil.Process().parent()
478
479
480
481
        self.memory_saver_adapter = TorchMemorySaverAdapter.create(
            enable=server_args.enable_memory_saver
        )

482
        # Init profiler
483
484
        self.torch_profiler = None
        self.torch_profiler_output_dir: Optional[str] = None
485
        self.profiler_activities: Optional[List[str]] = None
486
        self.profile_id: Optional[str] = None
487
        self.profiler_target_forward_ct: Optional[int] = None
488
489
490
491
492
493
494
495
        self.profiler_target_prefill_ct: Optional[int] = None
        self.profiler_target_decode_ct: Optional[int] = None
        self.profiler_prefill_ct: Optional[int] = None
        self.profiler_decode_ct: Optional[int] = None
        self.profile_by_stage: bool = False
        self.profile_steps: Optional[int] = None
        self.profile_in_progress: bool = False
        self.rpd_profiler = None
496

497
        # Init metrics stats
498
        self.init_metrics()
499
        self.init_kv_events(server_args.kv_events_config)
500

501
502
        # Init request dispatcher
        self._request_dispatcher = TypeBasedDispatcher(
503
504
505
            [
                (TokenizedGenerateReqInput, self.handle_generate_request),
                (TokenizedEmbeddingReqInput, self.handle_embedding_request),
506
                (FlushCacheReqInput, self.flush_cache_wrapped),
507
                (AbortReq, self.abort_request),
508
509
                (OpenSessionReqInput, self.open_session),
                (CloseSessionReqInput, self.close_session),
510
511
512
513
514
515
516
517
                (UpdateWeightFromDiskReqInput, self.update_weights_from_disk),
                (InitWeightsUpdateGroupReqInput, self.init_weights_update_group),
                (
                    UpdateWeightsFromDistributedReqInput,
                    self.update_weights_from_distributed,
                ),
                (UpdateWeightsFromTensorReqInput, self.update_weights_from_tensor),
                (GetWeightsByNameReqInput, self.get_weights_by_name),
518
519
                (ReleaseMemoryOccupationReqInput, self.release_memory_occupation),
                (ResumeMemoryOccupationReqInput, self.resume_memory_occupation),
520
                (SlowDownReqInput, self.slow_down),
521
                (ProfileReq, self.profile),
522
                (GetInternalStateReq, self.get_internal_state),
523
                (SetInternalStateReq, self.set_internal_state),
524
                (RpcReqInput, self.handle_rpc_request),
525
                (ExpertDistributionReq, self.expert_distribution_handle),
526
527
                (LoadLoRAAdapterReqInput, self.load_lora_adapter),
                (UnloadLoRAAdapterReqInput, self.unload_lora_adapter),
528
529
530
            ]
        )

Byron Hsu's avatar
Byron Hsu committed
531
532
533
534
535
        self.disaggregation_mode = DisaggregationMode(
            self.server_args.disaggregation_mode
        )
        self.init_disaggregation()

fzyzcjy's avatar
fzyzcjy committed
536
537
538
        if get_bool_env_var("SGLANG_GC_LOG"):
            configure_gc_logger()

539
540
541
542
    def maybe_sleep_on_idle(self):
        if self.idle_sleeper is not None:
            self.idle_sleeper.maybe_sleep()

543
544
    def init_tokenizer(self):
        server_args = self.server_args
Lianmin Zheng's avatar
Lianmin Zheng committed
545

546
        self.model_config = ModelConfig.from_server_args(server_args)
547
        self.is_generation = self.model_config.is_generation
548

549
550
551
552
553
554
555
556
557
        if server_args.skip_tokenizer_init:
            self.tokenizer = self.processor = None
        else:
            if self.model_config.is_multimodal:
                self.processor = get_processor(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
558
                    use_fast=not server_args.disable_fast_image_processor,
559
                )
xm:D's avatar
xm:D committed
560
                self.tokenizer = get_tokenizer_from_processor(self.processor)
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
            else:
                self.tokenizer = get_tokenizer(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
                )

    def init_memory_pool_and_cache(self):
        server_args = self.server_args

        self.req_to_token_pool, self.token_to_kv_pool_allocator = (
            self.tp_worker.get_memory_pool()
        )

        if (
            server_args.chunked_prefill_size is not None
            and server_args.disable_radix_cache
        ):
tarinkk's avatar
tarinkk committed
580
581
582
583
584
            if self.model_config.is_hybrid:
                ChunkCacheClass = SWAChunkCache
            else:
                ChunkCacheClass = ChunkCache
            self.tree_cache = ChunkCacheClass(
585
586
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
587
                page_size=self.page_size,
588
589
590
591
592
593
            )
        else:
            if self.enable_hierarchical_cache:
                self.tree_cache = HiRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
594
595
596
597
598
                    tp_cache_group=(
                        self.attn_tp_cpu_group
                        if self.server_args.enable_dp_attention
                        else self.tp_cpu_group
                    ),
599
                    page_size=self.page_size,
600
                    hicache_ratio=server_args.hicache_ratio,
Zhiqiang Xie's avatar
Zhiqiang Xie committed
601
602
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
603
                )
604
605
606
607
                self.tp_worker.register_hicache_layer_transfer_counter(
                    self.tree_cache.cache_controller.layer_done_counter
                )

608
609
610
611
            else:
                self.tree_cache = RadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Lianmin Zheng's avatar
Lianmin Zheng committed
612
                    page_size=self.page_size,
613
                    disable=server_args.disable_radix_cache,
614
                    enable_kv_cache_events=self.enable_kv_cache_events,
615
616
617
618
619
620
621
622
623
624
625
626
                )

        self.decode_mem_cache_buf_multiplier = (
            1
            if self.spec_algorithm.is_none()
            else (
                server_args.speculative_num_draft_tokens
                + (
                    server_args.speculative_eagle_topk
                    * server_args.speculative_num_steps
                )
            )
627
        )
628
629
630

    def init_metrics(self):
        self.last_gen_throughput: float = 0.0
Lianmin Zheng's avatar
Lianmin Zheng committed
631
        self.last_input_throughput: float = 0.0
632
633
634
635
636
637
638
639
640
641
642
643
644
645
        self.step_time_dict = defaultdict(list)  # Dict[batch size -> step time]
        self.spec_num_total_accepted_tokens = 0
        self.spec_num_total_forward_ct = 0
        self.cum_spec_accept_length = 0
        self.cum_spec_accept_count = 0
        self.stats = SchedulerStats()
        if self.enable_metrics:
            engine_type = "unified"
            self.metrics_collector = SchedulerMetricsCollector(
                labels={
                    "model_name": self.server_args.served_model_name,
                    "engine_type": engine_type,
                },
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
646

647
648
    def init_kv_events(self, kv_events_config: Optional[str]):
        if self.enable_kv_cache_events:
649
650
651
            self.kv_event_publisher = EventPublisherFactory.create(
                kv_events_config, self.attn_dp_rank
            )
652

Byron Hsu's avatar
Byron Hsu committed
653
    def init_disaggregation(self):
654
655
656
657
        self.transfer_backend = TransferBackend(
            self.server_args.disaggregation_transfer_backend
        )

Byron Hsu's avatar
Byron Hsu committed
658
659
660
661
        if (
            self.disaggregation_mode == DisaggregationMode.DECODE
        ):  # *2 for the headroom.
            buffer_size = (self.req_to_token_pool.size) * 2
Byron Hsu's avatar
Byron Hsu committed
662
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
663
664
                buffer_size
            )
665
666
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
667
668
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
669
670
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
671
672
673

            # The decode requests polling kv cache
            self.disagg_decode_transfer_queue = DecodeTransferQueue(
674
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
675
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
676
                tp_rank=self.tp_rank,
677
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
678
679
                scheduler=self,
                tree_cache=self.tree_cache,
Byron Hsu's avatar
Byron Hsu committed
680
681
682
683
684
685
            )

            # The decode requests pending for pre-allocation
            self.disagg_decode_prealloc_queue = DecodePreallocQueue(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Byron Hsu's avatar
Byron Hsu committed
686
687
688
689
690
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
691
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
692
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
693
694
695
                scheduler=self,
                transfer_queue=self.disagg_decode_transfer_queue,
                tree_cache=self.tree_cache,
696
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
697
698
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
699
700
                dp_size=self.server_args.dp_size,
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
701
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
702
703
                max_total_num_tokens=self.max_total_num_tokens,
                prefill_pp_size=self.server_args.disaggregation_prefill_pp,
704
                num_reserved_decode_tokens=self.server_args.num_reserved_decode_tokens,
705
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
706
            )
Liangsheng Yin's avatar
Liangsheng Yin committed
707
708
709
710

            # Metric for pre-allocation
            self.num_tokens_pre_allocated = 0

Byron Hsu's avatar
Byron Hsu committed
711
712
713
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            # *2 for the headroom.
            buffer_size = self.max_running_requests * 2
Byron Hsu's avatar
Byron Hsu committed
714
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
715
716
                buffer_size
            )
717
718
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
719
720
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
721
722
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
723

Liangsheng Yin's avatar
Liangsheng Yin committed
724
            self.disagg_prefill_bootstrap_queue = PrefillBootstrapQueue(
Byron Hsu's avatar
Byron Hsu committed
725
                token_to_kv_pool=self.token_to_kv_pool_allocator.get_kvcache(),
Byron Hsu's avatar
Byron Hsu committed
726
727
728
729
730
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
731
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
732
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
733
734
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
Byron Hsu's avatar
Byron Hsu committed
735
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
736
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
737
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
738
739
740
                max_total_num_tokens=self.max_total_num_tokens,
                decode_tp_size=self.server_args.disaggregation_decode_tp,
                decode_dp_size=self.server_args.disaggregation_decode_dp,
741
                scheduler=self,
Byron Hsu's avatar
Byron Hsu committed
742
743
744
                pp_rank=self.pp_rank,
                pp_size=self.pp_size,
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
745
746
            )
            # The prefill requests that are in the middle of kv sending
747
            self.disagg_prefill_inflight_queue: List[Req] = []
Byron Hsu's avatar
Byron Hsu committed
748

749
    @DynamicGradMode()
750
    def event_loop_normal(self):
751
        """A normal scheduler loop."""
752
        while True:
Lianmin Zheng's avatar
Lianmin Zheng committed
753
754
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)
755

756
            batch = self.get_next_batch_to_run()
Lianmin Zheng's avatar
Lianmin Zheng committed
757
            self.cur_batch = batch
758
759
760
761

            if batch:
                result = self.run_batch(batch)
                self.process_batch_result(batch, result)
Lianmin Zheng's avatar
Lianmin Zheng committed
762
            else:
Lianmin Zheng's avatar
Lianmin Zheng committed
763
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
764
                self.check_memory()
765
                self.new_token_ratio = self.init_new_token_ratio
766
                self.maybe_sleep_on_idle()
767
768

            self.last_batch = batch
769

770
    @DynamicGradMode()
Lianmin Zheng's avatar
Lianmin Zheng committed
771
    def event_loop_overlap(self):
772
        """A scheduler loop that overlaps the CPU processing and GPU computation."""
773
        self.result_queue = deque()
Lianmin Zheng's avatar
Lianmin Zheng committed
774
775
776
777
778
779
780

        while True:
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)

            batch = self.get_next_batch_to_run()
            self.cur_batch = batch
781

Lianmin Zheng's avatar
Lianmin Zheng committed
782
            if batch:
783
                batch.launch_done = threading.Event()
Lianmin Zheng's avatar
Lianmin Zheng committed
784
                result = self.run_batch(batch)
785
                self.result_queue.append((batch.copy(), result))
Lianmin Zheng's avatar
Lianmin Zheng committed
786

787
                if self.last_batch is None:
788
                    # Create a dummy first batch to start the pipeline for overlap schedule.
789
790
791
792
793
794
                    # It is now used for triggering the sampling_info_done event.
                    tmp_batch = ScheduleBatch(
                        reqs=None,
                        forward_mode=ForwardMode.DUMMY_FIRST,
                        next_batch_sampling_info=self.tp_worker.cur_sampling_info,
                    )
795
                    self.process_batch_result(tmp_batch, None, batch.launch_done)
796

Lianmin Zheng's avatar
Lianmin Zheng committed
797
            if self.last_batch:
798
                # Process the results of the last batch
799
                tmp_batch, tmp_result = self.result_queue.popleft()
800
801
802
                tmp_batch.next_batch_sampling_info = (
                    self.tp_worker.cur_sampling_info if batch else None
                )
803
804
805
806
                # NOTE: we should use current launched batch's launch_done event Instead of the last batch's
                self.process_batch_result(
                    tmp_batch, tmp_result, batch.launch_done if batch else None
                )
Lianmin Zheng's avatar
Lianmin Zheng committed
807
            elif batch is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
808
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
809
                self.check_memory()
810
                self.new_token_ratio = self.init_new_token_ratio
811
                self.maybe_sleep_on_idle()
Lianmin Zheng's avatar
Lianmin Zheng committed
812
813
814

            self.last_batch = batch

815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
    @DynamicGradMode()
    def event_loop_pp(self):
        """A non-overlap scheduler loop for pipeline parallelism."""
        mbs = [None] * self.pp_size
        last_mbs = [None] * self.pp_size
        self.running_mbs = [
            ScheduleBatch(reqs=[], batch_is_full=False) for _ in range(self.pp_size)
        ]
        bids = [None] * self.pp_size
        pp_outputs: Optional[PPProxyTensors] = None
        while True:
            server_is_idle = True
            for mb_id in range(self.pp_size):
                self.running_batch = self.running_mbs[mb_id]
                self.last_batch = last_mbs[mb_id]

                recv_reqs = self.recv_requests()
                self.process_input_requests(recv_reqs)
                mbs[mb_id] = self.get_next_batch_to_run()
                self.running_mbs[mb_id] = self.running_batch

                self.cur_batch = mbs[mb_id]
                if self.cur_batch:
                    server_is_idle = False
                    result = self.run_batch(self.cur_batch)

841
                # (last rank) send the outputs to the next step
842
843
844
845
846
847
                if self.pp_group.is_last_rank:
                    if self.cur_batch:
                        next_token_ids, bids[mb_id] = (
                            result.next_token_ids,
                            result.bid,
                        )
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
                        if self.cur_batch.return_logprob:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                    "extend_input_len_per_req": result.extend_input_len_per_req,
                                    "extend_logprob_start_len_per_req": result.extend_logprob_start_len_per_req,
                                }
                                | (
                                    {
                                        f"logits_output.{k}": v
                                        for k, v in result.logits_output.__dict__.items()
                                    }
                                    if result.logits_output is not None
                                    else {}
                                )
                            )
                        else:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                }
                            )
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
                        # send the output from the last round to let the next stage worker run post processing
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                # receive outputs and post-process (filter finished reqs) the coming microbatch
                next_mb_id = (mb_id + 1) % self.pp_size
                next_pp_outputs = None
                if mbs[next_mb_id] is not None:
                    next_pp_outputs: Optional[PPProxyTensors] = PPProxyTensors(
                        self.pp_group.recv_tensor_dict(
                            all_gather_group=self.attn_tp_group
                        )
                    )
                    mbs[next_mb_id].output_ids = next_pp_outputs["next_token_ids"]
886
887
888
889
890
891
892
893
894
                    logits_output_args = {
                        k[len("logits_output.") :]: v
                        for k, v in next_pp_outputs.tensors.items()
                        if k.startswith("logits_output.")
                    }
                    if len(logits_output_args) > 0:
                        logits_output = LogitsProcessorOutput(**logits_output_args)
                    else:
                        logits_output = None
895
                    output_result = GenerationBatchResult(
896
                        logits_output=logits_output,
897
898
                        pp_hidden_states_proxy_tensors=None,
                        next_token_ids=next_pp_outputs["next_token_ids"],
899
900
901
902
903
904
                        extend_input_len_per_req=next_pp_outputs.tensors.get(
                            "extend_input_len_per_req", None
                        ),
                        extend_logprob_start_len_per_req=next_pp_outputs.tensors.get(
                            "extend_logprob_start_len_per_req", None
                        ),
905
                        bid=bids[next_mb_id],
906
                        can_run_cuda_graph=result.can_run_cuda_graph,
907
908
909
910
                    )
                    self.process_batch_result(mbs[next_mb_id], output_result)
                    last_mbs[next_mb_id] = mbs[next_mb_id]

911
                # (not last rank)
912
913
914
                if not self.pp_group.is_last_rank:
                    if self.cur_batch:
                        bids[mb_id] = result.bid
915
916
                    # carry the outputs to the next stage
                    # send the outputs from the last round to let the next stage worker run post processing
917
918
919
920
921
922
923
                    if pp_outputs:
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                    # send out reqs to the next stage
924
                    dp_offset = self.attn_dp_rank * self.attn_tp_size
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
                    if self.attn_tp_rank == 0:
                        point_to_point_pyobj(
                            recv_reqs,
                            self.pp_rank * self.tp_size + dp_offset,
                            self.world_group.cpu_group,
                            self.pp_rank * self.tp_size + dp_offset,
                            (self.pp_rank + 1) * self.tp_size + dp_offset,
                        )

                    # send out proxy tensors to the next stage
                    if self.cur_batch:
                        self.pp_group.send_tensor_dict(
                            result.pp_hidden_states_proxy_tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                pp_outputs = next_pp_outputs

            # When the server is idle, self-check and re-init some states
            if server_is_idle:
                self.check_memory()
                self.new_token_ratio = self.init_new_token_ratio
947
                self.maybe_sleep_on_idle()
948

949
950
    def recv_requests(self) -> List[Req]:
        """Receive results at tp_rank = 0 and broadcast it to all other TP ranks."""
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
        if self.pp_rank == 0:
            if self.attn_tp_rank == 0:
                recv_reqs = []

                while True:
                    try:
                        recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_req)

                while True:
                    try:
                        recv_rpc = self.recv_from_rpc.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_rpc)
            else:
                recv_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
970
        else:
971
            if self.attn_tp_rank == 0:
972
                dp_offset = self.attn_dp_rank * self.attn_tp_size
973
974
975
976
977
978
979
980
981
                recv_reqs = point_to_point_pyobj(
                    [],
                    self.pp_rank * self.tp_size + dp_offset,
                    self.world_group.cpu_group,
                    (self.pp_rank - 1) * self.tp_size + dp_offset,
                    self.pp_rank * self.tp_size + dp_offset,
                )
            else:
                recv_reqs = None
982

983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
        if self.server_args.enable_dp_attention:
            if self.attn_tp_rank == 0:
                work_reqs = [
                    req
                    for req in recv_reqs
                    if isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
                control_reqs = [
                    req
                    for req in recv_reqs
                    if not isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
            else:
                work_reqs = None
                control_reqs = None

            if self.attn_tp_size != 1:
                work_reqs = broadcast_pyobj(
                    work_reqs,
1006
                    self.attn_tp_group.rank,
1007
                    self.attn_tp_cpu_group,
1008
                    src=self.attn_tp_group.ranks[0],
1009
1010
1011
                )
            if self.tp_size != 1:
                control_reqs = broadcast_pyobj(
1012
1013
1014
1015
                    control_reqs,
                    self.tp_group.rank,
                    self.tp_cpu_group,
                    src=self.tp_group.ranks[0],
1016
1017
1018
                )
            recv_reqs = work_reqs + control_reqs
        elif self.tp_size != 1:
1019
1020
1021
1022
1023
1024
            recv_reqs = broadcast_pyobj(
                recv_reqs,
                self.tp_group.rank,
                self.tp_cpu_group,
                src=self.tp_group.ranks[0],
            )
1025
1026
        return recv_reqs

Lianmin Zheng's avatar
Lianmin Zheng committed
1027
    def process_input_requests(self, recv_reqs: List):
1028
        for recv_req in recv_reqs:
1029
1030
            # If it is a health check generation request and there are running requests, ignore it.
            if is_health_check_generate_req(recv_req) and (
Lianmin Zheng's avatar
Lianmin Zheng committed
1031
                self.chunked_req is not None or not self.running_batch.is_empty()
1032
1033
1034
1035
            ):
                self.return_health_check_ct += 1
                continue

1036
            output = self._request_dispatcher(recv_req)
1037
            if output is not None:
1038
1039
1040
1041
1042
                if isinstance(output, RpcReqOutput):
                    if self.recv_from_rpc is not None:
                        self.recv_from_rpc.send_pyobj(output)
                else:
                    self.send_to_tokenizer.send_pyobj(output)
1043
1044
1045
1046
1047

    def handle_generate_request(
        self,
        recv_req: TokenizedGenerateReqInput,
    ):
1048
        # Create a new request
1049
1050
1051
1052
1053
        if (
            recv_req.session_params is None
            or recv_req.session_params.id is None
            or recv_req.session_params.id not in self.sessions
        ):
Rin Intachuen's avatar
Rin Intachuen committed
1054
1055
1056
1057
1058
1059
            if recv_req.input_embeds is not None:
                # Generate fake input_ids based on the length of input_embeds
                seq_length = len(recv_req.input_embeds)
                fake_input_ids = [1] * seq_length
                recv_req.input_ids = fake_input_ids

1060
1061
1062
1063
            if recv_req.bootstrap_port is None:
                # Use default bootstrap port
                recv_req.bootstrap_port = self.server_args.disaggregation_bootstrap_port

1064
1065
1066
1067
1068
            req = Req(
                recv_req.rid,
                recv_req.input_text,
                recv_req.input_ids,
                recv_req.sampling_params,
Lianmin Zheng's avatar
Lianmin Zheng committed
1069
1070
                return_logprob=recv_req.return_logprob,
                top_logprobs_num=recv_req.top_logprobs_num,
1071
                token_ids_logprob=recv_req.token_ids_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1072
                stream=recv_req.stream,
1073
                lora_path=recv_req.lora_path,
Rin Intachuen's avatar
Rin Intachuen committed
1074
                input_embeds=recv_req.input_embeds,
Lianmin Zheng's avatar
Lianmin Zheng committed
1075
                custom_logit_processor=recv_req.custom_logit_processor,
1076
                return_hidden_states=recv_req.return_hidden_states,
1077
                eos_token_ids=self.model_config.hf_eos_token_id,
1078
                bootstrap_host=recv_req.bootstrap_host,
1079
                bootstrap_port=recv_req.bootstrap_port,
1080
                bootstrap_room=recv_req.bootstrap_room,
1081
                data_parallel_rank=recv_req.data_parallel_rank,
1082
1083
            )
            req.tokenizer = self.tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
1084

1085
1086
1087
            if self.disaggregation_mode != DisaggregationMode.NULL:
                # Invalid request for disaggregated mode
                if recv_req.bootstrap_room is None:
1088
                    error_msg = (
1089
1090
1091
                        f"Invalid request: Disaggregated request received without "
                        f"boostrap room id. {req.rid=}"
                    )
1092
1093
                    logger.error(error_msg)
                    prepare_abort(req, error_msg)
1094
1095
1096
                    self.stream_output([req], req.return_logprob)
                    return

1097
1098
1099
1100
            if (
                recv_req.session_params is not None
                and recv_req.session_params.id is not None
            ):
1101
                req.finished_reason = FINISH_ABORT(
1102
                    f"Invalid request: session id {recv_req.session_params.id} does not exist"
1103
                )
1104
                self._add_request_to_queue(req)
1105
1106
                return
        else:
1107
1108
            # Create a new request from a previous session
            session = self.sessions[recv_req.session_params.id]
1109
            req = session.create_req(recv_req, self.tokenizer)
1110
            if isinstance(req.finished_reason, FINISH_ABORT):
1111
                self._add_request_to_queue(req)
1112
                return
1113

1114
        # Handle multimodal inputs
Mick's avatar
Mick committed
1115
1116
        if recv_req.mm_inputs is not None:
            image_inputs = MultimodalInputs.from_dict(recv_req.mm_inputs)
1117
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
1118
            req.origin_input_ids = self.pad_input_ids_func(
1119
                req.origin_input_ids, image_inputs
1120
            )
1121
            req.extend_image_inputs(image_inputs)
1122

1123
            if len(req.origin_input_ids) >= self.max_req_input_len:
1124
1125
1126
1127
1128
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1129
                )
1130
                self._add_request_to_queue(req)
1131
1132
                return

1133
        # Validate prompt length
1134
1135
1136
1137
1138
1139
        error_msg = validate_input_length(
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
        if error_msg:
1140
            req.set_finish_with_abort(error_msg)
1141
            self._add_request_to_queue(req)
1142
            return
1143

1144
        # Copy more attributes
1145
        if recv_req.logprob_start_len == -1 or not recv_req.return_logprob:
1146
1147
1148
1149
1150
            # By default, only return the logprobs for output tokens
            req.logprob_start_len = len(req.origin_input_ids) - 1
        else:
            req.logprob_start_len = recv_req.logprob_start_len

1151
        if req.logprob_start_len >= len(req.origin_input_ids):
1152
            error_msg = f"{req.logprob_start_len=} is higher than the number of input tokens {len(req.origin_input_ids)=}. Please use a smaller logprob_start_len."
1153
            req.logprob_start_len = len(req.origin_input_ids) - 1
1154
            req.set_finish_with_abort(error_msg)
1155
1156
1157
            self._add_request_to_queue(req)
            return

1158
1159
1160
1161
1162
1163
        req.sampling_params.max_new_tokens = min(
            (
                req.sampling_params.max_new_tokens
                if req.sampling_params.max_new_tokens is not None
                else 1 << 30
            ),
1164
            self.max_req_len - len(req.origin_input_ids) - 1,
1165
1166
        )

1167
1168
1169
1170
1171
        # Init grammar cache for this request
        add_to_grammar_queue = False
        if (
            req.sampling_params.json_schema is not None
            or req.sampling_params.regex is not None
1172
            or req.sampling_params.ebnf is not None
1173
            or req.sampling_params.structural_tag is not None
1174
1175
1176
1177
1178
1179
        ):
            assert self.grammar_backend is not None
            if req.sampling_params.json_schema is not None:
                key = ("json", req.sampling_params.json_schema)
            elif req.sampling_params.regex is not None:
                key = ("regex", req.sampling_params.regex)
1180
1181
            elif req.sampling_params.ebnf is not None:
                key = ("ebnf", req.sampling_params.ebnf)
1182
1183
            elif req.sampling_params.structural_tag:
                key = ("structural_tag", req.sampling_params.structural_tag)
1184

1185
1186
1187
1188
1189
            value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
            req.grammar = value

            if not cache_hit:
                req.grammar_key = key
1190
                add_to_grammar_queue = True
1191
1192
1193
1194
            else:
                if value is INVALID_GRAMMAR_OBJ:  # We hit a cached invalid grammar.
                    error_msg = f"Invalid grammar request with cache hit: {key=}"
                    req.set_finish_with_abort(error_msg)
1195
1196

        if add_to_grammar_queue:
1197
            req.queue_time_start = time.perf_counter()
1198
1199
            self.grammar_queue.append(req)
        else:
1200
1201
1202
            self._add_request_to_queue(req)

    def _add_request_to_queue(self, req: Req):
1203
        req.queue_time_start = time.perf_counter()
Byron Hsu's avatar
Byron Hsu committed
1204
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1205
1206
1207
            self.disagg_prefill_bootstrap_queue.add(
                req, self.model_config.num_key_value_heads
            )
Byron Hsu's avatar
Byron Hsu committed
1208
1209
1210
1211
1212
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            self.disagg_decode_prealloc_queue.add(req)
        else:
            self.waiting_queue.append(req)

1213
    def _extend_requests_to_queue(self, reqs: List[Req], is_retracted: bool = False):
1214
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1215
1216
1217
            self.disagg_prefill_bootstrap_queue.extend(
                reqs, self.model_config.num_key_value_heads
            )
1218
1219
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            # If this is a decode server, we put the request to the decode pending prealloc queue
1220
            self.disagg_decode_prealloc_queue.extend(reqs, is_retracted)
Byron Hsu's avatar
Byron Hsu committed
1221
1222
        else:
            self.waiting_queue.extend(reqs)
1223
1224
1225

    def handle_embedding_request(
        self,
1226
        recv_req: TokenizedEmbeddingReqInput,
1227
1228
1229
1230
1231
1232
    ):
        req = Req(
            recv_req.rid,
            recv_req.input_text,
            recv_req.input_ids,
            recv_req.sampling_params,
woodx's avatar
woodx committed
1233
            token_type_ids=recv_req.token_type_ids,
1234
1235
1236
        )
        req.tokenizer = self.tokenizer

1237
1238
        # Handle multimodal inputs
        if recv_req.image_inputs is not None:
Mick's avatar
Mick committed
1239
            image_inputs = MultimodalInputs.from_dict(recv_req.image_inputs)
1240
1241
1242
1243
1244
1245
1246
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
            req.origin_input_ids = self.pad_input_ids_func(
                req.origin_input_ids, image_inputs
            )
            req.extend_image_inputs(image_inputs)

            if len(req.origin_input_ids) >= self.max_req_input_len:
1247
1248
1249
1250
1251
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1252
                )
1253
                self._add_request_to_queue(req)
1254
1255
                return

1256
        # Validate prompts length
1257
        error_msg = validate_input_length(
1258
1259
1260
1261
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
1262
        if error_msg:
1263
            self._add_request_to_queue(req)
1264
            return
1265

1266
1267
        # Copy more attributes
        req.logprob_start_len = len(req.origin_input_ids) - 1
1268
        self._add_request_to_queue(req)
1269

1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
    def _emit_kv_metrics(self):
        kv_metrics = KvMetrics()
        kv_metrics.request_active_slots = self.stats.num_running_reqs
        kv_metrics.request_total_slots = self.max_running_requests
        kv_metrics.kv_active_blocks = int(
            self.stats.token_usage * self.max_total_num_tokens
        )
        kv_metrics.kv_total_blocks = self.max_total_num_tokens
        kv_metrics.num_requests_waiting = self.stats.num_queue_reqs
        kv_metrics.gpu_cache_usage_perc = self.stats.token_usage
        kv_metrics.gpu_prefix_cache_hit_rate = self.stats.cache_hit_rate
        kv_metrics.data_parallel_rank = self.dp_rank if self.dp_rank is not None else 0

        if not self.send_metrics_from_scheduler.closed:
            self.send_metrics_from_scheduler.send_pyobj(kv_metrics)

1286
1287
1288
1289
    def log_prefill_stats(
        self,
        adder: PrefillAdder,
        can_run_list: List[Req],
1290
        running_bs: int,
1291
    ):
1292
1293
        gap_latency = time.perf_counter() - self.last_prefill_stats_tic
        self.last_prefill_stats_tic = time.perf_counter()
Liangsheng Yin's avatar
Liangsheng Yin committed
1294
1295
        self.last_input_throughput = self.last_prefill_tokens / gap_latency
        self.last_prefill_tokens = adder.log_input_tokens
Lianmin Zheng's avatar
Lianmin Zheng committed
1296

tarinkk's avatar
tarinkk committed
1297
1298
        usage_msg, num_used = self.token_to_kv_pool_allocator.log_usage(
            self.tree_cache.evictable_size()
1299
1300
        )

1301
        num_new_seq = len(can_run_list)
1302
        f = (
1303
            f"Prefill batch. "
1304
            f"#new-seq: {num_new_seq}, "
1305
1306
            f"#new-token: {adder.log_input_tokens}, "
            f"#cached-token: {adder.log_hit_tokens}, "
tarinkk's avatar
tarinkk committed
1307
            f"{usage_msg}"
1308
        )
Liangsheng Yin's avatar
Liangsheng Yin committed
1309
1310
1311
1312

        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            f += f"#unbootstrapped-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, "
            f += f"#queue-req: {len(self.waiting_queue)}, "
fzyzcjy's avatar
fzyzcjy committed
1313
            f += f"#transferring-req: {len(self.disagg_prefill_inflight_queue)}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1314
            f += f"input throughput (token/s): {self.last_input_throughput:.2f} "
Liangsheng Yin's avatar
Liangsheng Yin committed
1315
        else:
Liangsheng Yin's avatar
Liangsheng Yin committed
1316
            f += f"#running-req: {running_bs}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1317
1318
            f += f"#queue-req: {len(self.waiting_queue)}"

1319
        logger.info(f)
1320
1321

        if self.enable_metrics:
1322
1323
1324
            cache_hit_rate = adder.log_hit_tokens / (
                adder.log_input_tokens + adder.log_hit_tokens
            )
1325
1326
1327
            self.stats.num_running_reqs = running_bs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = round(num_used / self.max_total_num_tokens, 2)
1328
1329
            self.stats.num_queue_reqs = len(self.waiting_queue)
            self.stats.cache_hit_rate = cache_hit_rate
1330
1331
1332
1333
1334
1335

            total_queue_latency = 0
            for req in can_run_list:
                total_queue_latency += req.queue_time_end - req.queue_time_start
            self.stats.avg_request_queue_latency = total_queue_latency / num_new_seq

1336
            self.metrics_collector.log_stats(self.stats)
1337
            self._emit_kv_metrics()
1338
        self._publish_kv_events()
1339

1340
1341
1342
    def log_decode_stats(
        self, can_run_cuda_graph: bool, running_batch: ScheduleBatch = None
    ):
1343
1344
        batch = running_batch or self.running_batch

1345
1346
        gap_latency = time.perf_counter() - self.last_decode_stats_tic
        self.last_decode_stats_tic = time.perf_counter()
1347
1348
        self.last_gen_throughput = self.num_generated_tokens / gap_latency
        self.num_generated_tokens = 0
1349
        num_running_reqs = len(batch.reqs)
tarinkk's avatar
tarinkk committed
1350
1351
        usage_msg, num_used = self.token_to_kv_pool_allocator.log_usage(
            self.tree_cache.evictable_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1352
        )
1353
1354
1355
1356
1357

        if RECORD_STEP_TIME:
            self.step_time_dict[num_running_reqs].append(
                gap_latency / self.server_args.decode_log_interval
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1358

tarinkk's avatar
tarinkk committed
1359
        msg = f"Decode batch. " f"#running-req: {num_running_reqs}, " f"{usage_msg}"
Liangsheng Yin's avatar
Liangsheng Yin committed
1360

1361
        if self.spec_algorithm.is_none():
1362
            spec_accept_length = 0
1363
        else:
1364
            spec_accept_length = (
1365
1366
                self.spec_num_total_accepted_tokens / self.spec_num_total_forward_ct
            )
1367
1368
            self.cum_spec_accept_length += self.spec_num_total_accepted_tokens
            self.cum_spec_accept_count += self.spec_num_total_forward_ct
1369
            self.spec_num_total_accepted_tokens = self.spec_num_total_forward_ct = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
1370
1371
1372
1373
            msg += f"accept len: {spec_accept_length:.2f}, "

        if self.disaggregation_mode == DisaggregationMode.DECODE:
            msg += f"pre-allocated usage: {self.num_tokens_pre_allocated / self.max_total_num_tokens:.2f}, "
1374
            msg += f"#retracted-req: {len(self.disagg_decode_prealloc_queue.retracted_queue)}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1375
1376

        msg += (
1377
            f"cuda graph: {can_run_cuda_graph}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1378
1379
1380
            f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
            f"#queue-req: {len(self.waiting_queue)}"
        )
1381
1382

        logger.info(msg)
1383
1384
1385
1386
        if self.enable_metrics:
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = num_used / self.max_total_num_tokens
1387
1388
            self.stats.cache_hit_rate = 0.0
            self.stats.gen_throughput = self.last_gen_throughput
1389
            self.stats.num_queue_reqs = len(self.waiting_queue)
1390
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1391
            self.stats.spec_accept_length = spec_accept_length
1392
            self.metrics_collector.log_stats(self.stats)
1393
            self._emit_kv_metrics()
1394
        self._publish_kv_events()
1395

Lianmin Zheng's avatar
Lianmin Zheng committed
1396
    def check_memory(self):
tarinkk's avatar
tarinkk committed
1397
1398
1399
1400
1401
        if isinstance(self.token_to_kv_pool_allocator, SWATokenToKVPoolAllocator):
            available_token_size = self.token_to_kv_pool_allocator.full_available_size()
        else:
            available_token_size = self.token_to_kv_pool_allocator.available_size()
        available_size = available_token_size + self.tree_cache.evictable_size()
1402
1403
1404
1405
1406
1407
1408
        protected_size = self.tree_cache.protected_size()
        memory_leak = available_size != (
            self.max_total_num_tokens
            if not self.enable_hierarchical_cache
            else self.max_total_num_tokens - protected_size
        )
        if memory_leak:
1409
            msg = (
1410
                "token_to_kv_pool_allocator memory leak detected! "
1411
                f"{available_size=}, {protected_size=}, {self.max_total_num_tokens=}\n"
tarinkk's avatar
tarinkk committed
1412
                f"{available_token_size=}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1413
                f"{self.tree_cache.evictable_size()=}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1414
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1415
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1416

1417
1418
1419
1420
1421
1422
1423
1424
        if self.disaggregation_mode == DisaggregationMode.DECODE:
            req_total_size = (
                self.req_to_token_pool.size + self.req_to_token_pool.pre_alloc_size
            )
        else:
            req_total_size = self.req_to_token_pool.size

        if len(self.req_to_token_pool.free_slots) != req_total_size:
1425
            msg = (
1426
                "req_to_token_pool memory leak detected!"
1427
1428
                f"available_size={len(self.req_to_token_pool.free_slots)}, "
                f"total_size={self.req_to_token_pool.size}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1429
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1430
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1431

1432
1433
1434
        if (
            self.enable_metrics
            and self.attn_tp_rank == 0
1435
            and time.perf_counter() > self.metrics_collector.last_log_time + 30
1436
1437
1438
        ):
            # During idle time, also collect metrics every 30 seconds.
            num_used = self.max_total_num_tokens - (
1439
                self.token_to_kv_pool_allocator.available_size()
1440
1441
                + self.tree_cache.evictable_size()
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1442
            num_running_reqs = len(self.running_batch.reqs)
1443
1444
1445
1446
1447
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
            self.stats.token_usage = num_used / self.max_total_num_tokens
            self.stats.gen_throughput = 0
            self.stats.num_queue_reqs = len(self.waiting_queue)
1448
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1449
            self.metrics_collector.log_stats(self.stats)
1450
        self._publish_kv_events()
1451

1452
    def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
1453
        # Merge the prefill batch into the running batch
1454
1455
1456
1457
1458
1459
1460
1461
        chunked_req_to_exclude = set()
        if self.chunked_req:
            # Move the chunked request out of the batch so that we can merge
            # only finished requests to running_batch.
            chunked_req_to_exclude.add(self.chunked_req)
            self.tree_cache.cache_unfinished_req(self.chunked_req)
            # chunked request keeps its rid but will get a new req_pool_idx
            self.req_to_token_pool.free(self.chunked_req.req_pool_idx)
Lianmin Zheng's avatar
Lianmin Zheng committed
1462
        if self.last_batch and self.last_batch.forward_mode.is_extend():
1463
1464
1465
1466
            if self.last_batch.chunked_req is not None:
                # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
                # We need to discard it.
                chunked_req_to_exclude.add(self.last_batch.chunked_req)
Lianmin Zheng's avatar
Lianmin Zheng committed
1467

1468
            # Filter batch
1469
            last_bs = self.last_batch.batch_size()
1470
1471
1472
            self.last_batch.filter_batch(
                chunked_req_to_exclude=list(chunked_req_to_exclude)
            )
1473
            if self.last_batch.batch_size() < last_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1474
                self.running_batch.batch_is_full = False
1475

1476
            # Merge the new batch into the running batch
1477
            if not self.last_batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1478
                if self.running_batch.is_empty():
1479
1480
                    self.running_batch = self.last_batch
                else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1481
                    # Merge running_batch with prefill batch
1482
                    self.running_batch.merge_batch(self.last_batch)
1483

1484
        new_batch = self.get_new_batch_prefill()
1485

1486
1487
1488
1489
1490
        need_dp_attn_preparation = require_mlp_sync(self.server_args)

        if need_dp_attn_preparation and not self.spec_algorithm.is_none():
            # In speculative decoding, prefill batches and decode batches cannot be processed in the same DP attention group.
            # We prepare idle batches in advance to skip preparing decode batches when there are prefill batches in the group.
Cheng Wan's avatar
Cheng Wan committed
1491
            new_batch, _ = self.prepare_mlp_sync_batch(new_batch)
1492
1493
1494
            need_dp_attn_preparation = new_batch is None

        if new_batch is not None:
1495
1496
1497
1498
            # Run prefill first if possible
            ret = new_batch
        else:
            # Run decode
Lianmin Zheng's avatar
Lianmin Zheng committed
1499
            if not self.running_batch.is_empty():
1500
                self.running_batch = self.update_running_batch(self.running_batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1501
1502
1503
                ret = self.running_batch if not self.running_batch.is_empty() else None
            else:
                ret = None
1504

1505
1506
        # Handle DP attention
        if need_dp_attn_preparation:
Cheng Wan's avatar
Cheng Wan committed
1507
            ret, _ = self.prepare_mlp_sync_batch(ret)
1508
1509

        return ret
1510

1511
1512
1513
1514
1515
1516
    def get_num_allocatable_reqs(self, running_bs):
        res = global_server_args_dict["max_micro_batch_size"] - running_bs
        if self.pp_size > 1:
            res = min(res, self.req_to_token_pool.available_size())
        return res

Lianmin Zheng's avatar
Lianmin Zheng committed
1517
    def get_new_batch_prefill(self) -> Optional[ScheduleBatch]:
Lianmin Zheng's avatar
Lianmin Zheng committed
1518
        # Check if the grammar is ready in the grammar queue
1519
        if self.grammar_queue:
1520
            self.move_ready_grammar_requests()
1521

Lianmin Zheng's avatar
Lianmin Zheng committed
1522
1523
        # Handle the cases where prefill is not allowed
        if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1524
            self.running_batch.batch_is_full or len(self.waiting_queue) == 0
1525
        ) and self.chunked_req is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
1526
1527
            return None

Lianmin Zheng's avatar
Lianmin Zheng committed
1528
        running_bs = len(self.running_batch.reqs)
1529
        # Ignore the check if self.chunked_req is not None.
1530
1531
1532
1533
1534
        # In the non-PP case, when self.chunked_req is not None, num_allocatable_reqs should always be greater than 0,
        # as the space for the chunked request has just been released.
        # In PP case, a chunked req can start in one microbatch and end in another microbatch, so the max_running_requests per microbatch should not be strict.
        # Instead, we should always allow chunked request to be added, otherwise, there will be a memory leak.
        if self.get_num_allocatable_reqs(running_bs) <= 0 and not self.chunked_req:
Lianmin Zheng's avatar
Lianmin Zheng committed
1535
            self.running_batch.batch_is_full = True
1536
1537
            return None

1538
        if self.enable_hierarchical_cache:
1539
            self.tree_cache.check_hicache_events()
1540

1541
        # Get priority queue
1542
        self.policy.calc_priority(self.waiting_queue)
1543

Lianmin Zheng's avatar
Lianmin Zheng committed
1544
        # Prefill policy
1545
        adder = PrefillAdder(
1546
            self.page_size,
1547
            self.tree_cache,
1548
            self.token_to_kv_pool_allocator,
1549
1550
1551
1552
            self.running_batch,
            self.new_token_ratio,
            self.max_prefill_tokens,
            self.chunked_prefill_size,
1553
            running_bs if self.is_mixed_chunk else 0,
1554
1555
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1556
        if self.chunked_req is not None:
1557
1558
            self.chunked_req.init_next_round_input()
            self.chunked_req = adder.add_chunked_req(self.chunked_req)
1559

Lianmin Zheng's avatar
Lianmin Zheng committed
1560
        if self.lora_paths:
Lianmin Zheng's avatar
Lianmin Zheng committed
1561
1562
            lora_set = set([req.lora_path for req in self.running_batch.reqs])

1563
        # Get requests from the waiting queue to a new prefill batch
1564
1565
        for req in self.waiting_queue:
            if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1566
                self.lora_paths
1567
1568
1569
1570
1571
1572
1573
                and len(
                    lora_set
                    | set([req.lora_path for req in adder.can_run_list])
                    | set([req.lora_path])
                )
                > self.max_loras_per_batch
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1574
                self.running_batch.batch_is_full = True
1575
1576
                break

1577
            if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):
Lianmin Zheng's avatar
Lianmin Zheng committed
1578
                self.running_batch.batch_is_full = True
1579
                break
1580

Byron Hsu's avatar
Byron Hsu committed
1581
1582
1583
1584
1585
1586
1587
            if self.disaggregation_mode == DisaggregationMode.PREFILL:
                # In prefill mode, prealloc queue and transfer queue can also take memory,
                # so we need to check if the available size for the actual available size.
                if len(adder.can_run_list) >= self.req_to_token_pool.available_size():
                    self.running_batch.batch_is_full = True
                    break

1588
1589
            req.init_next_round_input(self.tree_cache)
            res = adder.add_one_req(req, has_chunked_req=(self.chunked_req is not None))
1590

1591
1592
            if res != AddReqResult.CONTINUE:
                if res == AddReqResult.NO_TOKEN:
1593
1594
                    if self.enable_hierarchical_cache:
                        # Set batch_is_full after making sure there are requests that can be served
Lianmin Zheng's avatar
Lianmin Zheng committed
1595
1596
                        self.running_batch.batch_is_full = len(
                            adder.can_run_list
1597
                        ) > 0 or (not self.running_batch.is_empty())
1598
                    else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1599
                        self.running_batch.batch_is_full = True
1600
1601
                break

Lianmin Zheng's avatar
Lianmin Zheng committed
1602
        # Update waiting queue
1603
        can_run_list: List[Req] = adder.can_run_list
Lianmin Zheng's avatar
Lianmin Zheng committed
1604
1605
        if len(can_run_list) == 0:
            return None
1606
1607
1608
1609

        if self.enable_metrics:
            # only record queue time when enable_metrics is True to avoid overhead
            for req in can_run_list:
1610
                req.queue_time_end = time.perf_counter()
1611

Lianmin Zheng's avatar
Lianmin Zheng committed
1612
1613
1614
        self.waiting_queue = [
            x for x in self.waiting_queue if x not in set(can_run_list)
        ]
1615

1616
1617
1618
        if adder.new_chunked_req is not None:
            assert self.chunked_req is None
            self.chunked_req = adder.new_chunked_req
1619

1620
1621
        if self.chunked_req:
            self.chunked_req.is_chunked += 1
Lianmin Zheng's avatar
Lianmin Zheng committed
1622

1623
        # Print stats
1624
        if self.attn_tp_rank == 0:
1625
            self.log_prefill_stats(adder, can_run_list, running_bs)
1626

Lianmin Zheng's avatar
Lianmin Zheng committed
1627
        # Create a new batch
1628
1629
1630
        new_batch = ScheduleBatch.init_new(
            can_run_list,
            self.req_to_token_pool,
1631
            self.token_to_kv_pool_allocator,
1632
            self.tree_cache,
1633
            self.model_config,
1634
            self.enable_overlap,
1635
            self.spec_algorithm,
1636
            self.server_args.enable_custom_logit_processor,
1637
            chunked_req=self.chunked_req,
1638
        )
1639
1640
        if self.enable_hierarchical_cache:
            # todo (zhiqiang): disable cuda graph execution if hicache loading triggered
1641
1642
1643
            new_batch.hicache_consumer_index = (
                self.tree_cache.ready_to_load_host_cache()
            )
1644

1645
        new_batch.prepare_for_extend()
1646

Lianmin Zheng's avatar
Lianmin Zheng committed
1647
        # Mixed-style chunked prefill
1648
1649
        if (
            self.is_mixed_chunk
Lianmin Zheng's avatar
Lianmin Zheng committed
1650
            and not self.running_batch.is_empty()
1651
1652
1653
            and not (new_batch.return_logprob or self.running_batch.return_logprob)
        ):
            # TODO (lianmin): support return_logprob + mixed chunked prefill
1654
1655
            self.running_batch.filter_batch()
            if not self.running_batch.is_empty():
1656
                self.running_batch.prepare_for_decode()
1657
1658
                new_batch.mix_with_running(self.running_batch)
                new_batch.decoding_reqs = self.running_batch.reqs
Lianmin Zheng's avatar
Lianmin Zheng committed
1659
1660
1661
            self.running_batch = ScheduleBatch(
                reqs=[], batch_is_full=self.running_batch.batch_is_full
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1662
1663
        else:
            new_batch.decoding_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1664
1665
1666

        return new_batch

Lianmin Zheng's avatar
Lianmin Zheng committed
1667
    def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
1668
        """Update the current running decoding batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1669
        initial_bs = batch.batch_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1670

1671
1672
        batch.filter_batch()
        if batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1673
1674
            batch.batch_is_full = False
            return batch
1675

Lianmin Zheng's avatar
Lianmin Zheng committed
1676
        # Check if decode out of memory
1677
        if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or (
1678
            TEST_RETRACT and batch.batch_size() > 10
1679
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1680
1681
            old_ratio = self.new_token_ratio

1682
            retracted_reqs, new_token_ratio = batch.retract_decode(self.server_args)
Lianmin Zheng's avatar
Lianmin Zheng committed
1683
            self.new_token_ratio = new_token_ratio
1684

Lianmin Zheng's avatar
Lianmin Zheng committed
1685
            logger.info(
1686
                "KV cache pool is full. Retract requests. "
Lianmin Zheng's avatar
Lianmin Zheng committed
1687
1688
1689
                f"#retracted_reqs: {len(retracted_reqs)}, "
                f"#new_token_ratio: {old_ratio:.4f} -> {self.new_token_ratio:.4f}"
            )
1690
            self._extend_requests_to_queue(retracted_reqs, is_retracted=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
1691
1692
        else:
            self.new_token_ratio = max(
1693
                self.new_token_ratio - self.new_token_ratio_decay,
Lianmin Zheng's avatar
Lianmin Zheng committed
1694
1695
1696
                self.min_new_token_ratio,
            )

Lianmin Zheng's avatar
Lianmin Zheng committed
1697
        if batch.batch_size() < initial_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1698
            batch.batch_is_full = False
Lianmin Zheng's avatar
Lianmin Zheng committed
1699
1700

        # Update batch tensors
1701
        batch.prepare_for_decode()
Lianmin Zheng's avatar
Lianmin Zheng committed
1702
        return batch
Lianmin Zheng's avatar
Lianmin Zheng committed
1703

1704
1705
1706
    def run_batch(
        self, batch: ScheduleBatch
    ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
1707
        """Run a batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1708
1709
        self.forward_ct += 1

1710
1711
        # Whether to run the profiler
        self._profile_batch_predicate(batch)
1712
1713
1714
1715
        if self.forward_sleep_time is not None:
            logger.info(f"Scheduler.run_batch sleep {self.forward_sleep_time}s")
            time.sleep(self.forward_sleep_time)

1716
        # Run forward
1717
        if self.is_generation:
1718
1719
            if self.spec_algorithm.is_none():
                model_worker_batch = batch.get_model_worker_batch()
1720
1721
1722
1723
1724

                # update the consumer index of hicache to the running batch
                self.tp_worker.set_hicache_consumer(
                    model_worker_batch.hicache_consumer_index
                )
1725
                if self.pp_group.is_last_rank:
1726
                    logits_output, next_token_ids, can_run_cuda_graph = (
1727
1728
1729
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
                else:
1730
                    pp_hidden_states_proxy_tensors, _, can_run_cuda_graph = (
1731
1732
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
1733
                bid = model_worker_batch.bid
Lianmin Zheng's avatar
Lianmin Zheng committed
1734
            else:
1735
1736
1737
                (
                    logits_output,
                    next_token_ids,
1738
                    bid,
1739
                    num_accepted_tokens,
1740
                    can_run_cuda_graph,
1741
                ) = self.draft_worker.forward_batch_speculative_generation(batch)
1742
1743
1744
                bs = batch.batch_size()
                self.spec_num_total_accepted_tokens += num_accepted_tokens + bs
                self.spec_num_total_forward_ct += bs
1745
                self.num_generated_tokens += num_accepted_tokens
1746
1747
1748

            if self.pp_group.is_last_rank:
                batch.output_ids = next_token_ids
1749

1750
1751
1752
            # These 2 values are needed for processing the output, but the values can be
            # modified by overlap schedule. So we have to copy them here so that
            # we can use the correct values in output processing.
1753
            if batch.return_logprob or self.spec_algorithm.is_eagle():
1754
                extend_input_len_per_req = [req.extend_input_len for req in batch.reqs]
1755
1756
1757
            else:
                extend_input_len_per_req = None
            if batch.return_logprob:
1758
1759
1760
1761
1762
1763
                extend_logprob_start_len_per_req = [
                    req.extend_logprob_start_len for req in batch.reqs
                ]
            else:
                extend_logprob_start_len_per_req = None

1764
            ret = GenerationBatchResult(
1765
1766
1767
1768
1769
1770
1771
                logits_output=logits_output if self.pp_group.is_last_rank else None,
                pp_hidden_states_proxy_tensors=(
                    pp_hidden_states_proxy_tensors
                    if not self.pp_group.is_last_rank
                    else None
                ),
                next_token_ids=next_token_ids if self.pp_group.is_last_rank else None,
1772
1773
                extend_input_len_per_req=extend_input_len_per_req,
                extend_logprob_start_len_per_req=extend_logprob_start_len_per_req,
1774
                bid=bid,
1775
                can_run_cuda_graph=can_run_cuda_graph,
1776
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1777
1778
1779
        else:  # embedding or reward model
            model_worker_batch = batch.get_model_worker_batch()
            embeddings = self.tp_worker.forward_batch_embedding(model_worker_batch)
1780
1781
1782
            ret = EmbeddingBatchResult(
                embeddings=embeddings, bid=model_worker_batch.bid
            )
1783
        return ret
Chayenne's avatar
Chayenne committed
1784

1785
1786
1787
1788
    def process_batch_result(
        self,
        batch: ScheduleBatch,
        result: Union[GenerationBatchResult, EmbeddingBatchResult],
1789
        launch_done: Optional[threading.Event] = None,
1790
    ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1791
        if batch.forward_mode.is_decode():
1792
            self.process_batch_result_decode(batch, result, launch_done)
1793
        elif batch.forward_mode.is_extend():
1794
            self.process_batch_result_prefill(batch, result, launch_done)
1795
1796
        elif batch.forward_mode.is_idle():
            if self.enable_overlap:
1797
                self.tp_worker.resolve_last_batch_result(launch_done)
1798
                self.set_next_batch_sampling_info_done(batch)
1799
        elif batch.forward_mode.is_dummy_first():
1800
            self.set_next_batch_sampling_info_done(batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1801

1802
1803
1804
1805
1806
1807
1808
        if self.return_health_check_ct:
            # Return some signal for the health check.
            # This is used to prevent the health check signal being blocked by long context prefill.
            # However, one minor issue is that this code path does not check the status of detokenizer manager.
            self.return_health_check_ct -= 1
            self.send_to_tokenizer.send_pyobj(HealthCheckOutput())

1809
1810
    def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
        return self.prepare_mlp_sync_batch_raw(
1811
1812
1813
1814
1815
1816
1817
1818
            local_batch,
            dp_size=self.server_args.dp_size,
            attn_tp_size=self.attn_tp_size,
            tp_cpu_group=self.tp_cpu_group,
            get_idle_batch=self.get_idle_batch,
            disable_cuda_graph=self.server_args.disable_cuda_graph,
            spec_algorithm=self.spec_algorithm,
            speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens,
1819
1820
1821
            enable_two_batch_overlap=self.server_args.enable_two_batch_overlap,
            enable_deepep_moe=self.server_args.enable_deepep_moe,
            deepep_mode=DeepEPMode[self.server_args.deepep_mode],
1822
            require_mlp_tp_gather=require_mlp_tp_gather(self.server_args),
1823
1824
1825
        )

    @staticmethod
1826
    def prepare_mlp_sync_batch_raw(
1827
1828
1829
1830
1831
1832
1833
1834
        local_batch: ScheduleBatch,
        dp_size,
        attn_tp_size: int,
        tp_cpu_group,
        get_idle_batch,
        disable_cuda_graph: bool,
        spec_algorithm,
        speculative_num_draft_tokens,
1835
1836
1837
        enable_two_batch_overlap: bool,
        enable_deepep_moe: bool,
        deepep_mode: DeepEPMode,
1838
        require_mlp_tp_gather: bool,
1839
    ):
1840
1841
1842
        # Check if other DP workers have running batches
        if local_batch is None:
            num_tokens = 0
1843
            num_tokens_for_logprob = 0
1844
1845
        elif local_batch.forward_mode.is_decode():
            num_tokens = local_batch.batch_size()
1846
            num_tokens_for_logprob = num_tokens
1847
1848
        else:
            num_tokens = local_batch.extend_num_tokens
1849
            num_tokens_for_logprob = sum(
Lianmin Zheng's avatar
Lianmin Zheng committed
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
                [
                    # We should have at least 1 token for sample in every case.
                    max(extend_len - logprob_start_len, 1)
                    for logprob_start_len, extend_len in zip(
                        local_batch.extend_logprob_start_lens, local_batch.extend_lens
                    )
                ]
            )

        if local_batch is None or local_batch.forward_mode.is_decode_or_idle():
            can_cuda_graph = 1
        else:
            can_cuda_graph = 0

        is_extend_in_batch = (
            local_batch.forward_mode.is_extend() if local_batch else False
        )
1867
1868
1869

        tbo_preparer = TboDPAttentionPreparer()

Lianmin Zheng's avatar
Lianmin Zheng committed
1870
1871
1872
1873
        local_info = torch.tensor(
            [
                num_tokens,
                can_cuda_graph,
1874
                num_tokens_for_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1875
                is_extend_in_batch,
1876
1877
1878
1879
1880
1881
                *tbo_preparer.prepare_all_gather(
                    local_batch,
                    deepep_mode,
                    enable_deepep_moe,
                    enable_two_batch_overlap,
                ),
Lianmin Zheng's avatar
Lianmin Zheng committed
1882
1883
1884
1885
            ],
            dtype=torch.int64,
        )
        global_info = torch.empty(
1886
            (dp_size, attn_tp_size, 6),
Lianmin Zheng's avatar
Lianmin Zheng committed
1887
1888
            dtype=torch.int64,
        )
1889
        torch.distributed.all_gather_into_tensor(
Lianmin Zheng's avatar
Lianmin Zheng committed
1890
1891
            global_info.flatten(),
            local_info,
1892
            group=tp_cpu_group,
1893
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
1894
1895
1896
1897
        global_num_tokens = global_info[:, 0, 0].tolist()
        can_cuda_graph = min(global_info[:, 0, 1].tolist())
        global_num_tokens_for_logprob = global_info[:, 0, 2].tolist()
        is_extend_in_batch = global_info[:, 0, 3].tolist()
1898

1899
1900
1901
1902
        tbo_split_seq_index, global_forward_mode = tbo_preparer.compute_output(
            global_info[:, :, 4:6]
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1903
        if local_batch is None and max(global_num_tokens) > 0:
1904
            local_batch = get_idle_batch()
1905
1906

        if local_batch is not None:
1907
            # TODO: handle the case when moe_dense_tp_size != 1
1908
            if not require_mlp_tp_gather:
1909
1910
1911
1912
1913
1914
1915
                local_batch.global_num_tokens = [num_tokens]
                local_batch.global_num_tokens_for_logprob = [num_tokens_for_logprob]
            else:
                local_batch.global_num_tokens = global_num_tokens
                local_batch.global_num_tokens_for_logprob = (
                    global_num_tokens_for_logprob
                )
1916
            local_batch.is_extend_in_batch = any(is_extend_in_batch)
1917
1918
            local_batch.tbo_split_seq_index = tbo_split_seq_index
            local_batch.global_forward_mode = global_forward_mode
1919

1920
            # Check forward mode for cuda graph
1921
            if not disable_cuda_graph:
Lianmin Zheng's avatar
Lianmin Zheng committed
1922
                local_batch.can_run_dp_cuda_graph = can_cuda_graph
1923

1924
        # TODO(ch-wan): refactor: any(is_extend_in_batch) now is a part of local_batch. Remove it from here.
Lianmin Zheng's avatar
Lianmin Zheng committed
1925
        return local_batch, any(is_extend_in_batch)
1926
1927
1928
1929
1930

    def get_idle_batch(self):
        idle_batch = ScheduleBatch.init_new(
            [],
            self.req_to_token_pool,
1931
            self.token_to_kv_pool_allocator,
1932
1933
1934
            self.tree_cache,
            self.model_config,
            self.enable_overlap,
1935
            self.spec_algorithm,
1936
            self.server_args.enable_custom_logit_processor,
1937
1938
1939
1940
        )
        idle_batch.prepare_for_idle()
        return idle_batch

1941
1942
    def move_ready_grammar_requests(self):
        """Move requests whose grammar objects are ready from grammar_queue to waiting_queue."""
1943

1944
        num_ready_reqs = 0
1945
        num_timeout_reqs = 0
1946
1947
        for req in self.grammar_queue:
            try:
1948
1949
1950
                if req.finished():  # It is aborted by AbortReq
                    num_ready_reqs += 1
                    continue
1951
                req.grammar = req.grammar.result(timeout=0.03)
1952
1953
1954
1955
1956
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
1957
1958
                num_ready_reqs += 1
            except futures._base.TimeoutError:
1959
                req.grammar_wait_ct += 1
1960
1961
                # NOTE(lianmin): this timeout is the waiting time of the above line. It is
                # not the waiting time from it enters the grammar queue.
1962
                if req.grammar_wait_ct > GRAMMAR_TIMEOUT / 0.03:
1963
                    num_timeout_reqs = 1
1964
1965
                break

1966
        if self.server_args.enable_dp_attention:
1967
1968
            tp_size = self.attn_tp_size
            tp_group = self.attn_tp_cpu_group
1969
        else:
1970
1971
1972
1973
1974
            tp_size = self.tp_size
            tp_group = self.tp_cpu_group

        if tp_size > 1:
            # Sync across TP ranks to make sure they have the same number of ready requests
1975
            tensor = torch.tensor([num_ready_reqs, num_timeout_reqs], dtype=torch.int32)
1976
1977
1978
            torch.distributed.all_reduce(
                tensor, op=torch.distributed.ReduceOp.MAX, group=tp_group
            )
1979
            num_ready_reqs_max, num_timeout_reqs_max = tensor.tolist()
1980

1981
            for i in range(num_ready_reqs, num_ready_reqs_max):
1982
                req = self.grammar_queue[i]
1983
1984
                if req.finished():  # It is aborted by AbortReq
                    continue
1985
                req.grammar = req.grammar.result()
1986
1987
1988
1989
1990
1991
1992
1993
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
        else:
            num_ready_reqs_max = num_ready_reqs
            num_timeout_reqs_max = num_timeout_reqs
1994

1995
1996
1997
1998
1999
2000
2001
        for i in range(num_ready_reqs, num_ready_reqs + num_timeout_reqs_max):
            req = self.grammar_queue[i]
            req.grammar.cancel()
            error_msg = f"Grammar preprocessing timed out for {req.grammar_key=}"
            req.set_finish_with_abort(error_msg)
            self.grammar_backend.set_cache(req.grammar_key, INVALID_GRAMMAR_OBJ)
        num_ready_reqs = num_ready_reqs_max + num_timeout_reqs_max
2002

2003
        self._extend_requests_to_queue(self.grammar_queue[:num_ready_reqs])
2004
2005
        self.grammar_queue = self.grammar_queue[num_ready_reqs:]

2006
2007
2008
2009
2010
2011
2012
    def set_next_batch_sampling_info_done(self, batch: ScheduleBatch):
        if batch.next_batch_sampling_info:
            if batch.next_batch_sampling_info.grammars is not None:
                batch.next_batch_sampling_info.update_regex_vocab_mask()
                self.current_stream.synchronize()
            batch.next_batch_sampling_info.sampling_info_done.set()

2013
2014
2015
    def watchdog_thread(self):
        """A watch dog thread that will try to kill the server itself if one forward batch takes too long."""
        self.watchdog_last_forward_ct = 0
2016
        self.watchdog_last_time = time.perf_counter()
2017
2018

        while True:
2019
            current = time.perf_counter()
2020
2021
2022
2023
2024
2025
2026
2027
2028
            if self.cur_batch is not None:
                if self.watchdog_last_forward_ct == self.forward_ct:
                    if current > self.watchdog_last_time + self.watchdog_timeout:
                        break
                else:
                    self.watchdog_last_forward_ct = self.forward_ct
                    self.watchdog_last_time = current
            time.sleep(self.watchdog_timeout // 2)

Lianmin Zheng's avatar
Lianmin Zheng committed
2029
2030
2031
2032
2033
2034
2035
2036
2037
        if not disable_request_logging():
            # Print batch size and memory pool info to check whether there are de-sync issues.
            logger.error(
                f"{self.cur_batch.batch_size()=}, "
                f"{self.cur_batch.reqs=}, "
                f"{self.token_to_kv_pool_allocator.available_size()=}, "
                f"{self.tree_cache.evictable_size()=}, "
            )

2038
        pyspy_dump_schedulers()
Lianmin Zheng's avatar
Lianmin Zheng committed
2039
        logger.error(f"Watchdog timeout ({self.watchdog_timeout=})")
2040
2041
        print(file=sys.stderr, flush=True)
        print(file=sys.stdout, flush=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
2042
2043

        # Wait for some time so that the parent process can print the error.
2044
2045
2046
        time.sleep(5)
        self.parent_process.send_signal(signal.SIGQUIT)

2047
2048
2049
    def flush_cache_wrapped(self, recv_req: FlushCacheReqInput):
        success = self.flush_cache()
        return FlushCacheReqOutput(success=success)
2050

2051
    def flush_cache(self):
2052
        """Flush the memory pool and cache."""
2053
2054
2055
2056
2057
        if (
            len(self.waiting_queue) == 0
            and self.running_batch.is_empty()
            and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
        ):
2058
2059
            self.cur_batch = None
            self.last_batch = None
2060
            self.tree_cache.reset()
2061
            if self.grammar_backend:
Lianmin Zheng's avatar
Lianmin Zheng committed
2062
                self.grammar_backend.reset()
2063
            self.req_to_token_pool.clear()
2064
            self.token_to_kv_pool_allocator.clear()
2065
2066
2067

            if not self.spec_algorithm.is_none():
                self.draft_worker.model_runner.req_to_token_pool.clear()
2068
                self.draft_worker.model_runner.token_to_kv_pool_allocator.clear()
2069
2070
2071
2072
2073

            self.num_generated_tokens = 0
            self.forward_ct_decode = 0
            self.spec_num_total_accepted_tokens = 0
            self.spec_num_total_forward_ct = 0
2074
2075
            self.cum_spec_accept_length = 0
            self.cum_spec_accept_count = 0
2076
2077
2078
2079
2080
2081
2082
            torch.cuda.empty_cache()
            logger.info("Cache flushed successfully!")
            if_success = True
        else:
            logging.warning(
                f"Cache not flushed because there are pending requests. "
                f"#queue-req: {len(self.waiting_queue)}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2083
                f"#running-req: {len(self.running_batch.reqs)}"
2084
2085
2086
2087
            )
            if_success = False
        return if_success

Liangsheng Yin's avatar
Liangsheng Yin committed
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
    def get_load(self):
        # TODO(lsyin): use dynamically maintained num_waiting_tokens
        load = (
            self.max_total_num_tokens
            - self.token_to_kv_pool_allocator.available_size()
            - self.tree_cache.evictable_size()
        )
        load += sum(len(req.origin_input_ids) for req in self.waiting_queue)
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            load += sum(
                len(req.origin_input_ids)
                for req in self.disagg_prefill_bootstrap_queue.queue
            )
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            load += sum(
                len(req.req.origin_input_ids)
                for req in self.disagg_decode_prealloc_queue.queue
            )

        return load

2109
2110
2111
2112
2113
2114
2115
2116
2117
    def get_internal_state(self, recv_req: GetInternalStateReq):
        ret = dict(global_server_args_dict)
        ret["last_gen_throughput"] = self.last_gen_throughput
        if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
            ret["avg_spec_accept_length"] = (
                self.cum_spec_accept_length / self.cum_spec_accept_count
            )
        if RECORD_STEP_TIME:
            ret["step_time_dict"] = self.step_time_dict
Liangsheng Yin's avatar
Liangsheng Yin committed
2118
2119
2120
2121

        ret["load"] = self.get_load()

        return GetInternalStateReqOutput(internal_state=ret)
2122
2123
2124
2125
2126

    def set_internal_state(self, recv_req: SetInternalStateReq):
        server_args_dict = recv_req.server_args
        args_allow_update = set(
            [
2127
                "max_micro_batch_size",
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
                "speculative_accept_threshold_single",
                "speculative_accept_threshold_acc",
            ]
        )
        if_success = True
        for k, v in server_args_dict.items():
            if k not in args_allow_update:
                logging.warning(f"Updating {k} is not supported.")
                if_success = False
                break
2138
2139
2140
2141
2142
2143
2144
2145
            elif k == "max_micro_batch_size" and (
                v > self.max_running_requests // self.pp_size or v < 1
            ):
                logging.warning(
                    f"Updating {k} to {v} is rejected because it is out of the valid range [1, {self.max_running_requests // self.pp_size}]."
                )
                if_success = False
                break
2146
2147
2148
2149
2150
2151
2152
2153
2154
        if if_success:
            if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
                avg_spec_accept_length = (
                    self.cum_spec_accept_length / self.cum_spec_accept_count
                )
                logger.info(f"{avg_spec_accept_length=}")
            self.cum_spec_accept_length = self.cum_spec_accept_count = 0
            for k, v in server_args_dict.items():
                global_server_args_dict[k] = v
2155
            logger.info(f"Global server args updated! {global_server_args_dict=}")
2156
2157
2158
2159
2160
        return SetInternalStateReqOutput(
            updated=True,
            server_args=global_server_args_dict,
        )

2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
    def handle_rpc_request(self, recv_req: RpcReqInput):
        # Handle RPC requests
        logger.info(
            f"handle_rpc_request: {recv_req.method}, param: {recv_req.parameters}"
        )

        success = True
        exec = None
        try:
            func = getattr(self, recv_req.method)
            func(recv_req.parameters)
        except Exception as e:
            success = False
            exec = e
            logger.error(f"Failed to call rpc {recv_req.method}: {str(e)}")

        barrier()
        return RpcReqOutput(success, "" if not exec else str(exec))

    def save_remote_model(self, params):
        url = params["url"]

2183
        worker = self.tp_worker.worker
2184
2185
2186
2187

        worker.model_runner.save_remote_model(url)

    def save_sharded_model(self, params):
2188
        worker = self.tp_worker.worker
2189
2190
2191
2192
2193
2194
2195

        worker.model_runner.save_sharded_model(
            path=params["path"],
            pattern=params["pattern"],
            max_size=params["max_size"],
        )

2196
2197
    def abort_request(self, recv_req: AbortReq):
        # Delete requests in the waiting queue
Lianmin Zheng's avatar
Lianmin Zheng committed
2198
        to_del = []
2199
        for i, req in enumerate(self.waiting_queue):
Lianmin Zheng's avatar
Lianmin Zheng committed
2200
2201
            if req.rid.startswith(recv_req.rid):
                to_del.append(i)
2202

Lianmin Zheng's avatar
Lianmin Zheng committed
2203
        # Sort in reverse order to avoid index issues when deleting
Lianmin Zheng's avatar
Lianmin Zheng committed
2204
        for i in reversed(to_del):
2205
2206
2207
            # Abort method 1: directly pop from the queue
            # This only works for requests that have not started anything.
            # We still need to send something back to TokenizerManager to clean up the state.
Lianmin Zheng's avatar
Lianmin Zheng committed
2208
            req = self.waiting_queue.pop(i)
Lianmin Zheng's avatar
Lianmin Zheng committed
2209
            self.send_to_tokenizer.send_pyobj(AbortReq(req.rid))
2210
            logger.debug(f"Abort queued request. {req.rid=}")
2211

2212
2213
2214
2215
2216
2217
2218
        # Delete the requests in the grammar queue
        for req in self.grammar_queue:
            # Abort method 2: call `set_finish_with_abort`
            # The request will still run one prefill forward pass.
            # In this case, we change the input_ids to be only one token to make this prefill cheap.
            if req.rid.startswith(recv_req.rid):
                logger.debug(f"Abort grammar queue request. {req.rid=}")
2219
2220
                if req.grammar:
                    req.grammar.cancel()
2221
2222
                req.set_finish_with_abort("Aborted by AbortReq.")

2223
        # Delete requests in the running batch
Lianmin Zheng's avatar
Lianmin Zheng committed
2224
2225
2226
2227
2228
2229
        if self.cur_batch is self.running_batch or self.cur_batch is None:
            reqs = self.running_batch.reqs
        else:
            reqs = self.running_batch.reqs + self.cur_batch.reqs

        for req in reqs:
Lianmin Zheng's avatar
Lianmin Zheng committed
2230
            if req.rid.startswith(recv_req.rid) and not req.finished():
2231
2232
2233
                # Abort method 3: set `to_abort=True`
                # The request will still run one decode forward pass.
                # Then we reuse all existing code to clean up the KV cache allocation.
Lianmin Zheng's avatar
Lianmin Zheng committed
2234
2235
                logger.debug(f"Abort running request. {req.rid=}")
                req.to_abort = True
2236

2237
2238
2239
    def _pause_engine(self) -> Tuple[List[Req], int]:
        raise NotImplementedError()

Chayenne's avatar
Chayenne committed
2240
2241
2242
    def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
        """In-place update of the weights from disk."""
        success, message = self.tp_worker.update_weights_from_disk(recv_req)
2243
        if success:
Stefan He's avatar
Stefan He committed
2244
2245
            flush_cache_success = self.flush_cache()
            assert flush_cache_success, "Cache flush failed after updating weights"
2246
2247
        else:
            logger.error(message)
2248
        return UpdateWeightFromDiskReqOutput(success, message, 0)
2249

2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
    def load_lora_adapter(
        self, recv_req: LoadLoRAAdapterReqInput
    ) -> LoadLoRAAdapterReqOutput:
        """In-place loading a new lora adapter from disk or huggingface."""

        result = self.tp_worker.load_lora_adapter(recv_req)

        if result.success:
            flush_cache_success = self.flush_cache()
            assert flush_cache_success, "Cache flush failed after loading lora adapter."
        else:
            logger.error(result.error_message)
        return result

    def unload_lora_adapter(
        self, recv_req: UnloadLoRAAdapterReqInput
    ) -> UnloadLoRAAdapterReqOutput:
        """Unload the lora adapter."""

        result = self.tp_worker.unload_lora_adapter(recv_req)

        if result.success:
            flush_cache_success = self.flush_cache()
            assert (
                flush_cache_success
            ), "Cache flush failed after unloading LoRA weights"
        else:
            logger.error(result.error_message)
        return result

2280
2281
2282
    def init_weights_update_group(self, recv_req: InitWeightsUpdateGroupReqInput):
        """Initialize the online model parameter update group."""
        success, message = self.tp_worker.init_weights_update_group(recv_req)
2283
        return InitWeightsUpdateGroupReqOutput(success, message)
2284
2285

    def update_weights_from_distributed(
2286
2287
2288
        self,
        recv_req: UpdateWeightsFromDistributedReqInput,
    ) -> Tuple[bool, str]:
2289
2290
2291
        """Update the online model parameter."""
        success, message = self.tp_worker.update_weights_from_distributed(recv_req)
        if success:
Stefan He's avatar
Stefan He committed
2292
2293
            flush_cache_success = self.flush_cache()
            assert flush_cache_success, "Cache flush failed after updating weights"
2294
2295
        else:
            logger.error(message)
2296
        return UpdateWeightsFromDistributedReqOutput(success, message)
2297

2298
2299
2300
2301
2302
    def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
        """Update the online model parameter from tensors."""
        success, message = self.tp_worker.update_weights_from_tensor(recv_req)
        # TODO extract common code b/t update_weights_from_distributed and update_weights_from_tensor later
        if success:
2303
            if recv_req.flush_cache:
Stefan He's avatar
Stefan He committed
2304
2305
                flush_cache_success = self.flush_cache()
                assert flush_cache_success, "Cache flush failed after updating weights"
2306
2307
        else:
            logger.error(message)
2308
        barrier(group=self.tp_cpu_group)
2309
        return UpdateWeightsFromTensorReqOutput(success, message)
2310

2311
2312
    def get_weights_by_name(self, recv_req: GetWeightsByNameReqInput):
        parameter = self.tp_worker.get_weights_by_name(recv_req)
2313
        return GetWeightsByNameReqOutput(parameter)
2314

2315
    def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
        tags = recv_req.tags
        import subprocess

        if tags is None:
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_KV_CACHE)
            self.flush_cache()

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.stashed_model_static_state = _export_static_state(
                self.tp_worker.worker.model_runner.model
            )
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_WEIGHTS)

2332
        return ReleaseMemoryOccupationReqOutput()
2333

2334
    def resume_memory_occupation(self, recv_req: ResumeMemoryOccupationReqInput):
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
        tags = recv_req.tags
        if tags is None or len(tags) == 0:
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_WEIGHTS)
            _import_static_state(
                self.tp_worker.worker.model_runner.model,
                self.stashed_model_static_state,
            )
            del self.stashed_model_static_state

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_KV_CACHE)

2350
2351
        return ResumeMemoryOccupationReqOutput()

2352
2353
2354
2355
2356
2357
2358
    def slow_down(self, recv_req: SlowDownReqInput):
        t = recv_req.forward_sleep_time
        if t is not None and t <= 0:
            t = None
        self.forward_sleep_time = t
        return SlowDownReqOutput()

2359
    def profile(self, recv_req: ProfileReq):
2360
        if recv_req.type == ProfileReqType.START_PROFILE:
2361
2362
2363
2364
2365
2366
2367
2368
            if recv_req.profile_by_stage:
                return self.init_profile(
                    recv_req.output_dir,
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2369
                    recv_req.profile_id,
2370
2371
2372
2373
2374
2375
2376
2377
2378
                )
            else:
                self.init_profile(
                    recv_req.output_dir,
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2379
                    recv_req.profile_id,
2380
2381
                )
                return self.start_profile(True)
2382
        else:
2383
2384
            return self.stop_profile()

2385
    def init_profile(
2386
2387
2388
2389
        self,
        output_dir: Optional[str],
        num_steps: Optional[int],
        activities: Optional[List[str]],
2390
2391
        with_stack: Optional[bool],
        record_shapes: Optional[bool],
2392
        profile_by_stage: bool,
2393
        profile_id: str,
2394
2395
    ) -> ProfileReqOutput:
        if self.profile_in_progress:
2396
2397
2398
2399
2400
            return ProfileReqOutput(
                success=False,
                message="Profiling is already in progress. Call /stop_profile first.",
            )

2401
2402
        self.profile_by_stage = profile_by_stage

2403
2404
2405
2406
2407
2408
        if output_dir is None:
            output_dir = os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp")
        if activities is None:
            activities = ["CPU", "GPU"]

        self.torch_profiler_output_dir = output_dir
2409
2410
        self.torch_profiler_with_stack = with_stack
        self.torch_profiler_record_shapes = record_shapes
2411
        self.profiler_activities = activities
2412
        self.profile_id = profile_id
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432

        if num_steps:
            self.profile_steps = num_steps
            if self.profile_by_stage:
                self.profiler_target_prefill_ct = num_steps
                self.profiler_target_decode_ct = num_steps
                self.profiler_prefill_ct = 0
                self.profiler_decode_ct = 0
            else:
                self.profiler_target_forward_ct = self.forward_ct + num_steps
            # The caller will be notified when reaching profiler_target_forward_ct
        else:
            self.profiler_target_forward_ct = None

        return ProfileReqOutput(success=True, message="Succeeded")

    def start_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        stage_str = f" for {stage.__str__()}" if stage else ""
2433
        logger.info(
2434
            f"Profiling starts{stage_str}. Traces will be saved to: {self.torch_profiler_output_dir} (with profile id: {self.profile_id})",
2435
2436
        )

2437
2438
2439
2440
        activities = self.profiler_activities
        with_stack = self.torch_profiler_with_stack
        record_shapes = self.torch_profiler_record_shapes

2441
2442
2443
2444
2445
2446
2447
2448
        activity_map = {
            "CPU": torch.profiler.ProfilerActivity.CPU,
            "GPU": torch.profiler.ProfilerActivity.CUDA,
        }
        torchprof_activities = [
            activity_map[a] for a in activities if a in activity_map
        ]

2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
        if "RPD" in activities:
            from rpdTracerControl import rpdTracerControl

            rpdTracerControl.skipCreate()

            self.rpd_profile_path = os.path.join(
                self.torch_profiler_output_dir,
                "rpd-" + str(time.time()) + f"-TP-{self.tp_rank}" + ".trace.json.gz",
            )

            if self.tp_rank == 0:
                import sqlite3

                from rocpd.schema import RocpdSchema

                if os.path.exists("trace.rpd"):
                    os.unlink("trace.rpd")
                schema = RocpdSchema()
                connection = sqlite3.connect("trace.rpd")
                schema.writeSchema(connection)
                connection.commit()
                del connection
            torch.distributed.barrier(self.tp_cpu_group)

            self.rpd_profiler = rpdTracerControl()
            self.rpd_profiler.setPythonTrace(True)
            self.rpd_profiler.start()
            self.rpd_profiler.rangePush("", "rpd profile range", "")
            self.profile_in_progress = True
        elif torchprof_activities:
2479
2480
            self.torch_profiler = torch.profiler.profile(
                activities=torchprof_activities,
2481
2482
                with_stack=with_stack if with_stack is not None else True,
                record_shapes=record_shapes if record_shapes is not None else False,
2483
2484
            )
            self.torch_profiler.start()
2485
            self.profile_in_progress = True
2486
2487
2488

        if "MEM" in activities:
            torch.cuda.memory._record_memory_history(max_entries=100000)
2489
            self.profile_in_progress = True
2490

2491
2492
2493
        if "CUDA_PROFILER" in activities:
            torch.cuda.cudart().cudaProfilerStart()

2494
        return ProfileReqOutput(success=True, message="Succeeded")
2495

2496
2497
2498
2499
    def stop_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        if not self.profile_in_progress:
2500
2501
2502
2503
            return ProfileReqOutput(
                success=False,
                message="Profiling is not in progress. Call /start_profile first.",
            )
2504

2505
2506
2507
        if not Path(self.torch_profiler_output_dir).exists():
            Path(self.torch_profiler_output_dir).mkdir(parents=True, exist_ok=True)

2508
2509
        stage_suffix = f"-{stage.__str__()}" if stage else ""
        logger.info("Stop profiling" + stage_suffix + "...")
2510
2511
2512
2513
2514
        if self.torch_profiler is not None:
            self.torch_profiler.stop()
            self.torch_profiler.export_chrome_trace(
                os.path.join(
                    self.torch_profiler_output_dir,
2515
                    self.profile_id
2516
2517
2518
                    + f"-TP-{self.tp_rank}"
                    + stage_suffix
                    + ".trace.json.gz",
2519
2520
                )
            )
2521
2522
2523
2524
2525
2526
            torch.distributed.barrier(self.tp_cpu_group)

        if self.rpd_profiler is not None:
            self.rpd_profiler.rangePop()
            self.rpd_profiler.stop()
            self.rpd_profiler.flush()
2527

2528
2529
2530
2531
2532
2533
2534
2535
2536
            torch.distributed.barrier(self.tp_cpu_group)
            if self.tp_rank == 0:
                from sglang.srt.utils import rpd_to_chrome_trace

                rpd_to_chrome_trace("trace.rpd", self.rpd_profile_path)
            self.rpd_profiler = None
            self.rpd_profiler_path = None

        if self.profiler_activities is not None and "MEM" in self.profiler_activities:
2537
            memory_profile_path = os.path.join(
2538
                self.torch_profiler_output_dir,
2539
2540
2541
2542
                str(time.time())
                + f"-TP-{self.tp_rank}-memory"
                + stage_suffix
                + ".pickle",
2543
2544
2545
2546
            )
            torch.cuda.memory._dump_snapshot(memory_profile_path)
            torch.cuda.memory._record_memory_history(enabled=None)

2547
2548
2549
        if "CUDA_PROFILER" in self.profiler_activities:
            torch.cuda.cudart().cudaProfilerStop()

2550
2551
2552
        logger.info(
            "Profiling done. Traces are saved to: %s",
            self.torch_profiler_output_dir,
2553
        )
2554
        self.torch_profiler = None
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
        self.profile_in_progress = False

        return ProfileReqOutput(success=True, message="Succeeded.")

    def _profile_batch_predicate(self, batch):
        if self.profile_by_stage:
            if batch.forward_mode.is_prefill():
                if self.profiler_prefill_ct == 0:
                    self.start_profile(batch.forward_mode)
                self.profiler_prefill_ct += 1
                if self.profiler_prefill_ct > self.profiler_target_prefill_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.EXTEND)
            elif batch.forward_mode.is_decode():
                if self.profiler_decode_ct == 0:
                    if self.profile_in_progress:
                        # force trace flush
                        self.stop_profile(ForwardMode.EXTEND)
                    self.start_profile(batch.forward_mode)
                self.profiler_decode_ct += 1
                if self.profiler_decode_ct > self.profiler_target_decode_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.DECODE)
2578
2579
            elif batch.forward_mode.is_idle():
                pass
2580
            else:
2581
                raise RuntimeError(f"unsupported profile stage: {batch.forward_mode}")
2582
2583
2584
2585
2586
2587
2588
        else:
            # Check profiler
            if (
                self.profiler_target_forward_ct
                and self.profiler_target_forward_ct <= self.forward_ct
            ):
                self.stop_profile()
2589

2590
2591
    def expert_distribution_handle(self, recv_req: ExpertDistributionReq):
        if recv_req == ExpertDistributionReq.START_RECORD:
2592
            get_global_expert_distribution_recorder().start_record()
2593
        elif recv_req == ExpertDistributionReq.STOP_RECORD:
2594
            get_global_expert_distribution_recorder().stop_record()
2595
        elif recv_req == ExpertDistributionReq.DUMP_RECORD:
2596
            get_global_expert_distribution_recorder().dump_record()
2597
2598
        else:
            raise ValueError("Unrecognized ExpertDistributionReq value")
2599
        return ExpertDistributionReqOutput()
2600

2601
    def open_session(self, recv_req: OpenSessionReqInput):
2602
2603
2604
2605
        # handle error
        session_id = recv_req.session_id
        if session_id in self.sessions:
            logger.warning(f"session id {session_id} already exist, cannot open.")
2606
            return OpenSessionReqOutput(session_id, False)
2607
        elif session_id is None:
2608
            logger.warning("session id is None, cannot open.")
2609
            return OpenSessionReqOutput(session_id, False)
2610
2611
2612
2613
        else:
            self.sessions[session_id] = Session(
                recv_req.capacity_of_str_len, session_id
            )
2614
            return OpenSessionReqOutput(session_id, True)
2615
2616
2617
2618
2619
2620
2621
2622
2623

    def close_session(self, recv_req: CloseSessionReqInput):
        # handle error
        session_id = recv_req.session_id
        if session_id not in self.sessions:
            logger.warning(f"session id {session_id} does not exist, cannot delete.")
        else:
            del self.sessions[session_id]

2624
2625
    def get_print_prefix(self):
        prefix = ""
2626
2627
        if self.attn_dp_rank is not None:
            prefix += f" DP{self.attn_dp_rank}"
2628
2629
2630
2631
2632
2633
        if self.server_args.tp_size > 1:
            prefix += f" TP{self.tp_rank}"
        if self.pp_size > 1:
            prefix += f" PP{self.pp_rank}"
        return prefix

2634
2635
2636
2637
2638
2639
2640
    def _publish_kv_events(self):
        if self.enable_kv_cache_events:
            events = self.tree_cache.take_events()
            if events:
                batch = KVEventBatch(ts=time.time(), events=events)
                self.kv_event_publisher.publish(batch)

2641

2642
2643
2644
2645
def is_health_check_generate_req(recv_req):
    return getattr(recv_req, "rid", "").startswith("HEALTH_CHECK")


2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
def _export_static_state(model):
    return dict(
        buffers=[
            (name, buffer.detach().clone()) for name, buffer in model.named_buffers()
        ]
    )


def _import_static_state(model, static_params):
    self_named_buffers = dict(model.named_buffers())
    for name, tensor in static_params["buffers"]:
        self_named_buffers[name][...] = tensor


2660
2661
2662
2663
2664
def run_scheduler_process(
    server_args: ServerArgs,
    port_args: PortArgs,
    gpu_id: int,
    tp_rank: int,
2665
    pp_rank: int,
2666
    dp_rank: Optional[int],
2667
    pipe_writer,
2668
):
2669
    # Generate the prefix
2670
2671
2672
2673
2674
2675
2676
    prefix = ""
    if dp_rank is not None:
        prefix += f" DP{dp_rank}"
    if server_args.tp_size > 1:
        prefix += f" TP{tp_rank}"
    if server_args.pp_size > 1:
        prefix += f" PP{pp_rank}"
2677

2678
    # Config the process
2679
    kill_itself_when_parent_died()
2680
    setproctitle.setproctitle(f"sglang::scheduler{prefix.replace(' ', '_')}")
2681
    faulthandler.enable()
2682
    parent_process = psutil.Process().parent()
2683

2684
2685
2686
    # [For Router] if env var "SGLANG_DP_RANK" exist, set dp_rank to the value of the env var
    if dp_rank is None and "SGLANG_DP_RANK" in os.environ:
        dp_rank = int(os.environ["SGLANG_DP_RANK"])
2687

Wang Ran (汪然)'s avatar
Wang Ran (汪然) committed
2688
    # Configure the logger
2689
    configure_logger(server_args, prefix=prefix)
2690
    suppress_other_loggers()
2691

2692
    # Set cpu affinity to this gpu process
2693
2694
2695
    if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
        set_gpu_proc_affinity(server_args.tp_size, server_args.nnodes, gpu_id)

2696
2697
2698
2699
    embedding_cache_size = 100
    if "SGLANG_VLM_CACHE_SIZE_MB" in os.environ:
        embedding_cache_size = int(os.environ["SGLANG_VLM_CACHE_SIZE_MB"])
    init_embedding_cache(embedding_cache_size * 1024 * 1024)
2700
    # Create a scheduler and run the event loop
2701
    try:
2702
        scheduler = Scheduler(server_args, port_args, gpu_id, tp_rank, pp_rank, dp_rank)
2703
        pipe_writer.send(
Mick's avatar
Mick committed
2704
2705
2706
2707
2708
            {
                "status": "ready",
                "max_total_num_tokens": scheduler.max_total_num_tokens,
                "max_req_input_len": scheduler.max_req_input_len,
            }
2709
        )
Byron Hsu's avatar
Byron Hsu committed
2710
2711
2712
        disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode

        if disaggregation_mode == DisaggregationMode.NULL:
2713
2714
2715
            if server_args.pp_size > 1:
                scheduler.event_loop_pp()
            elif scheduler.enable_overlap:
Byron Hsu's avatar
Byron Hsu committed
2716
2717
2718
2719
                scheduler.event_loop_overlap()
            else:
                scheduler.event_loop_normal()
        elif disaggregation_mode == DisaggregationMode.PREFILL:
2720
2721
2722
2723
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_prefill()
            else:
                scheduler.event_loop_normal_disagg_prefill()
2724

Byron Hsu's avatar
Byron Hsu committed
2725
        elif disaggregation_mode == DisaggregationMode.DECODE:
2726
2727
2728
2729
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_decode()
            else:
                scheduler.event_loop_normal_disagg_decode()
Byron Hsu's avatar
Byron Hsu committed
2730

2731
    except Exception:
2732
2733
2734
        traceback = get_exception_traceback()
        logger.error(f"Scheduler hit an exception: {traceback}")
        parent_process.send_signal(signal.SIGQUIT)