scheduler.py 115 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
14
15
"""A scheduler that manages a tensor parallel GPU worker."""

16
import faulthandler
17
import logging
18
import os
19
import signal
20
import sys
Lianmin Zheng's avatar
Lianmin Zheng committed
21
import threading
22
import time
23
from collections import deque
Lianmin Zheng's avatar
Lianmin Zheng committed
24
from concurrent import futures
25
from dataclasses import dataclass
26
from http import HTTPStatus
27
from typing import Deque, Dict, List, Optional, Tuple, Union
28

29
import psutil
30
import setproctitle
31
import torch
32
import torch.distributed
33
import zmq
34
35
from torch.cuda import Stream as CudaStream
from torch.cuda import StreamContext as CudaStreamContext
36
from torch.distributed import barrier
37

Lianmin Zheng's avatar
Lianmin Zheng committed
38
from sglang.srt.configs.model_config import ModelConfig
39
40
41
42
from sglang.srt.constrained.base_grammar_backend import (
    INVALID_GRAMMAR_OBJ,
    create_grammar_backend,
)
Byron Hsu's avatar
Byron Hsu committed
43
44
45
46
47
from sglang.srt.disaggregation.decode import (
    DecodePreallocQueue,
    DecodeTransferQueue,
    SchedulerDisaggregationDecodeMixin,
)
48
49
50
from sglang.srt.disaggregation.decode_kvcache_offload_manager import (
    DecodeKVCacheOffloadManager,
)
Byron Hsu's avatar
Byron Hsu committed
51
52
53
54
55
56
from sglang.srt.disaggregation.prefill import (
    PrefillBootstrapQueue,
    SchedulerDisaggregationPrefillMixin,
)
from sglang.srt.disaggregation.utils import (
    DisaggregationMode,
57
    MetadataBuffers,
Byron Hsu's avatar
Byron Hsu committed
58
    ReqToMetadataIdxAllocator,
59
    TransferBackend,
60
    prepare_abort,
Byron Hsu's avatar
Byron Hsu committed
61
)
62
from sglang.srt.distributed import get_pp_group, get_world_group
63
from sglang.srt.environ import envs
fzyzcjy's avatar
fzyzcjy committed
64
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
65
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
66
from sglang.srt.layers.moe import initialize_moe_config
67
68
from sglang.srt.managers.io_struct import (
    AbortReq,
69
70
    BaseBatchReq,
    BaseReq,
71
72
    BatchTokenizedEmbeddingReqInput,
    BatchTokenizedGenerateReqInput,
73
74
    ClearHiCacheReqInput,
    ClearHiCacheReqOutput,
75
    CloseSessionReqInput,
76
    DestroyWeightsUpdateGroupReqInput,
77
    ExpertDistributionReq,
78
    ExpertDistributionReqOutput,
79
    ExpertDistributionReqType,
80
81
    FlushCacheReqInput,
    FlushCacheReqOutput,
82
    FreezeGCReq,
83
84
    GetInternalStateReq,
    GetInternalStateReqOutput,
85
86
    GetLoadReqInput,
    GetLoadReqOutput,
87
    GetWeightsByNameReqInput,
88
    HealthCheckOutput,
89
90
    InitWeightsSendGroupForRemoteInstanceReqInput,
    InitWeightsSendGroupForRemoteInstanceReqOutput,
91
    InitWeightsUpdateGroupReqInput,
92
93
    LoadLoRAAdapterReqInput,
    LoadLoRAAdapterReqOutput,
94
95
    OpenSessionReqInput,
    OpenSessionReqOutput,
96
    ProfileReq,
97
98
    ReleaseMemoryOccupationReqInput,
    ResumeMemoryOccupationReqInput,
99
100
    RpcReqInput,
    RpcReqOutput,
101
102
    SendWeightsToRemoteInstanceReqInput,
    SendWeightsToRemoteInstanceReqOutput,
103
104
    SetInternalStateReq,
    SetInternalStateReqOutput,
105
106
    SlowDownReqInput,
    SlowDownReqOutput,
107
108
    TokenizedEmbeddingReqInput,
    TokenizedGenerateReqInput,
109
110
    UnloadLoRAAdapterReqInput,
    UnloadLoRAAdapterReqOutput,
Chayenne's avatar
Chayenne committed
111
    UpdateWeightFromDiskReqInput,
112
    UpdateWeightsFromDistributedReqInput,
113
    UpdateWeightsFromIPCReqInput,
114
    UpdateWeightsFromTensorReqInput,
115
)
116
from sglang.srt.managers.mm_utils import init_embedding_cache
117
from sglang.srt.managers.overlap_utils import FutureMap
118
119
from sglang.srt.managers.schedule_batch import (
    FINISH_ABORT,
120
    ModelWorkerBatch,
Mick's avatar
Mick committed
121
    MultimodalInputs,
122
    Req,
123
    RequestStage,
124
125
    ScheduleBatch,
)
126
127
128
129
130
from sglang.srt.managers.schedule_policy import (
    AddReqResult,
    PrefillAdder,
    SchedulePolicy,
)
fzyzcjy's avatar
fzyzcjy committed
131
from sglang.srt.managers.scheduler_input_blocker import SchedulerInputBlocker
132
133
134
135
from sglang.srt.managers.scheduler_metrics_mixin import (
    RECORD_STEP_TIME,
    SchedulerMetricsMixin,
)
136
137
138
from sglang.srt.managers.scheduler_output_processor_mixin import (
    SchedulerOutputProcessorMixin,
)
139
from sglang.srt.managers.scheduler_pp_mixin import SchedulerPPMixin
140
from sglang.srt.managers.scheduler_profiler_mixin import SchedulerProfilerMixin
141
from sglang.srt.managers.scheduler_recv_skipper import SchedulerRecvSkipper
142
143
144
from sglang.srt.managers.scheduler_runtime_checker_mixin import (
    SchedulerRuntimeCheckerMixin,
)
145
146
147
from sglang.srt.managers.scheduler_update_weights_mixin import (
    SchedulerUpdateWeightsMixin,
)
148
from sglang.srt.managers.session_controller import Session
149
from sglang.srt.managers.utils import GenerationBatchResult, validate_input_length
tarinkk's avatar
tarinkk committed
150
from sglang.srt.mem_cache.chunk_cache import ChunkCache, SWAChunkCache
151
from sglang.srt.mem_cache.hiradix_cache import HiRadixCache
152
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
153
from sglang.srt.mem_cache.radix_cache import RadixCache
Hanming Lu's avatar
Hanming Lu committed
154
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
155
from sglang.srt.multiplex.multiplexing_mixin import SchedulerMultiplexMixin
156
from sglang.srt.parser.reasoning_parser import ReasoningParser
157
from sglang.srt.server_args import PortArgs, ServerArgs, get_global_server_args
158
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
159
160
from sglang.srt.tracing.trace import (
    process_tracing_init,
161
    trace_event_batch,
162
163
    trace_set_proc_propagate_context,
    trace_set_thread_info,
164
    trace_slice_batch,
165
166
167
    trace_slice_end,
    trace_slice_start,
)
168
from sglang.srt.two_batch_overlap import TboDPAttentionPreparer
169
from sglang.srt.utils import (
170
    DynamicGradMode,
171
    broadcast_pyobj,
fzyzcjy's avatar
fzyzcjy committed
172
    configure_gc_logger,
173
    configure_logger,
174
    freeze_gc,
175
    get_available_gpu_memory,
176
    get_bool_env_var,
177
    get_int_env_var,
178
    get_zmq_socket,
Lianmin Zheng's avatar
Lianmin Zheng committed
179
    kill_itself_when_parent_died,
180
    numa_bind_to_node,
181
    point_to_point_pyobj,
182
183
    require_mlp_sync,
    require_mlp_tp_gather,
184
    set_gpu_proc_affinity,
185
186
187
    set_random_seed,
    suppress_other_loggers,
)
188
189
190
191
192
from sglang.srt.utils.hf_transformers_utils import (
    get_processor,
    get_tokenizer,
    get_tokenizer_from_processor,
)
193
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
194
from sglang.utils import TypeBasedDispatcher, get_exception_traceback
195
196
197

logger = logging.getLogger(__name__)

198
# Test retract decode for debugging purposes
199
200
TEST_RETRACT = envs.SGLANG_TEST_RETRACT.get()
TEST_RETRACT_INTERVAL = envs.SGLANG_TEST_RETRACT_INTERVAL.get()
201
TEST_RETRACT_NO_PREFILL_BS = envs.SGLANG_TEST_RETRACT_NO_PREFILL_BS.get()
202
GRAMMAR_TIMEOUT = float(os.environ.get("SGLANG_GRAMMAR_TIMEOUT", 300))
203

204

205
206
207
208
209
@dataclass
class EmbeddingBatchResult:
    embeddings: torch.Tensor


Byron Hsu's avatar
Byron Hsu committed
210
211
class Scheduler(
    SchedulerOutputProcessorMixin,
212
213
214
    SchedulerUpdateWeightsMixin,
    SchedulerProfilerMixin,
    SchedulerMetricsMixin,
Byron Hsu's avatar
Byron Hsu committed
215
216
    SchedulerDisaggregationDecodeMixin,
    SchedulerDisaggregationPrefillMixin,
217
    SchedulerMultiplexMixin,
218
    SchedulerRuntimeCheckerMixin,
219
    SchedulerPPMixin,
Byron Hsu's avatar
Byron Hsu committed
220
):
221
222
223
224
225
226
227
228
    """A scheduler that manages a tensor parallel GPU worker."""

    def __init__(
        self,
        server_args: ServerArgs,
        port_args: PortArgs,
        gpu_id: int,
        tp_rank: int,
Cheng Wan's avatar
Cheng Wan committed
229
        moe_ep_rank: int,
230
        pp_rank: int,
231
        dp_rank: Optional[int],
232
233
    ):
        # Parse args
234
        self.server_args = server_args
235
        self.tp_rank = tp_rank
Cheng Wan's avatar
Cheng Wan committed
236
        self.moe_ep_rank = moe_ep_rank
237
        self.pp_rank = pp_rank
238
        self.dp_rank = dp_rank
239
        self.tp_size = server_args.tp_size
Cheng Wan's avatar
Cheng Wan committed
240
        self.moe_ep_size = server_args.ep_size
241
242
        self.pp_size = server_args.pp_size
        self.dp_size = server_args.dp_size
243
        self.schedule_policy = server_args.schedule_policy
244
        self.enable_priority_scheduling = server_args.enable_priority_scheduling
245
246
247
        self.abort_on_priority_when_disabled = (
            server_args.abort_on_priority_when_disabled
        )
248
249
250
251
252
253
        self.schedule_low_priority_values_first = (
            server_args.schedule_low_priority_values_first
        )
        self.priority_scheduling_preemption_threshold = (
            server_args.priority_scheduling_preemption_threshold
        )
254
        self.enable_lora = server_args.enable_lora
255
        self.max_loras_per_batch = server_args.max_loras_per_batch
256
        self.enable_overlap = not server_args.disable_overlap_schedule
257
        self.enable_pdmux = server_args.enable_pdmux
258
        self.skip_tokenizer_init = server_args.skip_tokenizer_init
259
        self.enable_metrics = server_args.enable_metrics
260
261
262
        self.enable_metrics_for_all_schedulers = (
            server_args.enable_metrics_for_all_schedulers
        )
263
264
265
        self.enable_kv_cache_events = bool(
            server_args.kv_events_config and tp_rank == 0
        )
266
        self.enable_trace = server_args.enable_trace
267
        self.stream_interval = server_args.stream_interval
268
269
270
        self.spec_algorithm = SpeculativeAlgorithm.from_string(
            server_args.speculative_algorithm
        )
271
        self.gpu_id = gpu_id
Lianmin Zheng's avatar
Lianmin Zheng committed
272
        self.page_size = server_args.page_size
273
        self.enable_hierarchical_cache = server_args.enable_hierarchical_cache
274
        self.enable_hicache_storage = server_args.hicache_storage_backend is not None
275

Lianmin Zheng's avatar
Lianmin Zheng committed
276
        # Distributed rank info
277
        self.attn_tp_rank, self.attn_tp_size, self.attn_dp_rank = (
278
279
280
281
282
283
284
285
            compute_dp_attention_world_info(
                server_args.enable_dp_attention,
                self.tp_rank,
                self.tp_size,
                self.dp_size,
            )
        )

286
287
288
        # Init model config
        self.model_config = ModelConfig.from_server_args(server_args)

289
        # Init inter-process communication
290
        self.init_sockets(server_args, port_args)
291

292
293
294
295
        # Init pdmux context
        if self.enable_pdmux:
            self.init_pdmux()

296
        # Init tokenizer
297
        self.init_tokenizer()
298

299
300
301
        # Init moe config
        self.init_moe_config()

302
303
304
305
        # Check whether overlap can be enabled
        if not self.is_generation:
            self.enable_overlap = False
            logger.info("Overlap scheduler is disabled for embedding models.")
306

307
        # Launch a tensor parallel worker
308
309
310
        from sglang.srt.managers.tp_worker import TpModelWorker

        self.tp_worker = TpModelWorker(
311
            server_args=server_args,
312
313
            gpu_id=gpu_id,
            tp_rank=tp_rank,
Cheng Wan's avatar
Cheng Wan committed
314
            moe_ep_rank=moe_ep_rank,
315
            pp_rank=pp_rank,
316
            dp_rank=dp_rank,
317
            nccl_port=port_args.nccl_port,
318
        )
319

320
        # Launch a draft worker for speculative decoding
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
        draft_worker_kwargs = dict(
            gpu_id=gpu_id,
            tp_rank=tp_rank,
            moe_ep_rank=moe_ep_rank,
            server_args=server_args,
            nccl_port=port_args.nccl_port,
            target_worker=self.tp_worker,
            dp_rank=dp_rank,
        )

        if server_args.speculative_draft_load_format is not None:
            server_args.load_format = server_args.speculative_draft_load_format
            logger.info(
                f"Using draft model load_format: '{server_args.speculative_draft_load_format}'"
            )

        # Draft workers are looked up via `SpeculativeAlgorithm` registry; new
        # algorithms should register their factory instead of patching this code.
        if self.spec_algorithm.name in {"EAGLE", "EAGLE3"}:
            draft_worker_kwargs["enable_overlap"] = self.enable_overlap
        self.draft_worker = self.spec_algorithm.create_draft_worker(
            **draft_worker_kwargs
343
        )
344

345
346
347
348
349
350
        # Dispatch the model worker
        if self.spec_algorithm.is_none():
            self.model_worker = self.tp_worker
        else:
            self.model_worker = self.draft_worker

351
        # Get token and memory info from the model worker
352
353
354
355
        (
            self.max_total_num_tokens,
            self.max_prefill_tokens,
            self.max_running_requests,
356
            self.max_queued_requests,
357
            self.max_req_len,
358
359
            self.max_req_input_len,
            self.random_seed,
360
            self.device,
361
362
363
364
            _,
            _,
            _,
        ) = self.tp_worker.get_worker_info()
365
366
        if get_global_server_args().pp_max_micro_batch_size is None:
            get_global_server_args().pp_max_micro_batch_size = max(
367
368
369
370
371
372
                self.max_running_requests // server_args.pp_size, 1
            )

        self.tp_group = self.tp_worker.get_tp_group()
        self.tp_cpu_group = self.tp_group.cpu_group
        self.attn_tp_group = self.tp_worker.get_attention_tp_group()
373
        self.attn_tp_cpu_group = self.tp_worker.get_attention_tp_cpu_group()
374
375
376
        self.pp_group = get_pp_group()
        self.world_group = get_world_group()

377
378
379
380
381
382
383
384
385
386
387
        # With DP attention enabled, the entry rank is attn_tp_rank==0;
        # otherwise the entry rank is TP group local rank 0.
        # For #11910, use the CPU communication group to broadcast VLM Python objects,
        # avoiding any coupling with CUDA streams/devices.
        if self.server_args.enable_dp_attention:
            self.cpu_group = self.attn_tp_cpu_group
            self.is_entry_rank = self.attn_tp_rank == 0
        else:
            self.cpu_group = self.tp_cpu_group
            self.is_entry_rank = self.tp_group.rank == 0

388
        self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
389
        set_random_seed(self.random_seed)
390

391
        # Hybrid memory pool
Hanming Lu's avatar
Hanming Lu committed
392
        self.is_hybrid = self.tp_worker.is_hybrid
393
        self.is_hybrid_gdn = self.tp_worker.model_runner.hybrid_gdn_config is not None
394

Hanming Lu's avatar
Hanming Lu committed
395
396
397
398
399
400
        if self.is_hybrid:
            self.sliding_window_size = self.tp_worker.sliding_window_size
            self.full_tokens_per_layer, self.swa_tokens_per_layer = (
                self.tp_worker.get_tokens_per_layer_info()
            )

401
        # Print debug info
402
        if tp_rank == 0:
403
404
405
            avail_mem = get_available_gpu_memory(
                self.device, self.gpu_id, empty_cache=False
            )
406
407
408
409
410
            logger.info(
                f"max_total_num_tokens={self.max_total_num_tokens}, "
                f"chunked_prefill_size={server_args.chunked_prefill_size}, "
                f"max_prefill_tokens={self.max_prefill_tokens}, "
                f"max_running_requests={self.max_running_requests}, "
411
                f"context_len={self.model_config.context_len}, "
412
                f"{'available_cpu_mem' if self.device == 'cpu' else 'available_gpu_mem'}={avail_mem:.2f} GB"
413
            )
414

Lianmin Zheng's avatar
Lianmin Zheng committed
415
        # Init memory pool and cache
416
        self.init_memory_pool_and_cache()
417
418
419

        # Init running status
        self.waiting_queue: List[Req] = []
420
        # The running decoding batch for continuous batching
Lianmin Zheng's avatar
Lianmin Zheng committed
421
        self.running_batch: ScheduleBatch = ScheduleBatch(reqs=[], batch_is_full=False)
422
        # The current forward batch
Lianmin Zheng's avatar
Lianmin Zheng committed
423
        self.cur_batch: Optional[ScheduleBatch] = None
424
425
        # The current split prefill batch
        self.split_prefill_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
426
        # The last forward batch
427
        self.last_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
428
429
        self.forward_ct = 0
        self.forward_ct_decode = 0
430
        self.num_generated_tokens = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
431
        self.last_prefill_tokens = 0
432
        self.return_health_check_ct = 0
433
434
435
        self.num_retracted_reqs: int = 0
        self.num_paused_reqs: int = 0
        self.sessions: Dict[str, Session] = {}
436
437
438
        self.default_stream: CudaStream = torch.get_device_module(
            self.device
        ).current_stream()
439
        if self.device == "cpu":
440
            self.default_stream.synchronize = lambda: None  # No-op for CPU
441
        self.forward_sleep_time = None
442

443
444
        # Init chunked prefill
        self.chunked_prefill_size = server_args.chunked_prefill_size
445
446
        if self.chunked_prefill_size <= 0:  # -1 means disable
            self.chunked_prefill_size = None
447
        self.chunked_req = None
448
449
450
451
        self.is_mixed_chunk = (
            self.chunked_prefill_size is not None and server_args.enable_mixed_chunk
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
452
        # Init the grammar backend for constrained generation
453
        self.grammar_queue: List[Req] = []
454
        if not server_args.skip_tokenizer_init:
455
            self.grammar_backend = create_grammar_backend(
456
457
458
459
                server_args,
                self.tokenizer,
                self.model_config.vocab_size,
                self.model_config.hf_eos_token_id,
460
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
461
462
        else:
            self.grammar_backend = None
463

464
        # Init schedule policy and new token estimation
465
        self.policy = SchedulePolicy(
Lianmin Zheng's avatar
Lianmin Zheng committed
466
467
468
            self.schedule_policy,
            self.tree_cache,
            self.enable_hierarchical_cache,
469
470
            self.enable_priority_scheduling,
            self.schedule_low_priority_values_first,
471
        )
472
473
        # Enable preemption for priority scheduling.
        self.try_preemption = self.enable_priority_scheduling
474
        self.init_new_token_ratio = min(
475
            envs.SGLANG_INIT_NEW_TOKEN_RATIO.get()
476
477
            * server_args.schedule_conservativeness,
            1.0,
478
        )
479
        self.min_new_token_ratio = min(
480
            self.init_new_token_ratio * envs.SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR.get(),
481
482
483
484
            1.0,
        )
        self.new_token_ratio_decay = (
            self.init_new_token_ratio - self.min_new_token_ratio
485
        ) / envs.SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS.get()
486
487
        self.new_token_ratio = self.init_new_token_ratio

Lianmin Zheng's avatar
Lianmin Zheng committed
488
489
490
491
        # Init watchdog thread
        self.watchdog_timeout = server_args.watchdog_timeout
        t = threading.Thread(target=self.watchdog_thread, daemon=True)
        t.start()
492
        self.parent_process = psutil.Process().parent()
493
494

        # Init memory saver, profiler and metric stats
495
496
497
        self.memory_saver_adapter = TorchMemorySaverAdapter.create(
            enable=server_args.enable_memory_saver
        )
498
        self.offload_tags = set()
limingshu's avatar
limingshu committed
499
        self.init_profiler()
500
        self.recv_skipper = SchedulerRecvSkipper.maybe_create(server_args)
fzyzcjy's avatar
fzyzcjy committed
501
502
503
504
505
506
        self.input_blocker = (
            SchedulerInputBlocker(noop=self.attn_tp_rank != 0)
            if get_bool_env_var("SGLANG_ENABLE_COLOCATED_BATCH_GEN")
            else None
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
507
508
509
        # Init disaggregation
        self.init_disaggregation()

510
        # Init metrics stats
511
        self.init_metrics(tp_rank, pp_rank, dp_rank)
512

513
514
515
        if self.enable_kv_cache_events:
            self.init_kv_events(server_args.kv_events_config)

516
        if envs.SGLANG_LOG_GC.get():
517
518
            configure_gc_logger()

519
520
        # Init prefill kv split size when deterministic inference is enabled with various attention backends
        self.init_deterministic_inference_config()
521

522
523
524
        # Init overlap
        self.init_overlap()

525
526
        # Init request dispatcher
        self._request_dispatcher = TypeBasedDispatcher(
527
528
529
            [
                (TokenizedGenerateReqInput, self.handle_generate_request),
                (TokenizedEmbeddingReqInput, self.handle_embedding_request),
530
531
                (BatchTokenizedGenerateReqInput, self.handle_batch_generate_request),
                (BatchTokenizedEmbeddingReqInput, self.handle_batch_embedding_request),
532
                (FlushCacheReqInput, self.flush_cache_wrapped),
533
                (ClearHiCacheReqInput, self.clear_hicache_storage_wrapped),
534
                (AbortReq, self.abort_request),
535
536
                (OpenSessionReqInput, self.open_session),
                (CloseSessionReqInput, self.close_session),
537
538
                (UpdateWeightFromDiskReqInput, self.update_weights_from_disk),
                (InitWeightsUpdateGroupReqInput, self.init_weights_update_group),
539
                (DestroyWeightsUpdateGroupReqInput, self.destroy_weights_update_group),
540
541
542
543
544
545
546
547
                (
                    InitWeightsSendGroupForRemoteInstanceReqInput,
                    self.init_weights_send_group_for_remote_instance,
                ),
                (
                    SendWeightsToRemoteInstanceReqInput,
                    self.send_weights_to_remote_instance,
                ),
548
549
550
551
552
                (
                    UpdateWeightsFromDistributedReqInput,
                    self.update_weights_from_distributed,
                ),
                (UpdateWeightsFromTensorReqInput, self.update_weights_from_tensor),
553
                (UpdateWeightsFromIPCReqInput, self.update_weights_from_ipc),
554
                (GetWeightsByNameReqInput, self.get_weights_by_name),
555
556
                (ReleaseMemoryOccupationReqInput, self.release_memory_occupation),
                (ResumeMemoryOccupationReqInput, self.resume_memory_occupation),
557
                (SlowDownReqInput, self.slow_down),
558
                (ProfileReq, self.profile),
559
                (FreezeGCReq, self.handle_freeze_gc),
560
                (GetInternalStateReq, self.get_internal_state),
561
                (SetInternalStateReq, self.set_internal_state),
562
                (RpcReqInput, self.handle_rpc_request),
563
                (ExpertDistributionReq, self.expert_distribution_handle),
564
565
                (LoadLoRAAdapterReqInput, self.load_lora_adapter),
                (UnloadLoRAAdapterReqInput, self.unload_lora_adapter),
566
                (GetLoadReqInput, self.get_load),
567
568
569
            ]
        )

570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
    def init_sockets(self, server_args: ServerArgs, port_args: PortArgs):
        context = zmq.Context(2)
        self.idle_sleeper = None

        class SenderWrapper:
            def __init__(self, socket: zmq.Socket):
                self.socket = socket

            def send_output(
                self,
                output: Union[BaseReq, BaseBatchReq],
                recv_obj: Optional[Union[BaseReq, BaseBatchReq]] = None,
            ):
                if self.socket is None:
                    return

                if (
                    isinstance(recv_obj, BaseReq)
                    and recv_obj.http_worker_ipc is not None
                    and output.http_worker_ipc is None
                ):
                    # handle communicator reqs for multi-http worker case
                    output.http_worker_ipc = recv_obj.http_worker_ipc

                self.socket.send_pyobj(output)

        if self.pp_rank == 0 and self.attn_tp_rank == 0:
            self.recv_from_tokenizer = get_zmq_socket(
                context, zmq.PULL, port_args.scheduler_input_ipc_name, False
            )
            self.recv_from_rpc = get_zmq_socket(
                context, zmq.DEALER, port_args.rpc_ipc_name, False
            )

            send_to_tokenizer = get_zmq_socket(
                context, zmq.PUSH, port_args.tokenizer_ipc_name, False
            )
            if server_args.skip_tokenizer_init:
                # Directly send to the TokenizerManager
                send_to_detokenizer = get_zmq_socket(
                    context, zmq.PUSH, port_args.tokenizer_ipc_name, False
                )
            else:
                # Send to the DetokenizerManager
                send_to_detokenizer = get_zmq_socket(
                    context, zmq.PUSH, port_args.detokenizer_ipc_name, False
                )

            self.send_to_tokenizer = SenderWrapper(send_to_tokenizer)
            self.send_to_detokenizer = SenderWrapper(send_to_detokenizer)

            if self.server_args.sleep_on_idle:
                self.idle_sleeper = IdleSleeper(
                    [
                        self.recv_from_tokenizer,
                        self.recv_from_rpc,
                    ]
                )
        else:
            self.recv_from_tokenizer = None
            self.recv_from_rpc = None
            self.send_to_tokenizer = SenderWrapper(None)
            self.send_to_detokenizer = SenderWrapper(None)

        if self.current_scheduler_metrics_enabled():
            self.send_metrics_from_scheduler = get_zmq_socket(
                context, zmq.PUSH, port_args.metrics_ipc_name, False
            )

639
640
641
642
643
644
    def init_deterministic_inference_config(self):
        """Initialize deterministic inference configuration for different attention backends."""
        if not self.server_args.enable_deterministic_inference:
            self.truncation_align_size = None
            return

645
646
647
648
649
        backend_sizes = {
            "flashinfer": ("SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE", 4096),
            "triton": ("SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE", 4096),
        }
        env_var, default_size = backend_sizes.get(
650
651
652
653
654
655
            self.server_args.attention_backend, (None, None)
        )
        self.truncation_align_size = (
            get_int_env_var(env_var, default_size) if env_var else None
        )

656
657
658
    def init_tokenizer(self):
        server_args = self.server_args
        self.is_generation = self.model_config.is_generation
659

660
661
662
663
664
665
666
667
668
        if server_args.skip_tokenizer_init:
            self.tokenizer = self.processor = None
        else:
            if self.model_config.is_multimodal:
                self.processor = get_processor(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
669
                    use_fast=not server_args.disable_fast_image_processor,
670
                )
xm:D's avatar
xm:D committed
671
                self.tokenizer = get_tokenizer_from_processor(self.processor)
672
673
674
675
676
677
678
679
            else:
                self.tokenizer = get_tokenizer(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
                )

Lianmin Zheng's avatar
Lianmin Zheng committed
680
681
682
683
684
685
686
687
688
        # Set reasoning_parser and think_end_id if --reasoning_parser is enabled
        if self.server_args.reasoning_parser and self.tokenizer:
            reasoning_parser = ReasoningParser(
                model_type=self.server_args.reasoning_parser, stream_reasoning=False
            )
            self.tokenizer.think_end_id = self.tokenizer.encode(
                reasoning_parser.detector.think_end_token, add_special_tokens=False
            )[0]

689
690
691
692
693
694
695
696
697
698
699
    def init_memory_pool_and_cache(self):
        server_args = self.server_args

        self.req_to_token_pool, self.token_to_kv_pool_allocator = (
            self.tp_worker.get_memory_pool()
        )

        if (
            server_args.chunked_prefill_size is not None
            and server_args.disable_radix_cache
        ):
Hanming Lu's avatar
Hanming Lu committed
700
            if self.is_hybrid:
tarinkk's avatar
tarinkk committed
701
702
703
704
                ChunkCacheClass = SWAChunkCache
            else:
                ChunkCacheClass = ChunkCache
            self.tree_cache = ChunkCacheClass(
705
706
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
707
                page_size=self.page_size,
708
709
            )
        else:
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
            if os.environ.get("SGLANG_EXPERIMENTAL_CPP_RADIX_TREE") == "1":
                # lazy import to avoid JIT overhead
                from sglang.srt.mem_cache.radix_cache_cpp import RadixCacheCpp

                self.tree_cache = RadixCacheCpp(
                    disable=False,
                    use_hicache=self.enable_hierarchical_cache,
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool=self.token_to_kv_pool_allocator,
                    tp_cache_group=self.tp_cpu_group,
                    page_size=self.page_size,
                    hicache_ratio=server_args.hicache_ratio,
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
                    enable_kv_cache_events=self.enable_kv_cache_events,
                )
            elif self.enable_hierarchical_cache:
727
728
729
                self.tree_cache = HiRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
730
731
732
733
734
                    tp_cache_group=(
                        self.attn_tp_cpu_group
                        if self.server_args.enable_dp_attention
                        else self.tp_cpu_group
                    ),
735
                    page_size=self.page_size,
736
                    eviction_policy=server_args.radix_eviction_policy,
737
                    hicache_ratio=server_args.hicache_ratio,
Zhiqiang Xie's avatar
Zhiqiang Xie committed
738
739
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
740
                    hicache_io_backend=server_args.hicache_io_backend,
741
                    hicache_mem_layout=server_args.hicache_mem_layout,
742
                    enable_metrics=self.enable_metrics,
743
                    hicache_storage_backend=server_args.hicache_storage_backend,
pansicheng's avatar
pansicheng committed
744
                    hicache_storage_prefetch_policy=server_args.hicache_storage_prefetch_policy,
745
746
                    model_name=server_args.served_model_name,
                    storage_backend_extra_config=server_args.hicache_storage_backend_extra_config,
Ke Bao's avatar
Ke Bao committed
747
                    is_eagle=self.spec_algorithm.is_eagle(),
748
                )
749
750
751
                self.tp_worker.register_hicache_layer_transfer_counter(
                    self.tree_cache.cache_controller.layer_done_counter
                )
Hanming Lu's avatar
Hanming Lu committed
752
753
754
755
756
757
758
            elif self.is_hybrid:
                self.tree_cache = SWARadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                    sliding_window_size=self.sliding_window_size,
                    page_size=self.page_size,
                    disable=server_args.disable_radix_cache,
759
                    is_eagle=self.spec_algorithm.is_eagle(),
Hanming Lu's avatar
Hanming Lu committed
760
                )
761
762
763
764
765
766
767
            elif self.is_hybrid_gdn:
                self.tree_cache = MambaRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                    page_size=self.page_size,
                    disable=server_args.disable_radix_cache,
                )
768
769
770
771
772
773
774
775
776
777
778
779
780
781
            elif server_args.enable_lmcache:
                from sglang.srt.mem_cache.storage.lmcache.lmc_radix_cache import (
                    LMCRadixCache,
                )

                self.tree_cache = LMCRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                    page_size=self.page_size,
                    disable=server_args.disable_radix_cache,
                    model_config=self.model_config,
                    tp_size=self.tp_size,
                    rank=self.tp_rank,
                    tp_group=self.tp_group,
782
                    eviction_policy=server_args.radix_eviction_policy,
783
                )
784
785
786
787
            else:
                self.tree_cache = RadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Lianmin Zheng's avatar
Lianmin Zheng committed
788
                    page_size=self.page_size,
789
                    disable=server_args.disable_radix_cache,
790
                    enable_kv_cache_events=self.enable_kv_cache_events,
791
                    eviction_policy=server_args.radix_eviction_policy,
Ke Bao's avatar
Ke Bao committed
792
                    is_eagle=self.spec_algorithm.is_eagle(),
793
794
                )

795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
        if (
            server_args.disaggregation_mode == "decode"
            and server_args.disaggregation_decode_enable_offload_kvcache
        ):
            self.decode_offload_manager = DecodeKVCacheOffloadManager(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                tp_group=(
                    self.attn_tp_cpu_group
                    if self.server_args.enable_dp_attention
                    else self.tp_cpu_group
                ),
                tree_cache=self.tree_cache,
                server_args=self.server_args,
            )
        else:
            self.decode_offload_manager = None

813
814
815
816
817
818
        self.decode_mem_cache_buf_multiplier = (
            1
            if self.spec_algorithm.is_none()
            else (
                server_args.speculative_num_draft_tokens
                + (
819
820
                    (server_args.speculative_eagle_topk or 1)
                    * (server_args.speculative_num_steps or 1)
821
822
                )
            )
823
        )
824

825
826
827
        embedding_cache_size = int(os.environ.get("SGLANG_VLM_CACHE_SIZE_MB", "100"))
        init_embedding_cache(embedding_cache_size * 1024 * 1024)

Byron Hsu's avatar
Byron Hsu committed
828
    def init_disaggregation(self):
Lianmin Zheng's avatar
Lianmin Zheng committed
829
830
831
        self.disaggregation_mode = DisaggregationMode(
            self.server_args.disaggregation_mode
        )
832
833
834
835
        self.transfer_backend = TransferBackend(
            self.server_args.disaggregation_transfer_backend
        )

Byron Hsu's avatar
Byron Hsu committed
836
837
838
839
        if (
            self.disaggregation_mode == DisaggregationMode.DECODE
        ):  # *2 for the headroom.
            buffer_size = (self.req_to_token_pool.size) * 2
Byron Hsu's avatar
Byron Hsu committed
840
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
841
842
                buffer_size
            )
843
844
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
845
                hidden_size=self.model_config.hf_text_config.hidden_size,
846
                hidden_states_dtype=self.model_config.dtype,
847
848
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
849
850
851

            # The decode requests polling kv cache
            self.disagg_decode_transfer_queue = DecodeTransferQueue(
852
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
853
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
854
                tp_rank=self.tp_rank,
855
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
856
857
                scheduler=self,
                tree_cache=self.tree_cache,
Byron Hsu's avatar
Byron Hsu committed
858
859
860
861
862
863
            )

            # The decode requests pending for pre-allocation
            self.disagg_decode_prealloc_queue = DecodePreallocQueue(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Byron Hsu's avatar
Byron Hsu committed
864
865
                draft_token_to_kv_pool=(
                    None
866
                    if self.draft_worker is None or self.spec_algorithm.is_ngram()
Byron Hsu's avatar
Byron Hsu committed
867
868
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
869
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
870
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
871
872
873
                scheduler=self,
                transfer_queue=self.disagg_decode_transfer_queue,
                tree_cache=self.tree_cache,
874
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
875
876
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
877
878
                dp_size=self.server_args.dp_size,
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
879
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
880
881
                max_total_num_tokens=self.max_total_num_tokens,
                prefill_pp_size=self.server_args.disaggregation_prefill_pp,
882
                num_reserved_decode_tokens=self.server_args.num_reserved_decode_tokens,
883
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
884
            )
Liangsheng Yin's avatar
Liangsheng Yin committed
885

Byron Hsu's avatar
Byron Hsu committed
886
887
888
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            # *2 for the headroom.
            buffer_size = self.max_running_requests * 2
Byron Hsu's avatar
Byron Hsu committed
889
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
890
891
                buffer_size
            )
892
893
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
894
                hidden_size=self.model_config.hf_text_config.hidden_size,
895
                hidden_states_dtype=self.model_config.dtype,
896
897
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
898

Liangsheng Yin's avatar
Liangsheng Yin committed
899
            self.disagg_prefill_bootstrap_queue = PrefillBootstrapQueue(
Byron Hsu's avatar
Byron Hsu committed
900
                token_to_kv_pool=self.token_to_kv_pool_allocator.get_kvcache(),
Byron Hsu's avatar
Byron Hsu committed
901
902
                draft_token_to_kv_pool=(
                    None
903
                    if self.draft_worker is None or self.spec_algorithm.is_ngram()
Byron Hsu's avatar
Byron Hsu committed
904
905
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
906
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
907
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
908
909
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
Byron Hsu's avatar
Byron Hsu committed
910
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
911
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
912
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
913
914
915
                max_total_num_tokens=self.max_total_num_tokens,
                decode_tp_size=self.server_args.disaggregation_decode_tp,
                decode_dp_size=self.server_args.disaggregation_decode_dp,
916
                scheduler=self,
Byron Hsu's avatar
Byron Hsu committed
917
918
919
                pp_rank=self.pp_rank,
                pp_size=self.pp_size,
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
920
921
            )
            # The prefill requests that are in the middle of kv sending
922
            self.disagg_prefill_inflight_queue: List[Req] = []
Byron Hsu's avatar
Byron Hsu committed
923

924
925
926
927
928
929
930
931
932
933
934
935
936
    def init_overlap(self):
        if not self.enable_overlap:
            return

        self.forward_stream: CudaStream = torch.get_device_module(self.device).Stream()
        self.forward_stream_ctx: CudaStreamContext = torch.get_device_module(
            self.device
        ).stream(self.forward_stream)
        self.copy_stream: CudaStream = torch.get_device_module(self.device).Stream()
        self.copy_stream_ctx: CudaStreamContext = torch.get_device_module(
            self.device
        ).stream(self.copy_stream)

937
938
939
        self.future_map = FutureMap(
            self.max_running_requests, self.device, self.spec_algorithm
        )
940
941
942
943
944
945
946
947
948
949
950
951
        self.batch_record_buf = [None] * 2
        self.batch_record_ct = 0

    def record_batch_in_overlap(self, model_worker_batch: ModelWorkerBatch):
        # FIXME(lsyin): hacky way to keep a reference to avoid GPU tensors being freed by torch GC
        # NOTE: More Reliable: record all tensors into the forward stream
        # NOTE: - for all future tensors, we shall always read from future map
        #       - for all non-future tensors (produced only by schedule stream),
        #       we shall keep its reference not being release during all the forwarding pass
        self.batch_record_ct = (self.batch_record_ct + 1) % 2
        self.batch_record_buf[self.batch_record_ct] = model_worker_batch

952
953
954
955
    def init_moe_config(self):
        if hasattr(self.model_config.hf_config, "num_experts_per_tok"):
            initialize_moe_config(self.server_args)

956
    @DynamicGradMode()
957
    def event_loop_normal(self):
958
        """A normal scheduler loop."""
959
        while True:
Lianmin Zheng's avatar
Lianmin Zheng committed
960
961
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)
962

963
            batch = self.get_next_batch_to_run()
Lianmin Zheng's avatar
Lianmin Zheng committed
964
            self.cur_batch = batch
965
966
967
968

            if batch:
                result = self.run_batch(batch)
                self.process_batch_result(batch, result)
Lianmin Zheng's avatar
Lianmin Zheng committed
969
            else:
Lianmin Zheng's avatar
Lianmin Zheng committed
970
                # When the server is idle, do self-check and re-init some states
971
                self.self_check_during_idle()
972
973

            self.last_batch = batch
974

975
    @DynamicGradMode()
Lianmin Zheng's avatar
Lianmin Zheng committed
976
    def event_loop_overlap(self):
977
        """A scheduler loop that overlaps the CPU processing and GPU computation."""
978
        self.result_queue: Deque[Tuple[ScheduleBatch, GenerationBatchResult]] = deque()
Lianmin Zheng's avatar
Lianmin Zheng committed
979
980
981
982
983
984
985

        while True:
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)

            batch = self.get_next_batch_to_run()
            self.cur_batch = batch
986

987
            batch_result = None
Lianmin Zheng's avatar
Lianmin Zheng committed
988
            if batch:
989
990
                batch_result = self.run_batch(batch)
                self.result_queue.append((batch.copy(), batch_result))
Lianmin Zheng's avatar
Lianmin Zheng committed
991
992

            if self.last_batch:
993
                # Process the results of the last batch
994
                tmp_batch, tmp_result = self.result_queue.popleft()
995
                self.process_batch_result(tmp_batch, tmp_result)
Lianmin Zheng's avatar
Lianmin Zheng committed
996
            elif batch is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
997
                # When the server is idle, do self-check and re-init some states
998
                self.self_check_during_idle()
Lianmin Zheng's avatar
Lianmin Zheng committed
999

1000
            self.launch_batch_sample_if_needed(batch_result)
Lianmin Zheng's avatar
Lianmin Zheng committed
1001
1002
            self.last_batch = batch

1003
1004
1005
            if envs.SGLANG_ENABLE_RUNTIME_MEM_LEAK_CHECK.get():
                self._check_runtime_mem_leak()

1006
1007
    def recv_requests(self) -> List[Req]:
        """Receive results at tp_rank = 0 and broadcast it to all other TP ranks."""
1008
1009
1010
1011
1012
1013
1014
1015

        if self.recv_skipper is not None:
            last_forward_mode = (
                self.last_batch.forward_mode if self.last_batch is not None else None
            )
            if not self.recv_skipper.handle(last_forward_mode):
                return []

1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
        if self.pp_rank == 0:
            if self.attn_tp_rank == 0:
                recv_reqs = []

                while True:
                    try:
                        recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_req)

                while True:
                    try:
                        recv_rpc = self.recv_from_rpc.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_rpc)
            else:
                recv_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1035
        else:
1036
            if self.attn_tp_rank == 0:
1037
                dp_offset = self.attn_dp_rank * self.attn_tp_size
1038
1039
1040
                recv_reqs = point_to_point_pyobj(
                    [],
                    self.pp_rank * self.tp_size + dp_offset,
1041
                    self.world_group.device_group,
1042
1043
1044
1045
1046
                    (self.pp_rank - 1) * self.tp_size + dp_offset,
                    self.pp_rank * self.tp_size + dp_offset,
                )
            else:
                recv_reqs = None
1047

fzyzcjy's avatar
fzyzcjy committed
1048
1049
1050
        if self.input_blocker is not None:
            recv_reqs = self.input_blocker.handle(recv_reqs)

1051
1052
1053
1054
1055
1056
        if self.server_args.enable_dp_attention:
            if self.attn_tp_rank == 0:
                work_reqs = [
                    req
                    for req in recv_reqs
                    if isinstance(
1057
1058
1059
1060
1061
1062
1063
                        req,
                        (
                            TokenizedGenerateReqInput,
                            TokenizedEmbeddingReqInput,
                            BatchTokenizedGenerateReqInput,
                            BatchTokenizedEmbeddingReqInput,
                        ),
1064
1065
1066
1067
1068
1069
                    )
                ]
                control_reqs = [
                    req
                    for req in recv_reqs
                    if not isinstance(
1070
1071
1072
1073
1074
1075
1076
                        req,
                        (
                            TokenizedGenerateReqInput,
                            TokenizedEmbeddingReqInput,
                            BatchTokenizedGenerateReqInput,
                            BatchTokenizedEmbeddingReqInput,
                        ),
1077
1078
1079
1080
1081
1082
1083
1084
1085
                    )
                ]
            else:
                work_reqs = None
                control_reqs = None

            if self.attn_tp_size != 1:
                work_reqs = broadcast_pyobj(
                    work_reqs,
1086
                    self.attn_tp_group.rank,
1087
                    self.attn_tp_cpu_group,
1088
                    src=self.attn_tp_group.ranks[0],
1089
1090
1091
                )
            if self.tp_size != 1:
                control_reqs = broadcast_pyobj(
1092
1093
1094
1095
                    control_reqs,
                    self.tp_group.rank,
                    self.tp_cpu_group,
                    src=self.tp_group.ranks[0],
1096
1097
1098
                )
            recv_reqs = work_reqs + control_reqs
        elif self.tp_size != 1:
1099
1100
1101
1102
1103
1104
            recv_reqs = broadcast_pyobj(
                recv_reqs,
                self.tp_group.rank,
                self.tp_cpu_group,
                src=self.tp_group.ranks[0],
            )
1105

1106
1107
1108
1109
1110
1111
1112
        if self.enable_trace:
            for req in recv_reqs:
                if isinstance(
                    req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                ):
                    trace_set_proc_propagate_context(req.rid, req.trace_context)
                    trace_slice_start("", req.rid, anonymous=True)
1113

1114
1115
        return recv_reqs

Lianmin Zheng's avatar
Lianmin Zheng committed
1116
    def process_input_requests(self, recv_reqs: List):
1117
        for recv_req in recv_reqs:
1118
1119
            # If it is a health check generation request and there are running requests, ignore it.
            if is_health_check_generate_req(recv_req) and (
1120
1121
1122
                self.chunked_req is not None
                or not self.running_batch.is_empty()
                or len(self.offload_tags) > 0
1123
1124
1125
1126
            ):
                self.return_health_check_ct += 1
                continue

1127
            output = self._request_dispatcher(recv_req)
1128
            if output is not None:
1129
1130
1131
1132
                if isinstance(output, RpcReqOutput):
                    if self.recv_from_rpc is not None:
                        self.recv_from_rpc.send_pyobj(output)
                else:
1133
                    self.send_to_tokenizer.send_output(output, recv_req)
1134

1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
    def init_req_max_new_tokens(self, req):
        req.sampling_params.max_new_tokens = min(
            (
                req.sampling_params.max_new_tokens
                if req.sampling_params.max_new_tokens is not None
                else 1 << 30
            ),
            self.max_req_len - len(req.origin_input_ids) - 1,
        )

1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
    def _process_and_broadcast_mm_inputs(
        self,
        raw_mm_inputs: Optional[dict],
    ):
        """Materialize MultimodalInputs once on the entry rank and broadcast to others.

        Entry rank:
        - constructs MultimodalInputs.from_dict(raw_mm_inputs) once
        - broadcasts to other ranks in self.cpu_group (if world_size > 1)

        Non-entry ranks:
        - receive the object via broadcast (if world_size > 1)
        - otherwise (single-rank / no group) fall back to local from_dict

        Returns:
            MultimodalInputs | None
        """
        if raw_mm_inputs is None:
            return None

        group_world_size = 1
        try:
            if (
                torch.distributed.is_available()
                and torch.distributed.is_initialized()
                and self.cpu_group is not None
            ):
                group_world_size = torch.distributed.get_world_size(
                    group=self.cpu_group
                )
        except Exception as e:
            logger.warning(
                f"Failed to get world size in mm_inputs handling with {e}, fallback to 1."
            )

        # In case tp size > 1, all the Scheduler TP ranks runs the duplicated computing
        # process in CPU which occupies the main thread CPU cycle. This computing logic
        # merely needs to be run on TP0 and be broadcast to other TP ranks.
        # Since the Scheduler is single-threaded, any large CPU cost will impact
        # handling of other messages. For example, CPU hits 99.9% can significantly
        # increase the CUDA kernel launch time.
        if self.is_entry_rank:
            # Only the entry rank materializes once from dict.
            image_inputs = MultimodalInputs.from_dict(raw_mm_inputs)
            # Broadcast to other TP ranks (use src=0 within the group).
            if group_world_size > 1:
                obj_list = [image_inputs]
                torch.distributed.broadcast_object_list(
                    obj_list, src=0, group=self.cpu_group
                )
                image_inputs = obj_list[0]
        else:
            # Non-entry ranks: receive if group size > 1; otherwise materialize locally.
            if group_world_size > 1:
                obj_list = [None]
                torch.distributed.broadcast_object_list(
                    obj_list, src=0, group=self.cpu_group
                )
                image_inputs = obj_list[0]
            else:
                image_inputs = MultimodalInputs.from_dict(raw_mm_inputs)

        return image_inputs

1209
1210
1211
1212
    def handle_generate_request(
        self,
        recv_req: TokenizedGenerateReqInput,
    ):
1213
        # Create a new request
1214
1215
1216
1217
1218
        if (
            recv_req.session_params is None
            or recv_req.session_params.id is None
            or recv_req.session_params.id not in self.sessions
        ):
Rin Intachuen's avatar
Rin Intachuen committed
1219
1220
1221
1222
1223
1224
            if recv_req.input_embeds is not None:
                # Generate fake input_ids based on the length of input_embeds
                seq_length = len(recv_req.input_embeds)
                fake_input_ids = [1] * seq_length
                recv_req.input_ids = fake_input_ids

1225
1226
1227
1228
            if recv_req.bootstrap_port is None:
                # Use default bootstrap port
                recv_req.bootstrap_port = self.server_args.disaggregation_bootstrap_port

1229
1230
1231
1232
1233
            req = Req(
                recv_req.rid,
                recv_req.input_text,
                recv_req.input_ids,
                recv_req.sampling_params,
Lianmin Zheng's avatar
Lianmin Zheng committed
1234
1235
                return_logprob=recv_req.return_logprob,
                top_logprobs_num=recv_req.top_logprobs_num,
1236
                token_ids_logprob=recv_req.token_ids_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1237
                stream=recv_req.stream,
1238
                lora_id=recv_req.lora_id,
Rin Intachuen's avatar
Rin Intachuen committed
1239
                input_embeds=recv_req.input_embeds,
Lianmin Zheng's avatar
Lianmin Zheng committed
1240
                custom_logit_processor=recv_req.custom_logit_processor,
1241
                return_hidden_states=recv_req.return_hidden_states,
1242
                eos_token_ids=self.model_config.hf_eos_token_id,
1243
                bootstrap_host=recv_req.bootstrap_host,
1244
                bootstrap_port=recv_req.bootstrap_port,
1245
                bootstrap_room=recv_req.bootstrap_room,
1246
                disagg_mode=self.disaggregation_mode,
1247
                data_parallel_rank=recv_req.data_parallel_rank,
1248
                vocab_size=self.model_config.vocab_size,
1249
                priority=recv_req.priority,
1250
1251
1252
                metrics_collector=(
                    self.metrics_collector if self.enable_metrics else None
                ),
1253
                http_worker_ipc=recv_req.http_worker_ipc,
1254
1255
            )
            req.tokenizer = self.tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
1256

1257
1258
1259
            if self.disaggregation_mode != DisaggregationMode.NULL:
                # Invalid request for disaggregated mode
                if recv_req.bootstrap_room is None:
1260
                    error_msg = (
1261
1262
1263
                        f"Invalid request: Disaggregated request received without "
                        f"boostrap room id. {req.rid=}"
                    )
1264
                    logger.error(error_msg)
1265
                    prepare_abort(req, error_msg, status_code=HTTPStatus.BAD_REQUEST)
1266
1267
1268
                    self.stream_output([req], req.return_logprob)
                    return

1269
1270
1271
1272
            if (
                recv_req.session_params is not None
                and recv_req.session_params.id is not None
            ):
1273
                req.set_finish_with_abort(
1274
                    f"Invalid request: session id {recv_req.session_params.id} does not exist"
1275
                )
1276
                self.init_req_max_new_tokens(req)
1277
                self._add_request_to_queue(req)
1278
1279
                return
        else:
1280
1281
            # Create a new request from a previous session
            session = self.sessions[recv_req.session_params.id]
1282
            req = session.create_req(recv_req, self.tokenizer)
1283
            if isinstance(req.finished_reason, FINISH_ABORT):
1284
                self.init_req_max_new_tokens(req)
1285
                self._add_request_to_queue(req)
1286
                return
1287

1288
        # Handle multimodal inputs
Mick's avatar
Mick committed
1289
        if recv_req.mm_inputs is not None:
1290
1291
1292
            image_inputs = self._process_and_broadcast_mm_inputs(recv_req.mm_inputs)

            # The following steps are already fast, execute locally on each rank.
1293
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
1294
            req.origin_input_ids = self.pad_input_ids_func(
1295
                req.origin_input_ids, image_inputs
1296
            )
1297
            req.extend_image_inputs(image_inputs)
1298

1299
            if len(req.origin_input_ids) >= self.max_req_input_len:
1300
1301
1302
1303
1304
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1305
                )
1306
                self.init_req_max_new_tokens(req)
1307
                self._add_request_to_queue(req)
1308
1309
                return

1310
1311
1312
        # initialize before returning
        self.init_req_max_new_tokens(req)

1313
        # Validate prompt length
1314
1315
1316
1317
1318
1319
        error_msg = validate_input_length(
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
        if error_msg:
1320
            req.set_finish_with_abort(error_msg)
1321
            self._add_request_to_queue(req)
1322
            return
1323

1324
        # Copy more attributes
1325
        if recv_req.logprob_start_len == -1 or not recv_req.return_logprob:
1326
            # By default, only return the logprobs for output tokens
1327
1328
1329
1330
1331
1332
1333
            # For prefill-only requests with logprob_start_len == -1, set logprob_start_len beyond input sequence
            # to skip input logprob computation entirely
            if req.is_prefill_only:
                req.logprob_start_len = len(req.origin_input_ids)
            else:
                # TODO: For text generation, evaluate setting logprob_start_len to len(req.origin_input_ids) as well
                req.logprob_start_len = len(req.origin_input_ids) - 1
1334
1335
1336
        else:
            req.logprob_start_len = recv_req.logprob_start_len

1337
1338
1339
        if not req.is_prefill_only and req.logprob_start_len >= len(
            req.origin_input_ids
        ):
1340
            error_msg = f"{req.logprob_start_len=} is higher than the number of input tokens {len(req.origin_input_ids)=}. Please use a smaller logprob_start_len."
1341
            req.logprob_start_len = len(req.origin_input_ids) - 1
1342
            req.set_finish_with_abort(error_msg)
1343
1344
1345
            self._add_request_to_queue(req)
            return

1346
1347
1348
1349
1350
        # Init grammar cache for this request
        add_to_grammar_queue = False
        if (
            req.sampling_params.json_schema is not None
            or req.sampling_params.regex is not None
1351
            or req.sampling_params.ebnf is not None
1352
            or req.sampling_params.structural_tag is not None
1353
        ):
1354
1355
1356
            if self.grammar_backend is None:
                error_msg = "Grammar-based generation (json_schema, regex, ebnf, structural_tag) is not supported when the server is launched with --grammar-backend none"
                req.set_finish_with_abort(error_msg)
1357
            else:
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
                if req.sampling_params.json_schema is not None:
                    key = ("json", req.sampling_params.json_schema)
                elif req.sampling_params.regex is not None:
                    key = ("regex", req.sampling_params.regex)
                elif req.sampling_params.ebnf is not None:
                    key = ("ebnf", req.sampling_params.ebnf)
                elif req.sampling_params.structural_tag:
                    key = ("structural_tag", req.sampling_params.structural_tag)

                value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
                req.grammar = value

                if not cache_hit:
                    req.grammar_key = key
                    add_to_grammar_queue = True
                else:
                    if value is INVALID_GRAMMAR_OBJ:  # We hit a cached invalid grammar.
                        error_msg = f"Invalid grammar request with cache hit: {key=}"
                        req.set_finish_with_abort(error_msg)
1377
1378

        if add_to_grammar_queue:
1379
1380
            self.grammar_queue.append(req)
        else:
1381
1382
            self._add_request_to_queue(req)

1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
    def handle_batch_generate_request(
        self,
        recv_req: BatchTokenizedGenerateReqInput,
    ):
        """Handle optimized batch generate request."""
        logger.debug(f"Processing batch generate request with {len(recv_req)} requests")

        # Process each request in the batch
        for tokenized_req in recv_req:
            self.handle_generate_request(tokenized_req)

1394
1395
1396
    def _prefetch_kvcache(self, req: Req):
        if self.enable_hicache_storage:
            req.init_next_round_input(self.tree_cache)
1397
1398
1399
1400
1401
            if req.last_node.backuped:
                # only to initiate the prefetch if the last node is backuped
                # otherwise, the allocated GPU memory must be locked for integrity
                last_hash = req.last_host_node.get_last_hash_value()
                matched_len = len(req.prefix_indices) + req.host_hit_length
1402
                new_input_tokens = req.fill_ids[matched_len:]
1403
1404
1405
1406
1407
1408

                prefix_keys = (
                    req.last_node.get_prefix_hash_values(req.last_node.parent)
                    if self.tree_cache.hicache_storage_pass_prefix_keys
                    else None
                )
1409
                self.tree_cache.prefetch_from_storage(
1410
1411
1412
1413
1414
                    req.rid,
                    req.last_host_node,
                    new_input_tokens,
                    last_hash,
                    prefix_keys,
1415
1416
                )

1417
1418
    def _add_request_to_queue(self, req: Req, is_retracted: bool = False):
        if self.disaggregation_mode == DisaggregationMode.NULL:
1419
1420
            if not self._set_or_validate_priority(req):
                return
1421
1422
1423
1424
1425
            if self._abort_on_queued_limit(req):
                return
            self._prefetch_kvcache(req)
            self.waiting_queue.append(req)
            req.time_stats.wait_queue_entry_time = time.perf_counter()
1426
            trace_slice_end(RequestStage.REQUEST_PROCESS, req.rid, auto_next_anon=True)
1427
1428
1429
1430
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            self._prefetch_kvcache(req)
            self.disagg_prefill_bootstrap_queue.add(
                req, self.model_config.num_key_value_heads
Byron Hsu's avatar
Byron Hsu committed
1431
            )
1432
            req.time_stats.prefill_bootstrap_queue_entry_time = time.perf_counter()
1433
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
1434
1435
1436
            self.disagg_decode_prealloc_queue.add(req, is_retracted=is_retracted)
            if not is_retracted:
                req.time_stats.decode_prealloc_queue_entry_time = time.perf_counter()
Byron Hsu's avatar
Byron Hsu committed
1437
        else:
1438
            raise ValueError(f"Invalid {self.disaggregation_mode=}")
1439

1440
    def _set_or_validate_priority(self, req: Req) -> bool:
1441
1442
1443
1444
1445
1446
        """Set the default priority value, or abort the request based on the priority scheduling mode."""
        if self.enable_priority_scheduling and req.priority is None:
            if self.schedule_low_priority_values_first:
                req.priority = sys.maxsize
            else:
                req.priority = -sys.maxsize - 1
1447
1448
1449
1450
1451
        elif (
            not self.enable_priority_scheduling
            and req.priority is not None
            and self.abort_on_priority_when_disabled
        ):
1452
1453
1454
1455
1456
1457
            abort_req = AbortReq(
                finished_reason={
                    "type": "abort",
                    "status_code": HTTPStatus.SERVICE_UNAVAILABLE,
                    "message": "Using priority is disabled for this server. Please send a new request without a priority.",
                },
1458
                rid=req.rid,
1459
            )
1460
            self.send_to_tokenizer.send_output(abort_req, req)
1461
1462
            return False
        return True
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482

    def _abort_on_queued_limit(self, recv_req: Req) -> bool:
        """Abort an incoming or existing request if the waiting queue is full. Returns True if the incoming request is aborted."""
        if (
            self.max_queued_requests is None
            or len(self.waiting_queue) + 1 <= self.max_queued_requests
        ):
            return False

        # Reject the incoming request by default.
        req_to_abort = recv_req
        message = "The request queue is full."
        if self.enable_priority_scheduling:
            # With priority scheduling, consider aboritng an existing request based on the priority.
            # direction = 1  => smaller number = higher priority; -1 => larger number = higher priority.
            # max(...) + (direction * priority, queue_time_start) picks the least-preferred request.
            # Tie: later queue_time_start (newer) is evicted first. Preempt only if strictly better.
            direction = 1 if self.schedule_low_priority_values_first else -1
            key_fn = lambda item: (
                direction * item[1].priority,
1483
                item[1].time_stats.wait_queue_entry_time,
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
            )
            idx, candidate_req = max(enumerate(self.waiting_queue), key=key_fn)
            abort_existing_req = (
                direction * recv_req.priority < direction * candidate_req.priority
            )
            if abort_existing_req:
                self.waiting_queue.pop(idx)
                req_to_abort = candidate_req
                message = "The request is aborted by a higher priority request."

1494
        self.send_to_tokenizer.send_output(
1495
1496
1497
1498
1499
1500
            AbortReq(
                finished_reason={
                    "type": "abort",
                    "status_code": HTTPStatus.SERVICE_UNAVAILABLE,
                    "message": message,
                },
1501
                rid=req_to_abort.rid,
1502
1503
            ),
            req_to_abort,
1504
1505
        )
        return req_to_abort.rid == recv_req.rid
1506
1507
1508

    def handle_embedding_request(
        self,
1509
        recv_req: TokenizedEmbeddingReqInput,
1510
1511
1512
1513
1514
1515
    ):
        req = Req(
            recv_req.rid,
            recv_req.input_text,
            recv_req.input_ids,
            recv_req.sampling_params,
woodx's avatar
woodx committed
1516
            token_type_ids=recv_req.token_type_ids,
1517
            priority=recv_req.priority,
1518
            dimensions=recv_req.dimensions,
1519
            http_worker_ipc=recv_req.http_worker_ipc,
1520
1521
1522
        )
        req.tokenizer = self.tokenizer

1523
1524
        # Handle multimodal inputs
        if recv_req.image_inputs is not None:
1525
            image_inputs = self._process_and_broadcast_mm_inputs(recv_req.image_inputs)
1526
1527
1528
1529
1530
1531
1532
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
            req.origin_input_ids = self.pad_input_ids_func(
                req.origin_input_ids, image_inputs
            )
            req.extend_image_inputs(image_inputs)

            if len(req.origin_input_ids) >= self.max_req_input_len:
1533
1534
1535
1536
1537
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1538
                )
1539
                self._add_request_to_queue(req)
1540
1541
                return

1542
        # Validate prompts length
1543
        error_msg = validate_input_length(
1544
1545
1546
1547
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
1548
        if error_msg:
1549
            self._add_request_to_queue(req)
1550
            return
1551

1552
1553
        # Copy more attributes
        req.logprob_start_len = len(req.origin_input_ids) - 1
1554
        self._add_request_to_queue(req)
1555

1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
    def handle_batch_embedding_request(
        self,
        recv_req: BatchTokenizedEmbeddingReqInput,
    ):
        """Handle optimized batch embedding request."""
        logger.debug(
            f"Processing batch embedding request with {len(recv_req)} requests"
        )

        # Process each request in the batch
        for tokenized_req in recv_req:
            self.handle_embedding_request(tokenized_req)

Hanming Lu's avatar
Hanming Lu committed
1569
1570
1571
1572
1573
1574
1575
    def _get_token_info(self):
        available_size = self.token_to_kv_pool_allocator.available_size()
        evictable_size = self.tree_cache.evictable_size()
        num_used = self.max_total_num_tokens - (available_size + evictable_size)
        token_usage = num_used / self.max_total_num_tokens
        return num_used, token_usage, available_size, evictable_size

1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
    def _get_mamba_token_info(self):
        is_radix_tree = isinstance(self.tree_cache, MambaRadixCache)
        full_available_size = self.token_to_kv_pool_allocator.available_size()
        full_evictable_size = (
            self.tree_cache.full_evictable_size() if is_radix_tree else 0
        )
        mamba_available_size = self.req_to_token_pool.mamba_pool.available_size()
        mamba_evictable_size = (
            self.tree_cache.mamba_evictable_size() if is_radix_tree else 0
        )
        full_num_used = self.token_to_kv_pool_allocator.size - (
            full_available_size + full_evictable_size
        )
        mamba_num_used = self.req_to_token_pool.mamba_pool.size - (
            mamba_available_size + mamba_evictable_size
        )
        full_token_usage = full_num_used / self.token_to_kv_pool_allocator.size
        mamba_usage = mamba_num_used / self.req_to_token_pool.mamba_pool.size
        return (
            full_num_used,
            mamba_num_used,
            full_token_usage,
            mamba_usage,
            full_available_size,
            full_evictable_size,
            mamba_available_size,
            mamba_evictable_size,
        )

Hanming Lu's avatar
Hanming Lu committed
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
    def _get_swa_token_info(self):
        full_available_size = self.token_to_kv_pool_allocator.full_available_size()
        full_evictable_size = self.tree_cache.full_evictable_size()
        swa_available_size = self.token_to_kv_pool_allocator.swa_available_size()
        swa_evictable_size = self.tree_cache.swa_evictable_size()
        full_num_used = self.full_tokens_per_layer - (
            full_available_size + full_evictable_size
        )
        swa_num_used = self.swa_tokens_per_layer - (
            swa_available_size + swa_evictable_size
        )
        full_token_usage = full_num_used / self.full_tokens_per_layer
        swa_token_usage = swa_num_used / self.swa_tokens_per_layer
        return (
            full_num_used,
            swa_num_used,
            full_token_usage,
            swa_token_usage,
            full_available_size,
            full_evictable_size,
            swa_available_size,
            swa_evictable_size,
        )

1629
    def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
1630
        # Merge the prefill batch into the running batch
1631
1632
1633
1634
1635
        chunked_req_to_exclude = set()
        if self.chunked_req:
            # Move the chunked request out of the batch so that we can merge
            # only finished requests to running_batch.
            chunked_req_to_exclude.add(self.chunked_req)
1636
            self.tree_cache.cache_unfinished_req(self.chunked_req, chunked=True)
1637
            # chunked request keeps its rid but will get a new req_pool_idx
1638
            if self.tp_worker.model_runner.mambaish_config is not None:
Yi Zhang's avatar
Yi Zhang committed
1639
1640
1641
1642
1643
                self.req_to_token_pool.free(
                    self.chunked_req.req_pool_idx, free_mamba_cache=False
                )
            else:
                self.req_to_token_pool.free(self.chunked_req.req_pool_idx)
Lianmin Zheng's avatar
Lianmin Zheng committed
1644
        if self.last_batch and self.last_batch.forward_mode.is_extend():
1645
1646
1647
1648
            if self.last_batch.chunked_req is not None:
                # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
                # We need to discard it.
                chunked_req_to_exclude.add(self.last_batch.chunked_req)
Lianmin Zheng's avatar
Lianmin Zheng committed
1649

1650
            # Filter batch
1651
            last_bs = self.last_batch.batch_size()
1652
1653
1654
            self.last_batch.filter_batch(
                chunked_req_to_exclude=list(chunked_req_to_exclude)
            )
1655
            if self.last_batch.batch_size() < last_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1656
                self.running_batch.batch_is_full = False
1657

1658
1659
1660
            # Merge the new batch into the running batch.
            # For prefill-only batch, we can avoid going through decoding step.
            if not self.last_batch.is_empty() and not self.last_batch.is_prefill_only:
Lianmin Zheng's avatar
Lianmin Zheng committed
1661
                if self.running_batch.is_empty():
1662
1663
                    self.running_batch = self.last_batch
                else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1664
                    # Merge running_batch with prefill batch
1665
                    self.running_batch.merge_batch(self.last_batch)
1666

1667
        new_batch = self.get_new_batch_prefill()
1668

1669
1670
1671
1672
1673
        need_dp_attn_preparation = require_mlp_sync(self.server_args)

        if need_dp_attn_preparation and not self.spec_algorithm.is_none():
            # In speculative decoding, prefill batches and decode batches cannot be processed in the same DP attention group.
            # We prepare idle batches in advance to skip preparing decode batches when there are prefill batches in the group.
1674
            new_batch = self.prepare_mlp_sync_batch(new_batch)
1675
1676
1677
            need_dp_attn_preparation = new_batch is None

        if new_batch is not None:
1678
1679
1680
1681
            # Run prefill first if possible
            ret = new_batch
        else:
            # Run decode
Lianmin Zheng's avatar
Lianmin Zheng committed
1682
            if not self.running_batch.is_empty():
1683
                self.running_batch = self.update_running_batch(self.running_batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1684
1685
1686
                ret = self.running_batch if not self.running_batch.is_empty() else None
            else:
                ret = None
1687

1688
1689
        # Handle DP attention
        if need_dp_attn_preparation:
1690
            ret = self.prepare_mlp_sync_batch(ret)
1691

1692
1693
1694
1695
        if ret:
            attrs = {"bid": hex(id(ret)), "batch_size": ret.batch_size()}
            trace_event_batch("schedule", ret.reqs, attrs=attrs)

1696
        return ret
1697

1698
    def get_num_allocatable_reqs(self, running_bs):
1699
        res = get_global_server_args().pp_max_micro_batch_size - running_bs
1700
1701
1702
1703
        if self.pp_size > 1:
            res = min(res, self.req_to_token_pool.available_size())
        return res

Lianmin Zheng's avatar
Lianmin Zheng committed
1704
    def get_new_batch_prefill(self) -> Optional[ScheduleBatch]:
Lianmin Zheng's avatar
Lianmin Zheng committed
1705
        # Check if the grammar is ready in the grammar queue
1706
        if self.grammar_queue:
1707
            self.move_ready_grammar_requests()
1708

1709
1710
1711
1712
        if self.try_preemption:
            # Reset batch_is_full to try preemption with a prefill adder.
            self.running_batch.batch_is_full = False

Lianmin Zheng's avatar
Lianmin Zheng committed
1713
1714
        # Handle the cases where prefill is not allowed
        if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1715
            self.running_batch.batch_is_full or len(self.waiting_queue) == 0
1716
        ) and self.chunked_req is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
1717
1718
            return None

Lianmin Zheng's avatar
Lianmin Zheng committed
1719
        running_bs = len(self.running_batch.reqs)
1720
        # Ignore the check if self.chunked_req is not None.
1721
1722
1723
1724
        # In the non-PP case, when self.chunked_req is not None, num_allocatable_reqs should always be greater than 0,
        # as the space for the chunked request has just been released.
        # In PP case, a chunked req can start in one microbatch and end in another microbatch, so the max_running_requests per microbatch should not be strict.
        # Instead, we should always allow chunked request to be added, otherwise, there will be a memory leak.
1725
1726
1727
1728
1729
        if (
            self.get_num_allocatable_reqs(running_bs) <= 0
            and not self.chunked_req
            and not self.try_preemption
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1730
            self.running_batch.batch_is_full = True
1731
1732
            return None

1733
        if self.enable_hierarchical_cache:
1734
            self.tree_cache.check_hicache_events()
1735

1736
        # Get priority queue
1737
        self.policy.calc_priority(self.waiting_queue)
1738

1739
1740
1741
1742
1743
1744
        if TEST_RETRACT and running_bs > TEST_RETRACT_NO_PREFILL_BS:
            # If we are testing retraction and the running batch size exceeds
            # TEST_RETRACT_NO_PREFILL_BS, we skip the prefill to keep the requests
            # in the waiting queue.
            return None

Lianmin Zheng's avatar
Lianmin Zheng committed
1745
        # Prefill policy
1746
        adder = PrefillAdder(
1747
            self.page_size,
1748
            self.tree_cache,
1749
            self.token_to_kv_pool_allocator,
1750
1751
1752
1753
            self.running_batch,
            self.new_token_ratio,
            self.max_prefill_tokens,
            self.chunked_prefill_size,
1754
            running_bs if self.is_mixed_chunk else 0,
1755
            self.priority_scheduling_preemption_threshold,
1756
1757
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1758
        if self.chunked_req is not None:
1759
1760
            self.chunked_req.init_next_round_input()
            self.chunked_req = adder.add_chunked_req(self.chunked_req)
1761

1762
        if self.enable_lora:
1763
            lora_set = set([req.lora_id for req in self.running_batch.reqs])
Lianmin Zheng's avatar
Lianmin Zheng committed
1764

1765
        # Get requests from the waiting queue to a new prefill batch
1766
        for req in self.waiting_queue:
1767
1768
1769
1770
1771

            if self.enable_lora and not self.tp_worker.can_run_lora_batch(
                lora_set
                | set([req.lora_id for req in adder.can_run_list])
                | set([req.lora_id])
1772
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1773
                self.running_batch.batch_is_full = True
1774
1775
                break

1776
            running_bs = len(self.running_batch.reqs) - len(adder.preempt_list)
1777
            if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):
Lianmin Zheng's avatar
Lianmin Zheng committed
1778
                self.running_batch.batch_is_full = True
Byron Hsu's avatar
Byron Hsu committed
1779
1780
1781
1782
1783
            if self.disaggregation_mode == DisaggregationMode.PREFILL:
                # In prefill mode, prealloc queue and transfer queue can also take memory,
                # so we need to check if the available size for the actual available size.
                if len(adder.can_run_list) >= self.req_to_token_pool.available_size():
                    self.running_batch.batch_is_full = True
1784
1785
1786
1787
1788

            if self.running_batch.batch_is_full:
                if not self.try_preemption:
                    break
                if not adder.preempt_to_schedule(req, self.server_args):
Byron Hsu's avatar
Byron Hsu committed
1789
1790
                    break

1791
            if self.enable_hicache_storage:
pansicheng's avatar
pansicheng committed
1792
1793
1794
1795
                prefetch_done = self.tree_cache.check_prefetch_progress(req.rid)
                if not prefetch_done:
                    # skip staging requests that are ongoing prefetch
                    continue
1796

1797
            req.init_next_round_input(self.tree_cache)
1798
1799
1800
1801
1802
            res = adder.add_one_req(
                req,
                has_chunked_req=(self.chunked_req is not None),
                truncation_align_size=self.truncation_align_size,
            )
1803

1804
1805
            if res != AddReqResult.CONTINUE:
                if res == AddReqResult.NO_TOKEN:
1806
1807
                    if self.enable_hierarchical_cache:
                        # Set batch_is_full after making sure there are requests that can be served
Lianmin Zheng's avatar
Lianmin Zheng committed
1808
1809
                        self.running_batch.batch_is_full = len(
                            adder.can_run_list
1810
                        ) > 0 or (not self.running_batch.is_empty())
1811
                    else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1812
                        self.running_batch.batch_is_full = True
1813
1814
                break

Lianmin Zheng's avatar
Lianmin Zheng committed
1815
        # Update waiting queue
1816
        can_run_list: List[Req] = adder.can_run_list
Lianmin Zheng's avatar
Lianmin Zheng committed
1817
1818
        if len(can_run_list) == 0:
            return None
1819
1820
1821
1822

        if self.enable_metrics:
            # only record queue time when enable_metrics is True to avoid overhead
            for req in can_run_list:
1823
                req.add_latency(RequestStage.PREFILL_WAITING)
1824

Lianmin Zheng's avatar
Lianmin Zheng committed
1825
1826
1827
        self.waiting_queue = [
            x for x in self.waiting_queue if x not in set(can_run_list)
        ]
1828
        if adder.preempt_list:
1829
1830
            for req in adder.preempt_list:
                self._add_request_to_queue(req)
1831

1832
1833
1834
        if adder.new_chunked_req is not None:
            assert self.chunked_req is None
            self.chunked_req = adder.new_chunked_req
1835

1836
1837
        if self.chunked_req:
            self.chunked_req.is_chunked += 1
Lianmin Zheng's avatar
Lianmin Zheng committed
1838

1839
        # Print stats
1840
        if self.current_scheduler_metrics_enabled():
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
            self.log_prefill_stats(adder, can_run_list, running_bs, 0)

        for req in can_run_list:
            if req.time_stats.forward_entry_time == 0:
                # Avoid update chunked request many times
                req.time_stats.forward_entry_time = time.perf_counter()
                if self.enable_metrics:
                    self.metrics_collector.observe_queue_time(
                        req.time_stats.get_queueing_time(),
                    )
1851

Lianmin Zheng's avatar
Lianmin Zheng committed
1852
        # Create a new batch
1853
1854
1855
        new_batch = ScheduleBatch.init_new(
            can_run_list,
            self.req_to_token_pool,
1856
            self.token_to_kv_pool_allocator,
1857
            self.tree_cache,
1858
            self.model_config,
1859
            self.enable_overlap,
1860
            self.spec_algorithm,
1861
            chunked_req=self.chunked_req,
1862
        )
1863
1864
        if self.enable_hierarchical_cache:
            # todo (zhiqiang): disable cuda graph execution if hicache loading triggered
1865
1866
1867
            new_batch.hicache_consumer_index = (
                self.tree_cache.ready_to_load_host_cache()
            )
1868

1869
        new_batch.prepare_for_extend()
1870

Lianmin Zheng's avatar
Lianmin Zheng committed
1871
        # Mixed-style chunked prefill
1872
1873
        if (
            self.is_mixed_chunk
Lianmin Zheng's avatar
Lianmin Zheng committed
1874
            and not self.running_batch.is_empty()
1875
1876
1877
            and not (new_batch.return_logprob or self.running_batch.return_logprob)
        ):
            # TODO (lianmin): support return_logprob + mixed chunked prefill
1878
1879
            self.running_batch.filter_batch()
            if not self.running_batch.is_empty():
1880
                self.running_batch.prepare_for_decode()
1881
1882
                new_batch.mix_with_running(self.running_batch)
                new_batch.decoding_reqs = self.running_batch.reqs
Lianmin Zheng's avatar
Lianmin Zheng committed
1883
1884
1885
            self.running_batch = ScheduleBatch(
                reqs=[], batch_is_full=self.running_batch.batch_is_full
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1886
1887
        else:
            new_batch.decoding_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1888
1889
1890

        return new_batch

Lianmin Zheng's avatar
Lianmin Zheng committed
1891
    def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
1892
        """Update the current running decoding batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1893
        initial_bs = batch.batch_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1894

1895
1896
        batch.filter_batch()
        if batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1897
1898
            batch.batch_is_full = False
            return batch
1899

Lianmin Zheng's avatar
Lianmin Zheng committed
1900
        # Check if decode out of memory
1901
        if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or (
1902
            TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0
1903
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1904
            old_ratio = self.new_token_ratio
1905
1906
1907
            retracted_reqs, new_token_ratio, reqs_to_abort = batch.retract_decode(
                self.server_args
            )
1908
            self.num_retracted_reqs = len(retracted_reqs)
Lianmin Zheng's avatar
Lianmin Zheng committed
1909
            self.new_token_ratio = new_token_ratio
1910
1911
1912
1913
1914
            for req in reqs_to_abort:
                abort_reason: FINISH_ABORT = req.to_finish
                self.send_to_tokenizer.send_output(
                    AbortReq(abort_message=abort_reason.message, rid=req.rid), req
                )
1915

Lianmin Zheng's avatar
Lianmin Zheng committed
1916
            logger.info(
1917
                "KV cache pool is full. Retract requests. "
1918
1919
                f"#retracted_reqs: {len(retracted_reqs)}, "
                f"#new_token_ratio: {old_ratio:.4f} -> {new_token_ratio:.4f}"
Lianmin Zheng's avatar
Lianmin Zheng committed
1920
            )
1921

1922
1923
            for req in retracted_reqs:
                self._add_request_to_queue(req, is_retracted=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
1924
1925
        else:
            self.new_token_ratio = max(
1926
                self.new_token_ratio - self.new_token_ratio_decay,
Lianmin Zheng's avatar
Lianmin Zheng committed
1927
1928
1929
                self.min_new_token_ratio,
            )

Lianmin Zheng's avatar
Lianmin Zheng committed
1930
        if batch.batch_size() < initial_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1931
            batch.batch_is_full = False
Lianmin Zheng's avatar
Lianmin Zheng committed
1932
1933

        # Update batch tensors
1934
        batch.prepare_for_decode()
Lianmin Zheng's avatar
Lianmin Zheng committed
1935
        return batch
Lianmin Zheng's avatar
Lianmin Zheng committed
1936

1937
1938
1939
1940
1941
1942
    # placeholder for override
    def update_cache_from_scheduler(
        self, schedule_batch: ScheduleBatch, batch_result: GenerationBatchResult
    ):
        pass

1943
1944
1945
    def run_batch(
        self, batch: ScheduleBatch
    ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
1946
        """Run a batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1947
1948
        self.forward_ct += 1

1949
1950
        # Whether to run the profiler
        self._profile_batch_predicate(batch)
1951
1952
1953
1954
        if self.forward_sleep_time is not None:
            logger.info(f"Scheduler.run_batch sleep {self.forward_sleep_time}s")
            time.sleep(self.forward_sleep_time)

1955
        # Run forward
1956
        if self.is_generation:
1957
1958
            batch_or_worker_batch = batch

1959
            if self.enable_overlap or self.spec_algorithm.is_none():
1960
1961
                # FIXME(lsyin): remove this if and finally unify the abstraction
                batch_or_worker_batch = batch.get_model_worker_batch()
1962

1963
1964
1965
1966
1967
1968
1969
            if self.enable_overlap:
                # FIXME: remove this assert
                assert isinstance(batch_or_worker_batch, ModelWorkerBatch)
                model_worker_batch = batch_or_worker_batch
                self.record_batch_in_overlap(model_worker_batch)

                # Sampling info will be modified during forward
1970
                model_worker_batch.sampling_info = (
1971
1972
1973
1974
                    model_worker_batch.sampling_info.copy_for_forward()
                )

                bs = len(model_worker_batch.seq_lens)
1975
                future_indices = self.future_map.alloc_future_indices(bs)
1976
1977
1978
1979
1980

                with self.forward_stream_ctx:
                    self.forward_stream.wait_stream(self.default_stream)
                    self.future_map.resolve_future(model_worker_batch)
                    batch_result = self.model_worker.forward_batch_generation(
1981
                        model_worker_batch
1982
1983
1984
1985
1986
                    )
                    # FIXME(lsyin): maybe move this to forward_batch_generation
                    batch_result.copy_done = torch.get_device_module(
                        self.device
                    ).Event()
1987
                    if batch_result.delay_sample_func is None:
1988
                        self.future_map.store_to_map(future_indices, batch_result)
1989
1990
                        batch_result.copy_to_cpu()
                    else:
1991
                        batch_result.future_indices = future_indices
1992
1993

                # FIXME(lsyin): move this assignment elsewhere
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
                future_indices_or_next_token_ids = -future_indices.indices

                if batch.is_v2_eagle:
                    # FIXME(lsyin): tmp code for eagle v2
                    # We only keep future indices for next draft input

                    batch.spec_info = batch_result.next_draft_input
                    batch.spec_info.future_indices = future_indices

                    # batch.spec_info = EagleDraftInput(
                    #     future_indices=future_indices,
                    #     verify_done=batch_result.next_draft_input.verify_done,
                    #     # FIXME(lsyin): remove the allocate_lens in EagleDraftInput
                    #     allocate_lens=batch_result.next_draft_input.allocate_lens,
                    # )

                    # The future value, usually for next batch preparation
                    # Current implementation strictly synchronizes the seq_lens
                    batch.seq_lens = batch_result.next_draft_input.new_seq_lens
2013
2014
2015
            elif self.enable_pdmux and batch.forward_mode.is_split_prefill():
                batch_result = self.tp_worker.forward_batch_split_prefill(batch)
                future_indices_or_next_token_ids = batch_result.next_token_ids
2016
2017
2018
2019
            else:
                batch_result = self.model_worker.forward_batch_generation(
                    batch_or_worker_batch
                )
2020
                future_indices_or_next_token_ids = batch_result.next_token_ids
2021
                self.update_cache_from_scheduler(batch, batch_result)
2022

2023
            # NOTE: future_indices_or_next_token_ids is used in ScheduleBatch,
2024
2025
2026
            #       which can probably be replaced by future_indices later [TODO(lsyin)].
            #       we shall still keep the original outputs, e.g. next_token_ids
            #       in the GenerationBatchOutput for processing after copy_done.
2027
            batch.output_ids = future_indices_or_next_token_ids
2028

2029
2030
2031
            # These 2 values are needed for processing the output, but the values can be
            # modified by overlap schedule. So we have to copy them here so that
            # we can use the correct values in output processing.
2032
            if batch.return_logprob or self.spec_algorithm.is_eagle():
2033
                extend_input_len_per_req = [req.extend_input_len for req in batch.reqs]
2034
2035
            else:
                extend_input_len_per_req = None
2036

2037
            if batch.return_logprob:
2038
2039
2040
2041
2042
2043
                extend_logprob_start_len_per_req = [
                    req.extend_logprob_start_len for req in batch.reqs
                ]
            else:
                extend_logprob_start_len_per_req = None

2044
2045
2046
            batch_result.extend_input_len_per_req = extend_input_len_per_req
            batch_result.extend_logprob_start_len_per_req = (
                extend_logprob_start_len_per_req
2047
            )
2048
            return batch_result
Lianmin Zheng's avatar
Lianmin Zheng committed
2049
2050
2051
        else:  # embedding or reward model
            model_worker_batch = batch.get_model_worker_batch()
            embeddings = self.tp_worker.forward_batch_embedding(model_worker_batch)
2052
            ret = EmbeddingBatchResult(embeddings=embeddings)
2053
        return ret
Chayenne's avatar
Chayenne committed
2054

2055
2056
    def launch_batch_sample_if_needed(
        self, batch_result: GenerationBatchResult
2057
    ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
2058
2059
2060
        # TODO(lsyin): make the delayed sample a default behavior after
        # unifying the forward_batch_generation interface (related to spec V2).
        if batch_result is None or batch_result.delay_sample_func is None:
2061
2062
2063
2064
            return

        with self.forward_stream_ctx:
            self.forward_stream.wait_stream(self.default_stream)
2065
2066
2067
2068
            _batch_result = batch_result.delay_sample_func()
            assert _batch_result is batch_result
            self.future_map.store_to_map(batch_result.future_indices, batch_result)
            batch_result.copy_to_cpu()
2069

2070
2071
2072
2073
2074
    def process_batch_result(
        self,
        batch: ScheduleBatch,
        result: Union[GenerationBatchResult, EmbeddingBatchResult],
    ):
Lianmin Zheng's avatar
Lianmin Zheng committed
2075
        if batch.forward_mode.is_decode():
2076
            self.process_batch_result_decode(batch, result)
2077
            trace_slice_batch(RequestStage.DECODE_LOOP, batch.reqs)
2078

2079
        elif batch.forward_mode.is_extend():
2080
            self.process_batch_result_prefill(batch, result)
2081

2082
2083
        elif batch.forward_mode.is_idle():
            if self.enable_overlap:
2084
2085
                if result.copy_done is not None:
                    result.copy_done.synchronize()
Lianmin Zheng's avatar
Lianmin Zheng committed
2086

2087
2088
2089
        self.maybe_send_health_check_signal()

    def maybe_send_health_check_signal(self):
2090
2091
2092
2093
2094
        if self.return_health_check_ct:
            # Return some signal for the health check.
            # This is used to prevent the health check signal being blocked by long context prefill.
            # However, one minor issue is that this code path does not check the status of detokenizer manager.
            self.return_health_check_ct -= 1
2095
            self.send_to_tokenizer.send_output(HealthCheckOutput())
2096

2097
2098
    def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
        return self.prepare_mlp_sync_batch_raw(
2099
2100
2101
            local_batch,
            dp_size=self.server_args.dp_size,
            attn_tp_size=self.attn_tp_size,
2102
            tp_group=self.tp_group,
2103
2104
2105
2106
            get_idle_batch=self.get_idle_batch,
            disable_cuda_graph=self.server_args.disable_cuda_graph,
            spec_algorithm=self.spec_algorithm,
            speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens,
2107
            require_mlp_tp_gather=require_mlp_tp_gather(self.server_args),
2108
            disable_overlap_schedule=self.server_args.disable_overlap_schedule,
2109
            offload_tags=self.offload_tags,
2110
2111
2112
        )

    @staticmethod
2113
    def prepare_mlp_sync_batch_raw(
2114
2115
2116
        local_batch: ScheduleBatch,
        dp_size,
        attn_tp_size: int,
2117
        tp_group,
2118
2119
2120
2121
        get_idle_batch,
        disable_cuda_graph: bool,
        spec_algorithm,
        speculative_num_draft_tokens,
2122
        require_mlp_tp_gather: bool,
2123
        disable_overlap_schedule: bool,
2124
        offload_tags: set[str],
2125
    ):
2126
2127
2128
        # Check if other DP workers have running batches
        if local_batch is None:
            num_tokens = 0
2129
            num_tokens_for_logprob = 0
2130
2131
        elif local_batch.forward_mode.is_decode():
            num_tokens = local_batch.batch_size()
2132
            num_tokens_for_logprob = num_tokens
2133
2134
        else:
            num_tokens = local_batch.extend_num_tokens
2135
2136
            if local_batch.return_logprob:
                num_tokens_for_logprob = sum(
Lianmin Zheng's avatar
Lianmin Zheng committed
2137
2138
2139
                    # We should have at least 1 token for sample in every case.
                    max(extend_len - logprob_start_len, 1)
                    for logprob_start_len, extend_len in zip(
2140
2141
                        local_batch.extend_logprob_start_lens,
                        local_batch.extend_lens,
Lianmin Zheng's avatar
Lianmin Zheng committed
2142
                    )
2143
2144
2145
2146
                )
            else:
                # When return_logprob = False, only need last token per request
                num_tokens_for_logprob = local_batch.batch_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
2147
2148
2149
2150
2151
2152
2153
2154
2155

        if local_batch is None or local_batch.forward_mode.is_decode_or_idle():
            can_cuda_graph = 1
        else:
            can_cuda_graph = 0

        is_extend_in_batch = (
            local_batch.forward_mode.is_extend() if local_batch else False
        )
2156
2157

        tbo_preparer = TboDPAttentionPreparer()
2158
        if len(offload_tags) == 0 and disable_overlap_schedule:
2159
2160
2161
2162
2163
            group = tp_group.device_group
            device = tp_group.device
        else:
            group = tp_group.cpu_group
            device = "cpu"
2164

Lianmin Zheng's avatar
Lianmin Zheng committed
2165
2166
2167
2168
        local_info = torch.tensor(
            [
                num_tokens,
                can_cuda_graph,
2169
                num_tokens_for_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
2170
                is_extend_in_batch,
2171
2172
2173
                *tbo_preparer.prepare_all_gather(
                    local_batch,
                ),
Lianmin Zheng's avatar
Lianmin Zheng committed
2174
2175
            ],
            dtype=torch.int64,
2176
            device=device,
Lianmin Zheng's avatar
Lianmin Zheng committed
2177
2178
        )
        global_info = torch.empty(
2179
            (dp_size, attn_tp_size, 6),
Lianmin Zheng's avatar
Lianmin Zheng committed
2180
            dtype=torch.int64,
2181
            device=device,
Lianmin Zheng's avatar
Lianmin Zheng committed
2182
        )
2183
        torch.distributed.all_gather_into_tensor(
Lianmin Zheng's avatar
Lianmin Zheng committed
2184
2185
            global_info.flatten(),
            local_info,
2186
            group=group,
2187
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
2188
2189
2190
2191
        global_num_tokens = global_info[:, 0, 0].tolist()
        can_cuda_graph = min(global_info[:, 0, 1].tolist())
        global_num_tokens_for_logprob = global_info[:, 0, 2].tolist()
        is_extend_in_batch = global_info[:, 0, 3].tolist()
2192

2193
2194
2195
2196
        tbo_split_seq_index, global_forward_mode = tbo_preparer.compute_output(
            global_info[:, :, 4:6]
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
2197
        if local_batch is None and max(global_num_tokens) > 0:
2198
            local_batch = get_idle_batch()
2199
2200

        if local_batch is not None:
2201
            # TODO: handle the case when moe_dense_tp_size != 1
2202
            if not require_mlp_tp_gather:
2203
2204
2205
2206
2207
2208
2209
                local_batch.global_num_tokens = [num_tokens]
                local_batch.global_num_tokens_for_logprob = [num_tokens_for_logprob]
            else:
                local_batch.global_num_tokens = global_num_tokens
                local_batch.global_num_tokens_for_logprob = (
                    global_num_tokens_for_logprob
                )
2210
            local_batch.is_extend_in_batch = any(is_extend_in_batch)
2211
2212
            local_batch.tbo_split_seq_index = tbo_split_seq_index
            local_batch.global_forward_mode = global_forward_mode
2213

2214
            # Check forward mode for cuda graph
2215
            if not disable_cuda_graph:
Lianmin Zheng's avatar
Lianmin Zheng committed
2216
                local_batch.can_run_dp_cuda_graph = can_cuda_graph
2217

2218
        return local_batch
2219
2220
2221
2222
2223

    def get_idle_batch(self):
        idle_batch = ScheduleBatch.init_new(
            [],
            self.req_to_token_pool,
2224
            self.token_to_kv_pool_allocator,
2225
2226
2227
            self.tree_cache,
            self.model_config,
            self.enable_overlap,
2228
            self.spec_algorithm,
2229
2230
2231
2232
        )
        idle_batch.prepare_for_idle()
        return idle_batch

2233
2234
    def move_ready_grammar_requests(self):
        """Move requests whose grammar objects are ready from grammar_queue to waiting_queue."""
2235

2236
        num_ready_reqs = 0
2237
        num_timeout_reqs = 0
2238
2239
        for req in self.grammar_queue:
            try:
2240
2241
2242
                if req.finished():  # It is aborted by AbortReq
                    num_ready_reqs += 1
                    continue
2243

2244
                req.grammar = req.grammar.result(timeout=0.03)
2245
2246
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
2247
2248
2249
                    error_msg = f"Invalid grammar request: {req.grammar_key=}"
                    req.set_finish_with_abort(error_msg)

2250
2251
                num_ready_reqs += 1
            except futures._base.TimeoutError:
2252
                req.grammar_wait_ct += 1
2253
2254
                # NOTE(lianmin): this timeout is the waiting time of the above line. It is
                # not the waiting time from it enters the grammar queue.
2255
                if req.grammar_wait_ct > GRAMMAR_TIMEOUT / 0.03:
2256
                    num_timeout_reqs = 1
2257
2258
                break

2259
        if self.server_args.enable_dp_attention:
2260
2261
            tp_size = self.attn_tp_size
            tp_group = self.attn_tp_cpu_group
2262
        else:
2263
2264
2265
2266
2267
            tp_size = self.tp_size
            tp_group = self.tp_cpu_group

        if tp_size > 1:
            # Sync across TP ranks to make sure they have the same number of ready requests
2268
            tensor = torch.tensor([num_ready_reqs, num_timeout_reqs], dtype=torch.int32)
2269
2270
2271
            torch.distributed.all_reduce(
                tensor, op=torch.distributed.ReduceOp.MAX, group=tp_group
            )
2272
            num_ready_reqs_max, num_timeout_reqs_max = tensor.tolist()
2273

2274
            for i in range(num_ready_reqs, num_ready_reqs_max):
2275
                req = self.grammar_queue[i]
2276
2277
                if req.finished():  # It is aborted by AbortReq
                    continue
2278
                req.grammar = req.grammar.result()
2279
2280
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
2281
2282
                    error_msg = f"Invalid grammar request: {req.grammar_key=}"
                    req.set_finish_with_abort(error_msg)
2283
2284
2285
        else:
            num_ready_reqs_max = num_ready_reqs
            num_timeout_reqs_max = num_timeout_reqs
2286

2287
2288
2289
        for i in range(num_ready_reqs, num_ready_reqs + num_timeout_reqs_max):
            req = self.grammar_queue[i]
            req.grammar.cancel()
2290
            self.grammar_backend.set_cache(req.grammar_key, INVALID_GRAMMAR_OBJ)
2291
2292
            error_msg = f"Grammar preprocessing timed out for {req.grammar_key=}"
            req.set_finish_with_abort(error_msg)
2293

2294
        num_ready_reqs = num_ready_reqs_max + num_timeout_reqs_max
2295

2296
2297
        for req in self.grammar_queue[:num_ready_reqs]:
            self._add_request_to_queue(req)
2298
2299
        self.grammar_queue = self.grammar_queue[num_ready_reqs:]

2300
2301
2302
    def flush_cache_wrapped(self, recv_req: FlushCacheReqInput):
        success = self.flush_cache()
        return FlushCacheReqOutput(success=success)
2303

2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
    def clear_hicache_storage_wrapped(self, recv_req: ClearHiCacheReqInput):
        if self.enable_hierarchical_cache:
            self.tree_cache.clear_storage_backend()
            logger.info("Hierarchical cache cleared successfully!")
            if_success = True
        else:
            logging.warning("Hierarchical cache is not enabled.")
            if_success = False
        return ClearHiCacheReqOutput(success=if_success)

cctry's avatar
cctry committed
2314
2315
    def _is_no_request(self):
        no_request = (
2316
2317
            len(self.waiting_queue) == 0
            and self.running_batch.is_empty()
cctry's avatar
cctry committed
2318
2319
2320
            and (self.last_batch is None or self.last_batch.is_empty())
            and (self.cur_batch is None or self.cur_batch.is_empty())
            and (not self.enable_overlap or len(self.result_queue) == 0)
2321
            and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
cctry's avatar
cctry committed
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
        )
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            no_request &= (
                len(self.disagg_prefill_bootstrap_queue.queue) == 0
                and len(self.disagg_prefill_inflight_queue) == 0
            )
        if self.disaggregation_mode == DisaggregationMode.DECODE:
            no_request &= (
                len(self.disagg_decode_prealloc_queue.queue) == 0
                and len(self.disagg_decode_transfer_queue.queue) == 0
            )
        return no_request

    def flush_cache(self):
        """Flush the memory pool and cache."""
        if self._is_no_request():
2338
2339
            self.cur_batch = None
            self.last_batch = None
2340
            self.tree_cache.reset()
2341
            if self.grammar_backend:
Lianmin Zheng's avatar
Lianmin Zheng committed
2342
                self.grammar_backend.reset()
2343
            self.req_to_token_pool.clear()
2344
            self.token_to_kv_pool_allocator.clear()
2345

2346
2347
            if self.draft_worker:
                self.draft_worker.clear_cache_pool()
2348
2349
2350

            self.num_generated_tokens = 0
            self.forward_ct_decode = 0
2351
2352
2353
2354
            self.spec_num_accepted_tokens = 0
            self.spec_num_forward_ct = 0
            self.spec_total_num_accepted_tokens = 0
            self.spec_total_num_forward_ct = 0
2355
2356
2357
2358
2359
2360
2361
            torch.cuda.empty_cache()
            logger.info("Cache flushed successfully!")
            if_success = True
        else:
            logging.warning(
                f"Cache not flushed because there are pending requests. "
                f"#queue-req: {len(self.waiting_queue)}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2362
                f"#running-req: {len(self.running_batch.reqs)}"
2363
2364
2365
2366
            )
            if_success = False
        return if_success

2367
    def get_load(self, recv_req: GetLoadReqInput = None) -> GetLoadReqOutput:
Liangsheng Yin's avatar
Liangsheng Yin committed
2368
        # TODO(lsyin): use dynamically maintained num_waiting_tokens
2369

Hanming Lu's avatar
Hanming Lu committed
2370
        if self.is_hybrid:
2371
            num_tokens_full = (
Hanming Lu's avatar
Hanming Lu committed
2372
2373
2374
2375
                self.full_tokens_per_layer
                - self.token_to_kv_pool_allocator.full_available_size()
                - self.tree_cache.full_evictable_size()
            )
2376
            num_tokens_swa = (
Hanming Lu's avatar
Hanming Lu committed
2377
2378
2379
2380
                self.swa_tokens_per_layer
                - self.token_to_kv_pool_allocator.swa_available_size()
                - self.tree_cache.swa_evictable_size()
            )
2381
            num_tokens = max(num_tokens_full, num_tokens_swa)
2382
2383
2384
2385
2386
2387
        elif self.is_hybrid_gdn:
            num_tokens = (
                self.max_total_num_tokens
                - self.token_to_kv_pool_allocator.available_size()
                - self.tree_cache.full_evictable_size()
            )
Hanming Lu's avatar
Hanming Lu committed
2388
        else:
2389
            num_tokens = (
Hanming Lu's avatar
Hanming Lu committed
2390
2391
2392
2393
                self.max_total_num_tokens
                - self.token_to_kv_pool_allocator.available_size()
                - self.tree_cache.evictable_size()
            )
2394
2395
2396
2397

        # Tokens in waiting queue, bootstrap queue, prealloc queue
        num_tokens += sum(len(req.origin_input_ids) for req in self.waiting_queue)
        num_waiting_reqs = len(self.waiting_queue)
Liangsheng Yin's avatar
Liangsheng Yin committed
2398
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
2399
            num_tokens += sum(
Liangsheng Yin's avatar
Liangsheng Yin committed
2400
2401
2402
                len(req.origin_input_ids)
                for req in self.disagg_prefill_bootstrap_queue.queue
            )
2403
            num_waiting_reqs += len(self.disagg_prefill_bootstrap_queue.queue)
Liangsheng Yin's avatar
Liangsheng Yin committed
2404
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
2405
            num_tokens += sum(
Liangsheng Yin's avatar
Liangsheng Yin committed
2406
2407
2408
                len(req.req.origin_input_ids)
                for req in self.disagg_decode_prealloc_queue.queue
            )
2409
            num_waiting_reqs += len(self.disagg_decode_prealloc_queue.queue)
Liangsheng Yin's avatar
Liangsheng Yin committed
2410

2411
2412
2413
2414
2415
2416
        return GetLoadReqOutput(
            dp_rank=self.dp_rank,
            num_reqs=len(self.running_batch.reqs) + num_waiting_reqs,
            num_waiting_reqs=num_waiting_reqs,
            num_tokens=num_tokens,
        )
Liangsheng Yin's avatar
Liangsheng Yin committed
2417

2418
    def get_internal_state(self, recv_req: GetInternalStateReq):
2419
        ret = vars(get_global_server_args())
2420
        ret["last_gen_throughput"] = self.last_gen_throughput
2421
        ret["memory_usage"] = {
2422
            "weight": round(self.tp_worker.model_runner.weight_load_mem_usage, 2),
2423
2424
2425
2426
2427
            "kvcache": round(
                self.token_to_kv_pool_allocator.get_kvcache().mem_usage, 2
            ),
            "token_capacity": int(self.max_total_num_tokens),
        }
2428

2429
        ret["memory_usage"]["graph"] = round(
2430
            self.tp_worker.model_runner.graph_mem_usage, 2
2431
        )
2432

2433
        if not self.spec_algorithm.is_none() and self.spec_total_num_forward_ct > 0:
2434
            ret["avg_spec_accept_length"] = (
2435
                self.spec_total_num_accepted_tokens / self.spec_total_num_forward_ct
2436
2437
2438
            )
        if RECORD_STEP_TIME:
            ret["step_time_dict"] = self.step_time_dict
Liangsheng Yin's avatar
Liangsheng Yin committed
2439

2440
2441
2442
        # This field is not serializable.
        ret.pop("model_config", None)

Liangsheng Yin's avatar
Liangsheng Yin committed
2443
        return GetInternalStateReqOutput(internal_state=ret)
2444
2445
2446
2447
2448

    def set_internal_state(self, recv_req: SetInternalStateReq):
        server_args_dict = recv_req.server_args
        args_allow_update = set(
            [
2449
                "pp_max_micro_batch_size",
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
                "speculative_accept_threshold_single",
                "speculative_accept_threshold_acc",
            ]
        )
        if_success = True
        for k, v in server_args_dict.items():
            if k not in args_allow_update:
                logging.warning(f"Updating {k} is not supported.")
                if_success = False
                break
2460
            elif k == "pp_max_micro_batch_size" and (
2461
2462
2463
2464
2465
2466
2467
                v > self.max_running_requests // self.pp_size or v < 1
            ):
                logging.warning(
                    f"Updating {k} to {v} is rejected because it is out of the valid range [1, {self.max_running_requests // self.pp_size}]."
                )
                if_success = False
                break
2468
        if if_success:
2469
            if not self.spec_algorithm.is_none() and self.spec_total_num_forward_ct > 0:
2470
                avg_spec_accept_length = (
2471
                    self.spec_total_num_accepted_tokens / self.spec_total_num_forward_ct
2472
2473
                )
                logger.info(f"{avg_spec_accept_length=}")
2474
            self.spec_total_num_accepted_tokens = self.spec_total_num_forward_ct = 0
2475
            for k, v in server_args_dict.items():
2476
2477
                setattr(get_global_server_args(), k, v)
            logger.info(f"Global server args updated! {get_global_server_args()=}")
2478
2479
        return SetInternalStateReqOutput(
            updated=True,
2480
            server_args=vars(get_global_server_args()),
2481
2482
        )

2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
    def handle_rpc_request(self, recv_req: RpcReqInput):
        # Handle RPC requests
        logger.info(
            f"handle_rpc_request: {recv_req.method}, param: {recv_req.parameters}"
        )

        success = True
        exec = None
        try:
            func = getattr(self, recv_req.method)
            func(recv_req.parameters)
        except Exception as e:
            success = False
            exec = e
            logger.error(f"Failed to call rpc {recv_req.method}: {str(e)}")

        barrier()
        return RpcReqOutput(success, "" if not exec else str(exec))

2502
2503
    def abort_request(self, recv_req: AbortReq):
        # Delete requests in the waiting queue
Lianmin Zheng's avatar
Lianmin Zheng committed
2504
        to_del = []
2505
        for i, req in enumerate(self.waiting_queue):
2506
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
Lianmin Zheng's avatar
Lianmin Zheng committed
2507
                to_del.append(i)
2508

Lianmin Zheng's avatar
Lianmin Zheng committed
2509
        # Sort in reverse order to avoid index issues when deleting
Lianmin Zheng's avatar
Lianmin Zheng committed
2510
        for i in reversed(to_del):
2511
2512
2513
            # Abort method 1: directly pop from the queue
            # This only works for requests that have not started anything.
            # We still need to send something back to TokenizerManager to clean up the state.
Lianmin Zheng's avatar
Lianmin Zheng committed
2514
            req = self.waiting_queue.pop(i)
2515
2516
2517
            if self.enable_hicache_storage:
                # to release prefetch events associated with the request
                self.tree_cache.release_aborted_request(req.rid)
2518
            self.send_to_tokenizer.send_output(AbortReq(rid=req.rid), req)
2519
2520
2521
2522
            # For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
            if self.disaggregation_mode == DisaggregationMode.DECODE:
                self.tree_cache.cache_finished_req(req)

2523
2524
2525
            # For mamba radix cache
            if req.mamba_pool_idx is not None:
                self.tree_cache.cache_finished_req(req, is_insert=False)
2526
            logger.debug(f"Abort queued request. {req.rid=}")
2527

2528
2529
2530
2531
2532
        # Delete the requests in the grammar queue
        for req in self.grammar_queue:
            # Abort method 2: call `set_finish_with_abort`
            # The request will still run one prefill forward pass.
            # In this case, we change the input_ids to be only one token to make this prefill cheap.
2533
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2534
                logger.debug(f"Abort grammar queue request. {req.rid=}")
2535
2536
                if req.grammar:
                    req.grammar.cancel()
2537
2538
                req.set_finish_with_abort("Aborted by AbortReq.")

2539
2540
2541
        # Delete requests not in the waiting queue when PD disaggregation is enabled
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            # Abort requests that have not yet been bootstrapped
2542
            for req in self.disagg_prefill_bootstrap_queue.queue:
2543
                if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2544
                    logger.debug(f"Abort bootstrap queue request. {req.rid=}")
2545
2546
2547
2548
                    if hasattr(req.disagg_kv_sender, "abort"):
                        req.disagg_kv_sender.abort()

            # Abort in-flight requests
2549
            for req in self.disagg_prefill_inflight_queue:
2550
                if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2551
                    logger.debug(f"Abort inflight queue request. {req.rid=}")
2552
2553
2554
2555
2556
                    if hasattr(req.disagg_kv_sender, "abort"):
                        req.disagg_kv_sender.abort()

        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            # Abort requests that have not yet finished preallocation
2557
            for decode_req in self.disagg_decode_prealloc_queue.queue:
2558
                if recv_req.abort_all or decode_req.req.rid.startswith(recv_req.rid):
2559
                    logger.debug(f"Abort prealloc queue request. {decode_req.req.rid=}")
2560
2561
2562
2563
                    if hasattr(decode_req.kv_receiver, "abort"):
                        decode_req.kv_receiver.abort()

            # Abort requests waiting for kvcache to release tree cache
2564
            for decode_req in self.disagg_decode_transfer_queue.queue:
2565
                if recv_req.abort_all or decode_req.req.rid.startswith(recv_req.rid):
2566
                    logger.debug(f"Abort transfer queue request. {decode_req.req.rid=}")
2567
2568
2569
                    if hasattr(decode_req.kv_receiver, "abort"):
                        decode_req.kv_receiver.abort()

2570
        # Delete requests in the running batch
Lianmin Zheng's avatar
Lianmin Zheng committed
2571
2572
2573
2574
2575
2576
        if self.cur_batch is self.running_batch or self.cur_batch is None:
            reqs = self.running_batch.reqs
        else:
            reqs = self.running_batch.reqs + self.cur_batch.reqs

        for req in reqs:
2577
2578
2579
            if not req.finished() and (
                recv_req.abort_all or req.rid.startswith(recv_req.rid)
            ):
2580
                # Abort method 3: set `to_finish`
2581
2582
                # The request will still run one decode forward pass.
                # Then we reuse all existing code to clean up the KV cache allocation.
Lianmin Zheng's avatar
Lianmin Zheng committed
2583
                logger.debug(f"Abort running request. {req.rid=}")
2584
                req.to_finish = FINISH_ABORT()
2585

2586
2587
2588
    def _pause_engine(self) -> Tuple[List[Req], int]:
        raise NotImplementedError()

2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
    def load_lora_adapter(
        self, recv_req: LoadLoRAAdapterReqInput
    ) -> LoadLoRAAdapterReqOutput:
        """In-place loading a new lora adapter from disk or huggingface."""

        result = self.tp_worker.load_lora_adapter(recv_req)
        return result

    def unload_lora_adapter(
        self, recv_req: UnloadLoRAAdapterReqInput
    ) -> UnloadLoRAAdapterReqOutput:
        """Unload the lora adapter."""

        result = self.tp_worker.unload_lora_adapter(recv_req)
        return result

2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
    def init_weights_send_group_for_remote_instance(
        self, recv_req: InitWeightsSendGroupForRemoteInstanceReqInput
    ):
        """Init the seed and client instance communication group."""
        success, message = self.tp_worker.init_weights_send_group_for_remote_instance(
            recv_req
        )
        return InitWeightsSendGroupForRemoteInstanceReqOutput(success, message)

    def send_weights_to_remote_instance(
        self, recv_req: SendWeightsToRemoteInstanceReqInput
    ):
        """Send the seed instance weights to the destination instance."""
        success, message = self.tp_worker.send_weights_to_remote_instance(recv_req)
        return SendWeightsToRemoteInstanceReqOutput(success, message)

2621
2622
2623
2624
2625
2626
2627
    def slow_down(self, recv_req: SlowDownReqInput):
        t = recv_req.forward_sleep_time
        if t is not None and t <= 0:
            t = None
        self.forward_sleep_time = t
        return SlowDownReqOutput()

2628
    def expert_distribution_handle(self, recv_req: ExpertDistributionReq):
2629
2630
        action = recv_req.action
        if action == ExpertDistributionReqType.START_RECORD:
2631
            get_global_expert_distribution_recorder().start_record()
2632
        elif action == ExpertDistributionReqType.STOP_RECORD:
2633
            get_global_expert_distribution_recorder().stop_record()
2634
        elif action == ExpertDistributionReqType.DUMP_RECORD:
2635
            get_global_expert_distribution_recorder().dump_record()
2636
        else:
2637
            raise ValueError(f"Unrecognized ExpertDistributionReq value: {recv_req=}")
2638
        return ExpertDistributionReqOutput()
2639

2640
    def open_session(self, recv_req: OpenSessionReqInput):
2641
2642
2643
2644
        # handle error
        session_id = recv_req.session_id
        if session_id in self.sessions:
            logger.warning(f"session id {session_id} already exist, cannot open.")
2645
            return OpenSessionReqOutput(session_id, False)
2646
        elif session_id is None:
2647
            logger.warning("session id is None, cannot open.")
2648
            return OpenSessionReqOutput(session_id, False)
2649
2650
2651
2652
        else:
            self.sessions[session_id] = Session(
                recv_req.capacity_of_str_len, session_id
            )
2653
            return OpenSessionReqOutput(session_id, True)
2654
2655
2656
2657
2658
2659
2660
2661
2662

    def close_session(self, recv_req: CloseSessionReqInput):
        # handle error
        session_id = recv_req.session_id
        if session_id not in self.sessions:
            logger.warning(f"session id {session_id} does not exist, cannot delete.")
        else:
            del self.sessions[session_id]

2663
2664
    def get_print_prefix(self):
        prefix = ""
2665
2666
        if self.attn_dp_rank is not None:
            prefix += f" DP{self.attn_dp_rank}"
2667
2668
2669
2670
2671
2672
        if self.server_args.tp_size > 1:
            prefix += f" TP{self.tp_rank}"
        if self.pp_size > 1:
            prefix += f" PP{self.pp_rank}"
        return prefix

2673
2674
    def current_scheduler_metrics_enabled(self):
        return self.attn_tp_rank == 0 or self.enable_metrics_for_all_schedulers
2675

2676
2677
2678
    def maybe_sleep_on_idle(self):
        if self.idle_sleeper is not None:
            self.idle_sleeper.maybe_sleep()
2679

2680
2681
2682
    def handle_freeze_gc(self, recv_req: FreezeGCReq):
        """Handle freeze_gc request: freeze scheduler's GC and forward to detokenizer."""
        freeze_gc("Scheduler")
2683
        self.send_to_detokenizer.send_output(recv_req, recv_req)
2684
2685
        return None

2686

2687
2688
2689
2690
2691
2692
2693
class IdleSleeper:
    """
    In setups which have long inactivity periods it is desirable to reduce
    system power consumption when sglang does nothing. This would lead not only
    to power savings, but also to more CPU thermal headroom when a request
    eventually comes. This is important in cases when multiple GPUs are connected
    as each GPU would otherwise pin one thread at 100% CPU usage.
2694

2695
2696
2697
    The simplest solution is to use zmq.Poller on all sockets that may receive
    data that needs handling immediately.
    """
2698

2699
2700
    def __init__(self, sockets):
        self.poller = zmq.Poller()
2701
        self.last_empty_time = time.time()
2702
2703
2704
        for s in sockets:
            self.poller.register(s, zmq.POLLIN)

2705
2706
        self.empty_cache_interval = envs.SGLANG_EMPTY_CACHE_INTERVAL.get()

2707
2708
    def maybe_sleep(self):
        self.poller.poll(1000)
2709
        if (
2710
2711
            self.empty_cache_interval > 0
            and time.time() - self.last_empty_time > self.empty_cache_interval
2712
2713
2714
        ):
            self.last_empty_time = time.time()
            torch.cuda.empty_cache()
2715

2716

2717
def is_health_check_generate_req(recv_req):
2718
2719
    rid = getattr(recv_req, "rid", None)
    return rid is not None and rid.startswith("HEALTH_CHECK")
2720

2721
2722

def is_work_request(recv_req):
2723
2724
2725
2726
2727
2728
2729
2730
2731
    return isinstance(
        recv_req,
        (
            TokenizedGenerateReqInput,
            TokenizedEmbeddingReqInput,
            BatchTokenizedGenerateReqInput,
            BatchTokenizedEmbeddingReqInput,
        ),
    )
2732
2733


2734
2735
2736
2737
2738
def run_scheduler_process(
    server_args: ServerArgs,
    port_args: PortArgs,
    gpu_id: int,
    tp_rank: int,
Cheng Wan's avatar
Cheng Wan committed
2739
    moe_ep_rank: int,
2740
    pp_rank: int,
2741
    dp_rank: Optional[int],
2742
    pipe_writer,
2743
):
2744
    # Generate the logger prefix
2745
    prefix = ""
2746
2747
2748
    if dp_rank is None and "SGLANG_DP_RANK" in os.environ:
        # [For Router] if env var "SGLANG_DP_RANK" exist, set dp_rank to the value of the env var
        dp_rank = int(os.environ["SGLANG_DP_RANK"])
2749
2750
2751
2752
    if dp_rank is not None:
        prefix += f" DP{dp_rank}"
    if server_args.tp_size > 1:
        prefix += f" TP{tp_rank}"
Cheng Wan's avatar
Cheng Wan committed
2753
2754
    if server_args.ep_size > 1:
        prefix += f" EP{moe_ep_rank}"
2755
2756
    if server_args.pp_size > 1:
        prefix += f" PP{pp_rank}"
2757

2758
    # Config the process
2759
    setproctitle.setproctitle(f"sglang::scheduler{prefix.replace(' ', '_')}")
2760
    faulthandler.enable()
2761
    kill_itself_when_parent_died()
2762
    parent_process = psutil.Process().parent()
2763

Wang Ran (汪然)'s avatar
Wang Ran (汪然) committed
2764
    # Configure the logger
2765
    configure_logger(server_args, prefix=prefix)
2766
    suppress_other_loggers()
2767

2768
    # Set cpu affinity to this gpu process
2769
    if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
2770
2771
2772
        set_gpu_proc_affinity(
            server_args.pp_size, server_args.tp_size, server_args.nnodes, gpu_id
        )
2773
2774
2775
2776
2777
    if (numa_node := server_args.numa_node) is not None:
        numa_bind_to_node(numa_node[gpu_id])

    # Set up tracing
    if server_args.enable_trace:
2778
2779
2780
2781
2782
2783
2784
        process_tracing_init(server_args.otlp_traces_endpoint, "sglang")
        thread_label = "Scheduler"
        if server_args.disaggregation_mode == "prefill":
            thread_label = "Prefill Scheduler"
        elif server_args.disaggregation_mode == "decode":
            thread_label = "Decode Scheduler"
        trace_set_thread_info(thread_label, tp_rank, dp_rank)
2785

2786
    # Create a scheduler and run the event loop
2787
    try:
Cheng Wan's avatar
Cheng Wan committed
2788
        scheduler = Scheduler(
2789
2790
2791
2792
2793
2794
2795
            server_args,
            port_args,
            gpu_id,
            tp_rank,
            moe_ep_rank,
            pp_rank,
            dp_rank,
Cheng Wan's avatar
Cheng Wan committed
2796
        )
2797
        pipe_writer.send(
Mick's avatar
Mick committed
2798
2799
2800
2801
2802
            {
                "status": "ready",
                "max_total_num_tokens": scheduler.max_total_num_tokens,
                "max_req_input_len": scheduler.max_req_input_len,
            }
2803
        )
Byron Hsu's avatar
Byron Hsu committed
2804

2805
        disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
Byron Hsu's avatar
Byron Hsu committed
2806
        if disaggregation_mode == DisaggregationMode.NULL:
2807
2808
2809
            if scheduler.enable_pdmux:
                scheduler.event_loop_pdmux()
            elif server_args.pp_size > 1:
2810
2811
                scheduler.event_loop_pp()
            elif scheduler.enable_overlap:
Byron Hsu's avatar
Byron Hsu committed
2812
2813
2814
2815
                scheduler.event_loop_overlap()
            else:
                scheduler.event_loop_normal()
        elif disaggregation_mode == DisaggregationMode.PREFILL:
2816
2817
2818
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_prefill()
            else:
2819
2820
2821
2822
                if server_args.pp_size > 1:
                    scheduler.event_loop_pp_disagg_prefill()
                else:
                    scheduler.event_loop_normal_disagg_prefill()
2823

Byron Hsu's avatar
Byron Hsu committed
2824
        elif disaggregation_mode == DisaggregationMode.DECODE:
2825
2826
2827
2828
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_decode()
            else:
                scheduler.event_loop_normal_disagg_decode()
Byron Hsu's avatar
Byron Hsu committed
2829

2830
    except Exception:
2831
2832
2833
        traceback = get_exception_traceback()
        logger.error(f"Scheduler hit an exception: {traceback}")
        parent_process.send_signal(signal.SIGQUIT)