scheduler.py 117 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
14
15
"""A scheduler that manages a tensor parallel GPU worker."""

16
import faulthandler
17
import logging
18
import os
19
import signal
20
import sys
Lianmin Zheng's avatar
Lianmin Zheng committed
21
import threading
22
import time
23
from collections import deque
Lianmin Zheng's avatar
Lianmin Zheng committed
24
from concurrent import futures
25
from dataclasses import dataclass
26
from http import HTTPStatus
27
from types import SimpleNamespace
28
from typing import Dict, List, Optional, Tuple, Union
29

30
import psutil
31
import setproctitle
32
import torch
33
import zmq
34
from torch.distributed import barrier
35

36
from sglang.global_config import global_config
Lianmin Zheng's avatar
Lianmin Zheng committed
37
from sglang.srt.configs.model_config import ModelConfig
38
39
40
41
from sglang.srt.constrained.base_grammar_backend import (
    INVALID_GRAMMAR_OBJ,
    create_grammar_backend,
)
Byron Hsu's avatar
Byron Hsu committed
42
43
44
45
46
from sglang.srt.disaggregation.decode import (
    DecodePreallocQueue,
    DecodeTransferQueue,
    SchedulerDisaggregationDecodeMixin,
)
47
48
49
from sglang.srt.disaggregation.decode_kvcache_offload_manager import (
    DecodeKVCacheOffloadManager,
)
Byron Hsu's avatar
Byron Hsu committed
50
51
52
53
54
55
from sglang.srt.disaggregation.prefill import (
    PrefillBootstrapQueue,
    SchedulerDisaggregationPrefillMixin,
)
from sglang.srt.disaggregation.utils import (
    DisaggregationMode,
56
    MetadataBuffers,
Byron Hsu's avatar
Byron Hsu committed
57
    ReqToMetadataIdxAllocator,
58
    TransferBackend,
59
    prepare_abort,
Byron Hsu's avatar
Byron Hsu committed
60
)
61
from sglang.srt.distributed import get_pp_group, get_world_group
fzyzcjy's avatar
fzyzcjy committed
62
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
63
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
64
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
65
from sglang.srt.layers.moe import initialize_moe_config
66
67
from sglang.srt.managers.io_struct import (
    AbortReq,
68
69
    BatchTokenizedEmbeddingReqInput,
    BatchTokenizedGenerateReqInput,
70
71
    ClearHiCacheReqInput,
    ClearHiCacheReqOutput,
72
    CloseSessionReqInput,
73
    DestroyWeightsUpdateGroupReqInput,
74
    ExpertDistributionReq,
75
    ExpertDistributionReqOutput,
76
    ExpertDistributionReqType,
77
78
    FlushCacheReqInput,
    FlushCacheReqOutput,
79
    FreezeGCReq,
80
81
    GetInternalStateReq,
    GetInternalStateReqOutput,
82
83
    GetLoadReqInput,
    GetLoadReqOutput,
84
    GetWeightsByNameReqInput,
85
    HealthCheckOutput,
86
87
    InitWeightsSendGroupForRemoteInstanceReqInput,
    InitWeightsSendGroupForRemoteInstanceReqOutput,
88
    InitWeightsUpdateGroupReqInput,
89
90
    LoadLoRAAdapterReqInput,
    LoadLoRAAdapterReqOutput,
91
    MultiTokenizerRegisterReq,
92
    MultiTokenizerWrapper,
93
94
    OpenSessionReqInput,
    OpenSessionReqOutput,
95
    ProfileReq,
96
97
    ReleaseMemoryOccupationReqInput,
    ResumeMemoryOccupationReqInput,
98
99
    RpcReqInput,
    RpcReqOutput,
100
101
    SendWeightsToRemoteInstanceReqInput,
    SendWeightsToRemoteInstanceReqOutput,
102
103
    SetInternalStateReq,
    SetInternalStateReqOutput,
104
105
    SlowDownReqInput,
    SlowDownReqOutput,
106
107
    TokenizedEmbeddingReqInput,
    TokenizedGenerateReqInput,
108
109
    UnloadLoRAAdapterReqInput,
    UnloadLoRAAdapterReqOutput,
Chayenne's avatar
Chayenne committed
110
    UpdateWeightFromDiskReqInput,
111
    UpdateWeightsFromDistributedReqInput,
112
    UpdateWeightsFromTensorReqInput,
113
)
114
from sglang.srt.managers.mm_utils import init_embedding_cache
115
116
from sglang.srt.managers.schedule_batch import (
    FINISH_ABORT,
Mick's avatar
Mick committed
117
    MultimodalInputs,
118
    Req,
119
    RequestStage,
120
    ScheduleBatch,
121
    global_server_args_dict,
122
)
123
124
125
126
127
from sglang.srt.managers.schedule_policy import (
    AddReqResult,
    PrefillAdder,
    SchedulePolicy,
)
fzyzcjy's avatar
fzyzcjy committed
128
from sglang.srt.managers.scheduler_input_blocker import SchedulerInputBlocker
129
130
131
132
from sglang.srt.managers.scheduler_metrics_mixin import (
    RECORD_STEP_TIME,
    SchedulerMetricsMixin,
)
133
134
135
from sglang.srt.managers.scheduler_output_processor_mixin import (
    SchedulerOutputProcessorMixin,
)
136
from sglang.srt.managers.scheduler_profiler_mixin import SchedulerProfilerMixin
137
from sglang.srt.managers.scheduler_recv_skipper import SchedulerRecvSkipper
138
139
140
from sglang.srt.managers.scheduler_update_weights_mixin import (
    SchedulerUpdateWeightsMixin,
)
141
from sglang.srt.managers.session_controller import Session
142
from sglang.srt.managers.tp_worker import TpModelWorker
143
from sglang.srt.managers.tp_worker_overlap_thread import TpModelWorkerClient
144
from sglang.srt.managers.utils import validate_input_length
tarinkk's avatar
tarinkk committed
145
from sglang.srt.mem_cache.chunk_cache import ChunkCache, SWAChunkCache
146
from sglang.srt.mem_cache.hiradix_cache import HiRadixCache
147
from sglang.srt.mem_cache.radix_cache import RadixCache
Hanming Lu's avatar
Hanming Lu committed
148
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
149
150
151
152
153
from sglang.srt.model_executor.forward_batch_info import (
    ForwardBatchOutput,
    ForwardMode,
    PPProxyTensors,
)
154
from sglang.srt.parser.reasoning_parser import ReasoningParser
155
from sglang.srt.server_args import PortArgs, ServerArgs
156
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
157
from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter
158
159
160
161
from sglang.srt.tracing.trace import (
    process_tracing_init,
    trace_set_proc_propagate_context,
    trace_set_thread_info,
162
    trace_slice_batch,
163
164
165
    trace_slice_end,
    trace_slice_start,
)
166
from sglang.srt.two_batch_overlap import TboDPAttentionPreparer
167
from sglang.srt.utils import (
168
    DynamicGradMode,
169
    broadcast_pyobj,
fzyzcjy's avatar
fzyzcjy committed
170
    configure_gc_logger,
171
    configure_logger,
Lianmin Zheng's avatar
Lianmin Zheng committed
172
    disable_request_logging,
173
    freeze_gc,
174
    get_available_gpu_memory,
175
    get_bool_env_var,
176
    get_int_env_var,
177
    get_zmq_socket,
Lianmin Zheng's avatar
Lianmin Zheng committed
178
    kill_itself_when_parent_died,
179
    numa_bind_to_node,
180
    point_to_point_pyobj,
181
    pyspy_dump_schedulers,
182
183
    require_mlp_sync,
    require_mlp_tp_gather,
184
    set_gpu_proc_affinity,
185
186
187
    set_random_seed,
    suppress_other_loggers,
)
188
189
190
191
192
from sglang.srt.utils.hf_transformers_utils import (
    get_processor,
    get_tokenizer,
    get_tokenizer_from_processor,
)
193
from sglang.utils import TypeBasedDispatcher, get_exception_traceback
194
195
196

logger = logging.getLogger(__name__)

197
# Test retract decode for debugging purposes
198
TEST_RETRACT = get_bool_env_var("SGLANG_TEST_RETRACT")
199
GRAMMAR_TIMEOUT = float(os.environ.get("SGLANG_GRAMMAR_TIMEOUT", 300))
200

201

202
203
@dataclass
class GenerationBatchResult:
204
    logits_output: Optional[LogitsProcessorOutput]
205
    pp_hidden_states_proxy_tensors: Optional[PPProxyTensors]
206
    next_token_ids: Optional[List[int]]
207
208
209
    can_run_cuda_graph: bool

    # For output processing
210
211
    extend_input_len_per_req: List[int]
    extend_logprob_start_len_per_req: List[int]
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248

    @classmethod
    def from_forward_batch_output(
        cls,
        forward_batch_output: ForwardBatchOutput,
        extend_input_len_per_req: List[int],
        extend_logprob_start_len_per_req: List[int],
    ):
        # TODO(lsyin): remove this workaround logic and try to unify output classes

        return cls(
            logits_output=forward_batch_output.logits_output,
            pp_hidden_states_proxy_tensors=forward_batch_output.pp_proxy_tensors,
            next_token_ids=forward_batch_output.next_token_ids,
            extend_input_len_per_req=extend_input_len_per_req,
            extend_logprob_start_len_per_req=extend_logprob_start_len_per_req,
            can_run_cuda_graph=forward_batch_output.can_run_cuda_graph,
        )

    @classmethod
    def from_pp_proxy(
        cls, logits_output, next_pp_outputs: PPProxyTensors, can_run_cuda_graph
    ):
        # TODO(lsyin): also simplify this logic
        # Current PP implementation in scheduler is not compatible with ForwardBatchOutput
        # Maybe introduce a ProxyBatchOutput for PP and the original ForwardBatchOutput for TP
        proxy_dict = next_pp_outputs.tensors
        return cls(
            logits_output=logits_output,
            pp_hidden_states_proxy_tensors=None,
            next_token_ids=next_pp_outputs["next_token_ids"],
            extend_input_len_per_req=proxy_dict.get("extend_input_len_per_req", None),
            extend_logprob_start_len_per_req=proxy_dict.get(
                "extend_logprob_start_len_per_req", None
            ),
            can_run_cuda_graph=can_run_cuda_graph,
        )
249
250
251
252
253
254
255


@dataclass
class EmbeddingBatchResult:
    embeddings: torch.Tensor


Byron Hsu's avatar
Byron Hsu committed
256
257
class Scheduler(
    SchedulerOutputProcessorMixin,
258
259
260
    SchedulerUpdateWeightsMixin,
    SchedulerProfilerMixin,
    SchedulerMetricsMixin,
Byron Hsu's avatar
Byron Hsu committed
261
262
263
    SchedulerDisaggregationDecodeMixin,
    SchedulerDisaggregationPrefillMixin,
):
264
265
266
267
268
269
270
271
    """A scheduler that manages a tensor parallel GPU worker."""

    def __init__(
        self,
        server_args: ServerArgs,
        port_args: PortArgs,
        gpu_id: int,
        tp_rank: int,
Cheng Wan's avatar
Cheng Wan committed
272
        moe_ep_rank: int,
273
        pp_rank: int,
274
        dp_rank: Optional[int],
275
276
    ):
        # Parse args
277
        self.server_args = server_args
278
        self.tp_rank = tp_rank
Cheng Wan's avatar
Cheng Wan committed
279
        self.moe_ep_rank = moe_ep_rank
280
        self.pp_rank = pp_rank
281
        self.dp_rank = dp_rank
282
        self.tp_size = server_args.tp_size
Cheng Wan's avatar
Cheng Wan committed
283
        self.moe_ep_size = server_args.ep_size
284
285
        self.pp_size = server_args.pp_size
        self.dp_size = server_args.dp_size
286
        self.schedule_policy = server_args.schedule_policy
287
288
289
290
291
292
293
        self.enable_priority_scheduling = server_args.enable_priority_scheduling
        self.schedule_low_priority_values_first = (
            server_args.schedule_low_priority_values_first
        )
        self.priority_scheduling_preemption_threshold = (
            server_args.priority_scheduling_preemption_threshold
        )
294
        self.enable_lora = server_args.enable_lora
295
        self.max_loras_per_batch = server_args.max_loras_per_batch
296
        self.enable_overlap = not server_args.disable_overlap_schedule
297
        self.skip_tokenizer_init = server_args.skip_tokenizer_init
298
        self.enable_metrics = server_args.enable_metrics
299
300
301
        self.enable_metrics_for_all_schedulers = (
            server_args.enable_metrics_for_all_schedulers
        )
302
        self.enable_kv_cache_events = server_args.kv_events_config and tp_rank == 0
303
        self.enable_trace = server_args.enable_trace
304
        self.stream_interval = server_args.stream_interval
305
306
307
        self.spec_algorithm = SpeculativeAlgorithm.from_string(
            server_args.speculative_algorithm
        )
308
309
        self.gpu_id = gpu_id
        self.enable_hierarchical_cache = server_args.enable_hierarchical_cache
310
        self.enable_hicache_storage = server_args.hicache_storage_backend is not None
Lianmin Zheng's avatar
Lianmin Zheng committed
311
        self.page_size = server_args.page_size
312

313
        self.attn_tp_rank, self.attn_tp_size, self.attn_dp_rank = (
314
315
316
317
318
319
320
321
            compute_dp_attention_world_info(
                server_args.enable_dp_attention,
                self.tp_rank,
                self.tp_size,
                self.dp_size,
            )
        )

322
323
324
        # Init model config
        self.model_config = ModelConfig.from_server_args(server_args)

325
326
        # Init inter-process communication
        context = zmq.Context(2)
327
        self.idle_sleeper = None
328
        if self.pp_rank == 0 and self.attn_tp_rank == 0:
329
            self.recv_from_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
330
                context, zmq.PULL, port_args.scheduler_input_ipc_name, False
331
            )
332
333
334
335
            self.recv_from_rpc = get_zmq_socket(
                context, zmq.DEALER, port_args.rpc_ipc_name, False
            )

336
            self.send_to_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
337
                context, zmq.PUSH, port_args.tokenizer_ipc_name, False
338
            )
339
            if server_args.skip_tokenizer_init:
340
                # Directly send to the TokenizerManager
341
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
342
                    context, zmq.PUSH, port_args.tokenizer_ipc_name, False
343
344
                )
            else:
345
                # Send to the DetokenizerManager
346
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
347
                    context, zmq.PUSH, port_args.detokenizer_ipc_name, False
348
                )
349

350
351
352
353
354
355
356
            if self.server_args.sleep_on_idle:
                self.idle_sleeper = IdleSleeper(
                    [
                        self.recv_from_tokenizer,
                        self.recv_from_rpc,
                    ]
                )
357
        else:
358
            self.recv_from_tokenizer = None
359
            self.recv_from_rpc = None
360
361
            self.send_to_tokenizer = SimpleNamespace(send_pyobj=lambda x: None)
            self.send_to_detokenizer = SimpleNamespace(send_pyobj=lambda x: None)
362

363
364
365
366
367
        if self.current_scheduler_metrics_enabled():
            self.send_metrics_from_scheduler = get_zmq_socket(
                context, zmq.PUSH, port_args.metrics_ipc_name, False
            )

368
        # Init tokenizer
369
        self.init_tokenizer()
370

371
372
373
        # Init moe config
        self.init_moe_config()

374
375
376
377
378
379
380
381
382
        # Set reasoning_parser and think_end_id if --reasoning_parser is enabled
        if self.server_args.reasoning_parser and self.tokenizer:
            reasoning_parser = ReasoningParser(
                model_type=self.server_args.reasoning_parser, stream_reasoning=False
            )
            self.tokenizer.think_end_id = self.tokenizer.encode(
                reasoning_parser.detector.think_end_token, add_special_tokens=False
            )[0]

383
384
385
386
        # Check whether overlap can be enabled
        if not self.is_generation:
            self.enable_overlap = False
            logger.info("Overlap scheduler is disabled for embedding models.")
387

388
        # Launch a tensor parallel worker
389
        if self.enable_overlap:
390
            TpWorkerClass = TpModelWorkerClient
391
392
        else:
            TpWorkerClass = TpModelWorker
393

394
        self.tp_worker = TpWorkerClass(
395
            server_args=server_args,
396
397
            gpu_id=gpu_id,
            tp_rank=tp_rank,
Cheng Wan's avatar
Cheng Wan committed
398
            moe_ep_rank=moe_ep_rank,
399
            pp_rank=pp_rank,
400
            dp_rank=dp_rank,
401
            nccl_port=port_args.nccl_port,
402
        )
403

404
        # Launch a draft worker for speculative decoding
405
406
407
408
409
410
        if self.spec_algorithm.is_eagle():
            from sglang.srt.speculative.eagle_worker import EAGLEWorker

            self.draft_worker = EAGLEWorker(
                gpu_id=gpu_id,
                tp_rank=tp_rank,
Cheng Wan's avatar
Cheng Wan committed
411
                moe_ep_rank=moe_ep_rank,
412
413
414
415
416
                server_args=server_args,
                nccl_port=port_args.nccl_port,
                target_worker=self.tp_worker,
                dp_rank=dp_rank,
            )
417
418
419
420
421
422
423
424
425
426
427
428
        elif self.spec_algorithm.is_standalone():
            from sglang.srt.speculative.standalone_worker import StandaloneWorker

            self.draft_worker = StandaloneWorker(
                gpu_id=gpu_id,
                tp_rank=tp_rank,
                moe_ep_rank=moe_ep_rank,
                server_args=server_args,
                nccl_port=port_args.nccl_port,
                target_worker=self.tp_worker,
                dp_rank=dp_rank,
            )
429
430
        elif self.spec_algorithm.is_ngram():
            from sglang.srt.speculative.ngram_worker import NGRAMWorker
431

432
            self.draft_worker = NGRAMWorker(
433
434
435
436
437
438
439
440
                gpu_id=gpu_id,
                tp_rank=tp_rank,
                moe_ep_rank=moe_ep_rank,
                server_args=server_args,
                nccl_port=port_args.nccl_port,
                target_worker=self.tp_worker,
                dp_rank=dp_rank,
            )
441
442
443
        else:
            self.draft_worker = None

444
445
446
447
448
449
        # Dispatch the model worker
        if self.spec_algorithm.is_none():
            self.model_worker = self.tp_worker
        else:
            self.model_worker = self.draft_worker

450
        # Get token and memory info from the model worker
451
452
453
454
        (
            self.max_total_num_tokens,
            self.max_prefill_tokens,
            self.max_running_requests,
455
            self.max_queued_requests,
456
            self.max_req_len,
457
458
            self.max_req_input_len,
            self.random_seed,
459
            self.device,
460
461
462
463
464
            worker_global_server_args_dict,
            _,
            _,
            _,
        ) = self.tp_worker.get_worker_info()
465
466
467
468
469
470
471
472
        if global_server_args_dict["max_micro_batch_size"] is None:
            global_server_args_dict["max_micro_batch_size"] = max(
                self.max_running_requests // server_args.pp_size, 1
            )

        self.tp_group = self.tp_worker.get_tp_group()
        self.tp_cpu_group = self.tp_group.cpu_group
        self.attn_tp_group = self.tp_worker.get_attention_tp_group()
473
        self.attn_tp_cpu_group = self.tp_worker.get_attention_tp_cpu_group()
474
475
476
        self.pp_group = get_pp_group()
        self.world_group = get_world_group()

477
        self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
478
        global_server_args_dict.update(worker_global_server_args_dict)
479
        set_random_seed(self.random_seed)
480

481
        # Hybrid memory pool
Hanming Lu's avatar
Hanming Lu committed
482
483
484
485
486
487
488
        self.is_hybrid = self.tp_worker.is_hybrid
        if self.is_hybrid:
            self.sliding_window_size = self.tp_worker.sliding_window_size
            self.full_tokens_per_layer, self.swa_tokens_per_layer = (
                self.tp_worker.get_tokens_per_layer_info()
            )

489
        # Print debug info
490
        if tp_rank == 0:
491
492
493
            avail_mem = get_available_gpu_memory(
                self.device, self.gpu_id, empty_cache=False
            )
494
495
496
497
498
            logger.info(
                f"max_total_num_tokens={self.max_total_num_tokens}, "
                f"chunked_prefill_size={server_args.chunked_prefill_size}, "
                f"max_prefill_tokens={self.max_prefill_tokens}, "
                f"max_running_requests={self.max_running_requests}, "
499
                f"context_len={self.model_config.context_len}, "
500
                f"{'available_cpu_mem' if self.device == 'cpu' else 'available_gpu_mem'}={avail_mem:.2f} GB"
501
            )
502

Lianmin Zheng's avatar
Lianmin Zheng committed
503
        # Init memory pool and cache
504
        self.init_memory_pool_and_cache()
505
506
507

        # Init running status
        self.waiting_queue: List[Req] = []
508
        # The running decoding batch for continuous batching
Lianmin Zheng's avatar
Lianmin Zheng committed
509
        self.running_batch: ScheduleBatch = ScheduleBatch(reqs=[], batch_is_full=False)
510
        # The current forward batch
Lianmin Zheng's avatar
Lianmin Zheng committed
511
        self.cur_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
512
        # The last forward batch
513
        self.last_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
514
515
        self.forward_ct = 0
        self.forward_ct_decode = 0
516
        self.num_generated_tokens = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
517
        self.last_prefill_tokens = 0
518
519
        self.last_decode_stats_tic = time.perf_counter()
        self.last_prefill_stats_tic = time.perf_counter()
520
        self.return_health_check_ct = 0
521
522
523
524
525
        self.num_retracted_reqs: int = 0
        self.num_paused_reqs: int = 0
        self.kv_transfer_speed_gb_s: float = 0.0
        self.kv_transfer_latency_ms: float = 0.0
        self.sessions: Dict[str, Session] = {}
526
        self.current_stream = torch.get_device_module(self.device).current_stream()
527
528
        if self.device == "cpu":
            self.current_stream.synchronize = lambda: None  # No-op for CPU
529
        self.forward_sleep_time = None
530

531
532
        # Init chunked prefill
        self.chunked_prefill_size = server_args.chunked_prefill_size
533
534
        if self.chunked_prefill_size <= 0:  # -1 means disable
            self.chunked_prefill_size = None
535
        self.chunked_req = None
536
537
538
539
        self.is_mixed_chunk = (
            self.chunked_prefill_size is not None and server_args.enable_mixed_chunk
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
540
        # Init the grammar backend for constrained generation
541
        self.grammar_queue: List[Req] = []
542
        if not server_args.skip_tokenizer_init:
543
            self.grammar_backend = create_grammar_backend(
544
545
546
547
                server_args,
                self.tokenizer,
                self.model_config.vocab_size,
                self.model_config.hf_eos_token_id,
548
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
549
550
        else:
            self.grammar_backend = None
551

552
        # Init schedule policy and new token estimation
553
        self.policy = SchedulePolicy(
Lianmin Zheng's avatar
Lianmin Zheng committed
554
555
556
            self.schedule_policy,
            self.tree_cache,
            self.enable_hierarchical_cache,
557
558
            self.enable_priority_scheduling,
            self.schedule_low_priority_values_first,
559
        )
560
561
562
        # Enable preemption for priority scheduling.
        self.try_preemption = self.enable_priority_scheduling

563
564
565
        assert (
            server_args.schedule_conservativeness >= 0
        ), "Invalid schedule_conservativeness"
566
567
        self.init_new_token_ratio = min(
            global_config.default_init_new_token_ratio
568
569
            * server_args.schedule_conservativeness,
            1.0,
570
        )
571
572
573
574
575
576
577
578
579
580
        self.min_new_token_ratio = min(
            self.init_new_token_ratio
            * global_config.default_min_new_token_ratio_factor,
            1.0,
        )
        self.new_token_ratio_decay = (
            self.init_new_token_ratio - self.min_new_token_ratio
        ) / global_config.default_new_token_ratio_decay_steps
        self.new_token_ratio = self.init_new_token_ratio

Lianmin Zheng's avatar
Lianmin Zheng committed
581
582
583
584
        # Init watchdog thread
        self.watchdog_timeout = server_args.watchdog_timeout
        t = threading.Thread(target=self.watchdog_thread, daemon=True)
        t.start()
585
        self.parent_process = psutil.Process().parent()
586
587

        # Init memory saver, profiler and metric stats
588
589
590
        self.memory_saver_adapter = TorchMemorySaverAdapter.create(
            enable=server_args.enable_memory_saver
        )
591
        self.offload_tags = set()
limingshu's avatar
limingshu committed
592
        self.init_profiler()
593

594
        self.recv_skipper = SchedulerRecvSkipper.maybe_create(server_args)
fzyzcjy's avatar
fzyzcjy committed
595
596
597
598
599
600
        self.input_blocker = (
            SchedulerInputBlocker(noop=self.attn_tp_rank != 0)
            if get_bool_env_var("SGLANG_ENABLE_COLOCATED_BATCH_GEN")
            else None
        )

601
        # Init metrics stats
602
        self.init_metrics(tp_rank, pp_rank, dp_rank)
603

604
605
606
        if self.enable_kv_cache_events:
            self.init_kv_events(server_args.kv_events_config)

607
608
609
610
611
612
613
614
615
        # Init disaggregation
        self.disaggregation_mode = DisaggregationMode(
            self.server_args.disaggregation_mode
        )
        self.init_disaggregation()

        if get_bool_env_var("SGLANG_GC_LOG"):
            configure_gc_logger()

616
617
        # Init prefill kv split size when deterministic inference is enabled with various attention backends
        self.init_deterministic_inference_config()
618

619
620
        # Init request dispatcher
        self._request_dispatcher = TypeBasedDispatcher(
621
622
623
            [
                (TokenizedGenerateReqInput, self.handle_generate_request),
                (TokenizedEmbeddingReqInput, self.handle_embedding_request),
624
625
                (BatchTokenizedGenerateReqInput, self.handle_batch_generate_request),
                (BatchTokenizedEmbeddingReqInput, self.handle_batch_embedding_request),
626
                (FlushCacheReqInput, self.flush_cache_wrapped),
627
                (ClearHiCacheReqInput, self.clear_hicache_storage_wrapped),
628
                (AbortReq, self.abort_request),
629
630
                (OpenSessionReqInput, self.open_session),
                (CloseSessionReqInput, self.close_session),
631
632
                (UpdateWeightFromDiskReqInput, self.update_weights_from_disk),
                (InitWeightsUpdateGroupReqInput, self.init_weights_update_group),
633
                (DestroyWeightsUpdateGroupReqInput, self.destroy_weights_update_group),
634
635
636
637
638
639
640
641
                (
                    InitWeightsSendGroupForRemoteInstanceReqInput,
                    self.init_weights_send_group_for_remote_instance,
                ),
                (
                    SendWeightsToRemoteInstanceReqInput,
                    self.send_weights_to_remote_instance,
                ),
642
643
644
645
646
647
                (
                    UpdateWeightsFromDistributedReqInput,
                    self.update_weights_from_distributed,
                ),
                (UpdateWeightsFromTensorReqInput, self.update_weights_from_tensor),
                (GetWeightsByNameReqInput, self.get_weights_by_name),
648
649
                (ReleaseMemoryOccupationReqInput, self.release_memory_occupation),
                (ResumeMemoryOccupationReqInput, self.resume_memory_occupation),
650
                (SlowDownReqInput, self.slow_down),
651
                (ProfileReq, self.profile),
652
                (FreezeGCReq, self.handle_freeze_gc),
653
                (GetInternalStateReq, self.get_internal_state),
654
                (SetInternalStateReq, self.set_internal_state),
655
                (RpcReqInput, self.handle_rpc_request),
656
                (ExpertDistributionReq, self.expert_distribution_handle),
657
658
                (LoadLoRAAdapterReqInput, self.load_lora_adapter),
                (UnloadLoRAAdapterReqInput, self.unload_lora_adapter),
659
                (MultiTokenizerRegisterReq, self.register_multi_tokenizer),
660
                (GetLoadReqInput, self.get_load),
661
662
663
            ]
        )

664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
    def init_deterministic_inference_config(self):
        """Initialize deterministic inference configuration for different attention backends."""
        if not self.server_args.enable_deterministic_inference:
            self.truncation_align_size = None
            return

        backend_sizes = {
            "flashinfer": ("SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE", 4096),
            "triton": ("SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE", 4096),
        }
        env_var, default_size = backend_sizes.get(
            self.server_args.attention_backend, (None, None)
        )
        self.truncation_align_size = (
            get_int_env_var(env_var, default_size) if env_var else None
        )

681
682
683
    def init_tokenizer(self):
        server_args = self.server_args
        self.is_generation = self.model_config.is_generation
684

685
686
687
688
689
690
691
692
693
        if server_args.skip_tokenizer_init:
            self.tokenizer = self.processor = None
        else:
            if self.model_config.is_multimodal:
                self.processor = get_processor(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
694
                    use_fast=not server_args.disable_fast_image_processor,
695
                )
xm:D's avatar
xm:D committed
696
                self.tokenizer = get_tokenizer_from_processor(self.processor)
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
            else:
                self.tokenizer = get_tokenizer(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
                )

    def init_memory_pool_and_cache(self):
        server_args = self.server_args

        self.req_to_token_pool, self.token_to_kv_pool_allocator = (
            self.tp_worker.get_memory_pool()
        )

        if (
            server_args.chunked_prefill_size is not None
            and server_args.disable_radix_cache
        ):
Hanming Lu's avatar
Hanming Lu committed
716
            if self.is_hybrid:
tarinkk's avatar
tarinkk committed
717
718
719
720
                ChunkCacheClass = SWAChunkCache
            else:
                ChunkCacheClass = ChunkCache
            self.tree_cache = ChunkCacheClass(
721
722
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
723
                page_size=self.page_size,
724
725
            )
        else:
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
            if os.environ.get("SGLANG_EXPERIMENTAL_CPP_RADIX_TREE") == "1":
                # lazy import to avoid JIT overhead
                from sglang.srt.mem_cache.radix_cache_cpp import RadixCacheCpp

                self.tree_cache = RadixCacheCpp(
                    disable=False,
                    use_hicache=self.enable_hierarchical_cache,
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool=self.token_to_kv_pool_allocator,
                    tp_cache_group=self.tp_cpu_group,
                    page_size=self.page_size,
                    hicache_ratio=server_args.hicache_ratio,
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
                    enable_kv_cache_events=self.enable_kv_cache_events,
                )
            elif self.enable_hierarchical_cache:
743
744
745
                self.tree_cache = HiRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
746
747
748
749
750
                    tp_cache_group=(
                        self.attn_tp_cpu_group
                        if self.server_args.enable_dp_attention
                        else self.tp_cpu_group
                    ),
751
                    page_size=self.page_size,
752
                    eviction_policy=server_args.radix_eviction_policy,
753
                    hicache_ratio=server_args.hicache_ratio,
Zhiqiang Xie's avatar
Zhiqiang Xie committed
754
755
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
756
                    hicache_io_backend=server_args.hicache_io_backend,
757
                    hicache_mem_layout=server_args.hicache_mem_layout,
758
                    enable_metrics=self.enable_metrics,
759
                    hicache_storage_backend=server_args.hicache_storage_backend,
pansicheng's avatar
pansicheng committed
760
                    hicache_storage_prefetch_policy=server_args.hicache_storage_prefetch_policy,
761
762
                    model_name=server_args.served_model_name,
                    storage_backend_extra_config=server_args.hicache_storage_backend_extra_config,
763
                )
764
765
766
                self.tp_worker.register_hicache_layer_transfer_counter(
                    self.tree_cache.cache_controller.layer_done_counter
                )
Hanming Lu's avatar
Hanming Lu committed
767
768
769
770
771
772
773
774
775
776
777
            elif self.is_hybrid:
                assert (
                    self.server_args.disaggregation_mode == "null"
                ), "Hybrid mode does not support disaggregation yet"
                self.tree_cache = SWARadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                    sliding_window_size=self.sliding_window_size,
                    page_size=self.page_size,
                    disable=server_args.disable_radix_cache,
                )
778
779
780
781
782
783
784
785
786
787
788
789
790
791
            elif server_args.enable_lmcache:
                from sglang.srt.mem_cache.storage.lmcache.lmc_radix_cache import (
                    LMCRadixCache,
                )

                self.tree_cache = LMCRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                    page_size=self.page_size,
                    disable=server_args.disable_radix_cache,
                    model_config=self.model_config,
                    tp_size=self.tp_size,
                    rank=self.tp_rank,
                    tp_group=self.tp_group,
792
                    eviction_policy=server_args.radix_eviction_policy,
793
                )
794
795
796
797
            else:
                self.tree_cache = RadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Lianmin Zheng's avatar
Lianmin Zheng committed
798
                    page_size=self.page_size,
799
                    disable=server_args.disable_radix_cache,
800
                    enable_kv_cache_events=self.enable_kv_cache_events,
801
                    eviction_policy=server_args.radix_eviction_policy,
Ke Bao's avatar
Ke Bao committed
802
                    is_eagle=self.spec_algorithm.is_eagle(),
803
804
                )

805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
        if (
            server_args.disaggregation_mode == "decode"
            and server_args.disaggregation_decode_enable_offload_kvcache
        ):
            self.decode_offload_manager = DecodeKVCacheOffloadManager(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                tp_group=(
                    self.attn_tp_cpu_group
                    if self.server_args.enable_dp_attention
                    else self.tp_cpu_group
                ),
                tree_cache=self.tree_cache,
                server_args=self.server_args,
            )
        else:
            self.decode_offload_manager = None

823
824
825
826
827
828
        self.decode_mem_cache_buf_multiplier = (
            1
            if self.spec_algorithm.is_none()
            else (
                server_args.speculative_num_draft_tokens
                + (
829
830
                    (server_args.speculative_eagle_topk or 1)
                    * (server_args.speculative_num_steps or 1)
831
832
                )
            )
833
        )
834

835
836
837
        embedding_cache_size = int(os.environ.get("SGLANG_VLM_CACHE_SIZE_MB", "100"))
        init_embedding_cache(embedding_cache_size * 1024 * 1024)

Byron Hsu's avatar
Byron Hsu committed
838
    def init_disaggregation(self):
839
840
841
842
        self.transfer_backend = TransferBackend(
            self.server_args.disaggregation_transfer_backend
        )

Byron Hsu's avatar
Byron Hsu committed
843
844
845
846
        if (
            self.disaggregation_mode == DisaggregationMode.DECODE
        ):  # *2 for the headroom.
            buffer_size = (self.req_to_token_pool.size) * 2
Byron Hsu's avatar
Byron Hsu committed
847
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
848
849
                buffer_size
            )
850
851
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
852
                hidden_size=self.model_config.hf_text_config.hidden_size,
853
                hidden_states_dtype=self.model_config.dtype,
854
855
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
856
857
858

            # The decode requests polling kv cache
            self.disagg_decode_transfer_queue = DecodeTransferQueue(
859
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
860
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
861
                tp_rank=self.tp_rank,
862
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
863
864
                scheduler=self,
                tree_cache=self.tree_cache,
Byron Hsu's avatar
Byron Hsu committed
865
866
867
868
869
870
            )

            # The decode requests pending for pre-allocation
            self.disagg_decode_prealloc_queue = DecodePreallocQueue(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Byron Hsu's avatar
Byron Hsu committed
871
872
                draft_token_to_kv_pool=(
                    None
873
                    if self.draft_worker is None or self.spec_algorithm.is_ngram()
Byron Hsu's avatar
Byron Hsu committed
874
875
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
876
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
877
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
878
879
880
                scheduler=self,
                transfer_queue=self.disagg_decode_transfer_queue,
                tree_cache=self.tree_cache,
881
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
882
883
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
884
885
                dp_size=self.server_args.dp_size,
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
886
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
887
888
                max_total_num_tokens=self.max_total_num_tokens,
                prefill_pp_size=self.server_args.disaggregation_prefill_pp,
889
                num_reserved_decode_tokens=self.server_args.num_reserved_decode_tokens,
890
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
891
            )
Liangsheng Yin's avatar
Liangsheng Yin committed
892

Byron Hsu's avatar
Byron Hsu committed
893
894
895
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            # *2 for the headroom.
            buffer_size = self.max_running_requests * 2
Byron Hsu's avatar
Byron Hsu committed
896
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
897
898
                buffer_size
            )
899
900
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
901
                hidden_size=self.model_config.hf_text_config.hidden_size,
902
                hidden_states_dtype=self.model_config.dtype,
903
904
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
905

Liangsheng Yin's avatar
Liangsheng Yin committed
906
            self.disagg_prefill_bootstrap_queue = PrefillBootstrapQueue(
Byron Hsu's avatar
Byron Hsu committed
907
                token_to_kv_pool=self.token_to_kv_pool_allocator.get_kvcache(),
Byron Hsu's avatar
Byron Hsu committed
908
909
                draft_token_to_kv_pool=(
                    None
910
                    if self.draft_worker is None or self.spec_algorithm.is_ngram()
Byron Hsu's avatar
Byron Hsu committed
911
912
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
913
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
914
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
915
916
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
Byron Hsu's avatar
Byron Hsu committed
917
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
918
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
919
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
920
921
922
                max_total_num_tokens=self.max_total_num_tokens,
                decode_tp_size=self.server_args.disaggregation_decode_tp,
                decode_dp_size=self.server_args.disaggregation_decode_dp,
923
                scheduler=self,
Byron Hsu's avatar
Byron Hsu committed
924
925
926
                pp_rank=self.pp_rank,
                pp_size=self.pp_size,
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
927
928
            )
            # The prefill requests that are in the middle of kv sending
929
            self.disagg_prefill_inflight_queue: List[Req] = []
Byron Hsu's avatar
Byron Hsu committed
930

931
932
933
934
    def init_moe_config(self):
        if hasattr(self.model_config.hf_config, "num_experts_per_tok"):
            initialize_moe_config(self.server_args)

935
    @DynamicGradMode()
936
    def event_loop_normal(self):
937
        """A normal scheduler loop."""
938
        while True:
Lianmin Zheng's avatar
Lianmin Zheng committed
939
940
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)
941

942
            batch = self.get_next_batch_to_run()
Lianmin Zheng's avatar
Lianmin Zheng committed
943
            self.cur_batch = batch
944
945
946
947

            if batch:
                result = self.run_batch(batch)
                self.process_batch_result(batch, result)
Lianmin Zheng's avatar
Lianmin Zheng committed
948
            else:
Lianmin Zheng's avatar
Lianmin Zheng committed
949
                # When the server is idle, do self-check and re-init some states
950
                self.self_check_during_idle()
951
952

            self.last_batch = batch
953

954
    @DynamicGradMode()
Lianmin Zheng's avatar
Lianmin Zheng committed
955
    def event_loop_overlap(self):
956
        """A scheduler loop that overlaps the CPU processing and GPU computation."""
957
        self.result_queue = deque()
Lianmin Zheng's avatar
Lianmin Zheng committed
958
959
960
961
962
963
964

        while True:
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)

            batch = self.get_next_batch_to_run()
            self.cur_batch = batch
965

Lianmin Zheng's avatar
Lianmin Zheng committed
966
            if batch:
967
                batch.launch_done = threading.Event()
Lianmin Zheng's avatar
Lianmin Zheng committed
968
                result = self.run_batch(batch)
969
                self.result_queue.append((batch.copy(), result))
Lianmin Zheng's avatar
Lianmin Zheng committed
970

971
                if self.last_batch is None:
972
                    # Create a dummy first batch to start the pipeline for overlap schedule.
973
974
975
976
977
978
                    # It is now used for triggering the sampling_info_done event.
                    tmp_batch = ScheduleBatch(
                        reqs=None,
                        forward_mode=ForwardMode.DUMMY_FIRST,
                        next_batch_sampling_info=self.tp_worker.cur_sampling_info,
                    )
979
                    self.process_batch_result(tmp_batch, None, batch.launch_done)
980

Lianmin Zheng's avatar
Lianmin Zheng committed
981
            if self.last_batch:
982
                # Process the results of the last batch
983
                tmp_batch, tmp_result = self.result_queue.popleft()
984
985
986
                tmp_batch.next_batch_sampling_info = (
                    self.tp_worker.cur_sampling_info if batch else None
                )
987
988
989
990
                # NOTE: we should use current launched batch's launch_done event Instead of the last batch's
                self.process_batch_result(
                    tmp_batch, tmp_result, batch.launch_done if batch else None
                )
Lianmin Zheng's avatar
Lianmin Zheng committed
991
            elif batch is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
992
                # When the server is idle, do self-check and re-init some states
993
                self.self_check_during_idle()
Lianmin Zheng's avatar
Lianmin Zheng committed
994
995
996

            self.last_batch = batch

997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
    @DynamicGradMode()
    def event_loop_pp(self):
        """A non-overlap scheduler loop for pipeline parallelism."""
        mbs = [None] * self.pp_size
        last_mbs = [None] * self.pp_size
        self.running_mbs = [
            ScheduleBatch(reqs=[], batch_is_full=False) for _ in range(self.pp_size)
        ]
        pp_outputs: Optional[PPProxyTensors] = None
        while True:
            server_is_idle = True
            for mb_id in range(self.pp_size):
                self.running_batch = self.running_mbs[mb_id]
                self.last_batch = last_mbs[mb_id]

                recv_reqs = self.recv_requests()
                self.process_input_requests(recv_reqs)
                mbs[mb_id] = self.get_next_batch_to_run()
                self.running_mbs[mb_id] = self.running_batch

                self.cur_batch = mbs[mb_id]
                if self.cur_batch:
                    server_is_idle = False
                    result = self.run_batch(self.cur_batch)

1022
                # (last rank) send the outputs to the next step
1023
1024
                if self.pp_group.is_last_rank:
                    if self.cur_batch:
1025
                        next_token_ids = result.next_token_ids
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
                        if self.cur_batch.return_logprob:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                    "extend_input_len_per_req": result.extend_input_len_per_req,
                                    "extend_logprob_start_len_per_req": result.extend_logprob_start_len_per_req,
                                }
                                | (
                                    {
                                        f"logits_output.{k}": v
                                        for k, v in result.logits_output.__dict__.items()
                                    }
                                    if result.logits_output is not None
                                    else {}
                                )
                            )
                        else:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                }
                            )
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
                        # send the output from the last round to let the next stage worker run post processing
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                # receive outputs and post-process (filter finished reqs) the coming microbatch
                next_mb_id = (mb_id + 1) % self.pp_size
                next_pp_outputs = None
                if mbs[next_mb_id] is not None:
                    next_pp_outputs: Optional[PPProxyTensors] = PPProxyTensors(
                        self.pp_group.recv_tensor_dict(
                            all_gather_group=self.attn_tp_group
                        )
                    )
                    mbs[next_mb_id].output_ids = next_pp_outputs["next_token_ids"]
1064
1065
1066
1067
1068
1069
1070
1071
1072
                    logits_output_args = {
                        k[len("logits_output.") :]: v
                        for k, v in next_pp_outputs.tensors.items()
                        if k.startswith("logits_output.")
                    }
                    if len(logits_output_args) > 0:
                        logits_output = LogitsProcessorOutput(**logits_output_args)
                    else:
                        logits_output = None
1073
1074

                    output_result = GenerationBatchResult.from_pp_proxy(
1075
                        logits_output=logits_output,
1076
                        next_pp_outputs=next_pp_outputs,
1077
                        can_run_cuda_graph=result.can_run_cuda_graph,
1078
1079
1080
1081
                    )
                    self.process_batch_result(mbs[next_mb_id], output_result)
                    last_mbs[next_mb_id] = mbs[next_mb_id]

1082
                # (not last rank)
1083
                if not self.pp_group.is_last_rank:
1084
1085
                    # carry the outputs to the next stage
                    # send the outputs from the last round to let the next stage worker run post processing
1086
1087
1088
1089
1090
1091
1092
                    if pp_outputs:
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                    # send out reqs to the next stage
1093
                    dp_offset = self.attn_dp_rank * self.attn_tp_size
1094
1095
1096
1097
                    if self.attn_tp_rank == 0:
                        point_to_point_pyobj(
                            recv_reqs,
                            self.pp_rank * self.tp_size + dp_offset,
1098
                            self.world_group.device_group,
1099
1100
1101
1102
1103
1104
                            self.pp_rank * self.tp_size + dp_offset,
                            (self.pp_rank + 1) * self.tp_size + dp_offset,
                        )

                    # send out proxy tensors to the next stage
                    if self.cur_batch:
1105
1106
                        # FIXME(lsyin): remove this assert
                        assert result.pp_hidden_states_proxy_tensors.tensors is not None
1107
                        self.pp_group.send_tensor_dict(
1108
                            result.pp_hidden_states_proxy_tensors.tensors,
1109
1110
1111
1112
1113
1114
1115
                            all_gather_group=self.attn_tp_group,
                        )

                pp_outputs = next_pp_outputs

            # When the server is idle, self-check and re-init some states
            if server_is_idle:
1116
1117
                # When the server is idle, do self-check and re-init some states
                self.self_check_during_idle()
1118

1119
1120
    def recv_requests(self) -> List[Req]:
        """Receive results at tp_rank = 0 and broadcast it to all other TP ranks."""
1121
1122
1123
1124
1125
1126
1127
1128

        if self.recv_skipper is not None:
            last_forward_mode = (
                self.last_batch.forward_mode if self.last_batch is not None else None
            )
            if not self.recv_skipper.handle(last_forward_mode):
                return []

1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
        if self.pp_rank == 0:
            if self.attn_tp_rank == 0:
                recv_reqs = []

                while True:
                    try:
                        recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_req)

                while True:
                    try:
                        recv_rpc = self.recv_from_rpc.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_rpc)
            else:
                recv_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1148
        else:
1149
            if self.attn_tp_rank == 0:
1150
                dp_offset = self.attn_dp_rank * self.attn_tp_size
1151
1152
1153
                recv_reqs = point_to_point_pyobj(
                    [],
                    self.pp_rank * self.tp_size + dp_offset,
1154
                    self.world_group.device_group,
1155
1156
1157
1158
1159
                    (self.pp_rank - 1) * self.tp_size + dp_offset,
                    self.pp_rank * self.tp_size + dp_offset,
                )
            else:
                recv_reqs = None
1160

fzyzcjy's avatar
fzyzcjy committed
1161
1162
1163
        if self.input_blocker is not None:
            recv_reqs = self.input_blocker.handle(recv_reqs)

1164
1165
1166
1167
1168
1169
        if self.server_args.enable_dp_attention:
            if self.attn_tp_rank == 0:
                work_reqs = [
                    req
                    for req in recv_reqs
                    if isinstance(
1170
1171
1172
1173
1174
1175
1176
                        req,
                        (
                            TokenizedGenerateReqInput,
                            TokenizedEmbeddingReqInput,
                            BatchTokenizedGenerateReqInput,
                            BatchTokenizedEmbeddingReqInput,
                        ),
1177
1178
1179
1180
1181
1182
                    )
                ]
                control_reqs = [
                    req
                    for req in recv_reqs
                    if not isinstance(
1183
1184
1185
1186
1187
1188
1189
                        req,
                        (
                            TokenizedGenerateReqInput,
                            TokenizedEmbeddingReqInput,
                            BatchTokenizedGenerateReqInput,
                            BatchTokenizedEmbeddingReqInput,
                        ),
1190
1191
1192
1193
1194
1195
1196
1197
1198
                    )
                ]
            else:
                work_reqs = None
                control_reqs = None

            if self.attn_tp_size != 1:
                work_reqs = broadcast_pyobj(
                    work_reqs,
1199
                    self.attn_tp_group.rank,
1200
                    self.attn_tp_cpu_group,
1201
                    src=self.attn_tp_group.ranks[0],
1202
1203
1204
                )
            if self.tp_size != 1:
                control_reqs = broadcast_pyobj(
1205
1206
1207
1208
                    control_reqs,
                    self.tp_group.rank,
                    self.tp_cpu_group,
                    src=self.tp_group.ranks[0],
1209
1210
1211
                )
            recv_reqs = work_reqs + control_reqs
        elif self.tp_size != 1:
1212
1213
1214
1215
1216
1217
            recv_reqs = broadcast_pyobj(
                recv_reqs,
                self.tp_group.rank,
                self.tp_cpu_group,
                src=self.tp_group.ranks[0],
            )
1218

1219
1220
1221
1222
1223
1224
1225
        if self.enable_trace:
            for req in recv_reqs:
                if isinstance(
                    req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                ):
                    trace_set_proc_propagate_context(req.rid, req.trace_context)
                    trace_slice_start("", req.rid, anonymous=True)
1226

1227
1228
        return recv_reqs

Lianmin Zheng's avatar
Lianmin Zheng committed
1229
    def process_input_requests(self, recv_reqs: List):
1230
        for recv_req in recv_reqs:
1231
1232
            # If it is a health check generation request and there are running requests, ignore it.
            if is_health_check_generate_req(recv_req) and (
1233
1234
1235
                self.chunked_req is not None
                or not self.running_batch.is_empty()
                or len(self.offload_tags) > 0
1236
1237
1238
1239
            ):
                self.return_health_check_ct += 1
                continue

1240
1241
            # If it is a MultiTokenizerWrapper, unwrap it and handle the inner request.
            if isinstance(recv_req, MultiTokenizerWrapper):
1242
1243
1244
1245
                worker_id = recv_req.worker_id
                recv_req = recv_req.obj
                output = self._request_dispatcher(recv_req)
                if output is not None:
1246
                    output = MultiTokenizerWrapper(worker_id, output)
1247
1248
1249
                    self.send_to_tokenizer.send_pyobj(output)
                continue

1250
            output = self._request_dispatcher(recv_req)
1251
            if output is not None:
1252
1253
1254
1255
1256
                if isinstance(output, RpcReqOutput):
                    if self.recv_from_rpc is not None:
                        self.recv_from_rpc.send_pyobj(output)
                else:
                    self.send_to_tokenizer.send_pyobj(output)
1257

1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
    def init_req_max_new_tokens(self, req):
        req.sampling_params.max_new_tokens = min(
            (
                req.sampling_params.max_new_tokens
                if req.sampling_params.max_new_tokens is not None
                else 1 << 30
            ),
            self.max_req_len - len(req.origin_input_ids) - 1,
        )

1268
1269
1270
1271
    def handle_generate_request(
        self,
        recv_req: TokenizedGenerateReqInput,
    ):
1272
        # Create a new request
1273
1274
1275
1276
1277
        if (
            recv_req.session_params is None
            or recv_req.session_params.id is None
            or recv_req.session_params.id not in self.sessions
        ):
Rin Intachuen's avatar
Rin Intachuen committed
1278
1279
1280
1281
1282
1283
            if recv_req.input_embeds is not None:
                # Generate fake input_ids based on the length of input_embeds
                seq_length = len(recv_req.input_embeds)
                fake_input_ids = [1] * seq_length
                recv_req.input_ids = fake_input_ids

1284
1285
1286
1287
            if recv_req.bootstrap_port is None:
                # Use default bootstrap port
                recv_req.bootstrap_port = self.server_args.disaggregation_bootstrap_port

1288
1289
1290
1291
1292
            req = Req(
                recv_req.rid,
                recv_req.input_text,
                recv_req.input_ids,
                recv_req.sampling_params,
Lianmin Zheng's avatar
Lianmin Zheng committed
1293
1294
                return_logprob=recv_req.return_logprob,
                top_logprobs_num=recv_req.top_logprobs_num,
1295
                token_ids_logprob=recv_req.token_ids_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1296
                stream=recv_req.stream,
1297
                lora_id=recv_req.lora_id,
Rin Intachuen's avatar
Rin Intachuen committed
1298
                input_embeds=recv_req.input_embeds,
Lianmin Zheng's avatar
Lianmin Zheng committed
1299
                custom_logit_processor=recv_req.custom_logit_processor,
1300
                return_hidden_states=recv_req.return_hidden_states,
1301
                eos_token_ids=self.model_config.hf_eos_token_id,
1302
                bootstrap_host=recv_req.bootstrap_host,
1303
                bootstrap_port=recv_req.bootstrap_port,
1304
                bootstrap_room=recv_req.bootstrap_room,
1305
                disagg_mode=self.disaggregation_mode,
1306
                data_parallel_rank=recv_req.data_parallel_rank,
1307
                vocab_size=self.model_config.vocab_size,
1308
                priority=recv_req.priority,
1309
1310
1311
                metrics_collector=(
                    self.metrics_collector if self.enable_metrics else None
                ),
1312
1313
            )
            req.tokenizer = self.tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
1314

1315
1316
1317
            if self.disaggregation_mode != DisaggregationMode.NULL:
                # Invalid request for disaggregated mode
                if recv_req.bootstrap_room is None:
1318
                    error_msg = (
1319
1320
1321
                        f"Invalid request: Disaggregated request received without "
                        f"boostrap room id. {req.rid=}"
                    )
1322
                    logger.error(error_msg)
1323
                    prepare_abort(req, error_msg, status_code=HTTPStatus.BAD_REQUEST)
1324
1325
1326
                    self.stream_output([req], req.return_logprob)
                    return

1327
1328
1329
1330
            if (
                recv_req.session_params is not None
                and recv_req.session_params.id is not None
            ):
1331
                req.set_finish_with_abort(
1332
                    f"Invalid request: session id {recv_req.session_params.id} does not exist"
1333
                )
1334
                self.init_req_max_new_tokens(req)
1335
                self._add_request_to_queue(req)
1336
1337
                return
        else:
1338
1339
            # Create a new request from a previous session
            session = self.sessions[recv_req.session_params.id]
1340
            req = session.create_req(recv_req, self.tokenizer)
1341
            if isinstance(req.finished_reason, FINISH_ABORT):
1342
                self.init_req_max_new_tokens(req)
1343
                self._add_request_to_queue(req)
1344
                return
1345

1346
        # Handle multimodal inputs
Mick's avatar
Mick committed
1347
1348
        if recv_req.mm_inputs is not None:
            image_inputs = MultimodalInputs.from_dict(recv_req.mm_inputs)
1349
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
1350
            req.origin_input_ids = self.pad_input_ids_func(
1351
                req.origin_input_ids, image_inputs
1352
            )
1353
            req.extend_image_inputs(image_inputs)
1354

1355
            if len(req.origin_input_ids) >= self.max_req_input_len:
1356
1357
1358
1359
1360
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1361
                )
1362
                self.init_req_max_new_tokens(req)
1363
                self._add_request_to_queue(req)
1364
1365
                return

1366
1367
1368
        # initialize before returning
        self.init_req_max_new_tokens(req)

1369
        # Validate prompt length
1370
1371
1372
1373
1374
1375
        error_msg = validate_input_length(
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
        if error_msg:
1376
            req.set_finish_with_abort(error_msg)
1377
            self._add_request_to_queue(req)
1378
            return
1379

1380
        # Copy more attributes
1381
        if recv_req.logprob_start_len == -1 or not recv_req.return_logprob:
1382
            # By default, only return the logprobs for output tokens
1383
1384
1385
1386
1387
1388
1389
            # For prefill-only requests with logprob_start_len == -1, set logprob_start_len beyond input sequence
            # to skip input logprob computation entirely
            if req.is_prefill_only:
                req.logprob_start_len = len(req.origin_input_ids)
            else:
                # TODO: For text generation, evaluate setting logprob_start_len to len(req.origin_input_ids) as well
                req.logprob_start_len = len(req.origin_input_ids) - 1
1390
1391
1392
        else:
            req.logprob_start_len = recv_req.logprob_start_len

1393
1394
1395
        if not req.is_prefill_only and req.logprob_start_len >= len(
            req.origin_input_ids
        ):
1396
            error_msg = f"{req.logprob_start_len=} is higher than the number of input tokens {len(req.origin_input_ids)=}. Please use a smaller logprob_start_len."
1397
            req.logprob_start_len = len(req.origin_input_ids) - 1
1398
            req.set_finish_with_abort(error_msg)
1399
1400
1401
            self._add_request_to_queue(req)
            return

1402
1403
1404
1405
1406
        # Init grammar cache for this request
        add_to_grammar_queue = False
        if (
            req.sampling_params.json_schema is not None
            or req.sampling_params.regex is not None
1407
            or req.sampling_params.ebnf is not None
1408
            or req.sampling_params.structural_tag is not None
1409
1410
1411
1412
1413
1414
        ):
            assert self.grammar_backend is not None
            if req.sampling_params.json_schema is not None:
                key = ("json", req.sampling_params.json_schema)
            elif req.sampling_params.regex is not None:
                key = ("regex", req.sampling_params.regex)
1415
1416
            elif req.sampling_params.ebnf is not None:
                key = ("ebnf", req.sampling_params.ebnf)
1417
1418
            elif req.sampling_params.structural_tag:
                key = ("structural_tag", req.sampling_params.structural_tag)
1419

1420
1421
1422
1423
1424
            value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
            req.grammar = value

            if not cache_hit:
                req.grammar_key = key
1425
                add_to_grammar_queue = True
1426
1427
1428
1429
            else:
                if value is INVALID_GRAMMAR_OBJ:  # We hit a cached invalid grammar.
                    error_msg = f"Invalid grammar request with cache hit: {key=}"
                    req.set_finish_with_abort(error_msg)
1430
1431

        if add_to_grammar_queue:
1432
1433
            self.grammar_queue.append(req)
        else:
1434
1435
            self._add_request_to_queue(req)

1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
    def handle_batch_generate_request(
        self,
        recv_req: BatchTokenizedGenerateReqInput,
    ):
        """Handle optimized batch generate request."""
        logger.debug(f"Processing batch generate request with {len(recv_req)} requests")

        # Process each request in the batch
        for tokenized_req in recv_req:
            self.handle_generate_request(tokenized_req)

1447
1448
1449
    def _prefetch_kvcache(self, req: Req):
        if self.enable_hicache_storage:
            req.init_next_round_input(self.tree_cache)
1450
1451
1452
1453
1454
            if req.last_node.backuped:
                # only to initiate the prefetch if the last node is backuped
                # otherwise, the allocated GPU memory must be locked for integrity
                last_hash = req.last_host_node.get_last_hash_value()
                matched_len = len(req.prefix_indices) + req.host_hit_length
1455
1456
1457
1458
1459
                new_input_tokens = req.fill_ids[matched_len:]
                self.tree_cache.prefetch_from_storage(
                    req.rid, req.last_host_node, new_input_tokens, last_hash
                )

1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
    def _add_request_to_queue(self, req: Req, is_retracted: bool = False):
        if self.disaggregation_mode == DisaggregationMode.NULL:
            self._set_or_validate_priority(req)
            if self._abort_on_queued_limit(req):
                return
            self._prefetch_kvcache(req)
            self.waiting_queue.append(req)
            req.time_stats.wait_queue_entry_time = time.perf_counter()
            trace_slice_end("process req", req.rid, auto_next_anon=True)
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            self._prefetch_kvcache(req)
            self.disagg_prefill_bootstrap_queue.add(
                req, self.model_config.num_key_value_heads
Byron Hsu's avatar
Byron Hsu committed
1473
            )
1474
            req.time_stats.prefill_bootstrap_queue_entry_time = time.perf_counter()
1475
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
1476
1477
1478
            self.disagg_decode_prealloc_queue.add(req, is_retracted=is_retracted)
            if not is_retracted:
                req.time_stats.decode_prealloc_queue_entry_time = time.perf_counter()
Byron Hsu's avatar
Byron Hsu committed
1479
        else:
1480
            raise ValueError(f"Invalid {self.disaggregation_mode=}")
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495

    def _set_or_validate_priority(self, req: Req):
        """Set the default priority value, or abort the request based on the priority scheduling mode."""
        if self.enable_priority_scheduling and req.priority is None:
            if self.schedule_low_priority_values_first:
                req.priority = sys.maxsize
            else:
                req.priority = -sys.maxsize - 1
        elif not self.enable_priority_scheduling and req.priority is not None:
            abort_req = AbortReq(
                finished_reason={
                    "type": "abort",
                    "status_code": HTTPStatus.SERVICE_UNAVAILABLE,
                    "message": "Using priority is disabled for this server. Please send a new request without a priority.",
                },
1496
                rid=req.rid,
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
            )
            self.send_to_tokenizer.send_pyobj(abort_req)

    def _abort_on_queued_limit(self, recv_req: Req) -> bool:
        """Abort an incoming or existing request if the waiting queue is full. Returns True if the incoming request is aborted."""
        if (
            self.max_queued_requests is None
            or len(self.waiting_queue) + 1 <= self.max_queued_requests
        ):
            return False

        # Reject the incoming request by default.
        req_to_abort = recv_req
        message = "The request queue is full."
        if self.enable_priority_scheduling:
            # With priority scheduling, consider aboritng an existing request based on the priority.
            # direction = 1  => smaller number = higher priority; -1 => larger number = higher priority.
            # max(...) + (direction * priority, queue_time_start) picks the least-preferred request.
            # Tie: later queue_time_start (newer) is evicted first. Preempt only if strictly better.
            direction = 1 if self.schedule_low_priority_values_first else -1
            key_fn = lambda item: (
                direction * item[1].priority,
1519
                item[1].time_stats.wait_queue_entry_time,
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
            )
            idx, candidate_req = max(enumerate(self.waiting_queue), key=key_fn)
            abort_existing_req = (
                direction * recv_req.priority < direction * candidate_req.priority
            )
            if abort_existing_req:
                self.waiting_queue.pop(idx)
                req_to_abort = candidate_req
                message = "The request is aborted by a higher priority request."

        self.send_to_tokenizer.send_pyobj(
            AbortReq(
                finished_reason={
                    "type": "abort",
                    "status_code": HTTPStatus.SERVICE_UNAVAILABLE,
                    "message": message,
                },
1537
                rid=req_to_abort.rid,
1538
1539
1540
            )
        )
        return req_to_abort.rid == recv_req.rid
1541
1542
1543

    def handle_embedding_request(
        self,
1544
        recv_req: TokenizedEmbeddingReqInput,
1545
1546
1547
1548
1549
1550
    ):
        req = Req(
            recv_req.rid,
            recv_req.input_text,
            recv_req.input_ids,
            recv_req.sampling_params,
woodx's avatar
woodx committed
1551
            token_type_ids=recv_req.token_type_ids,
1552
            priority=recv_req.priority,
1553
1554
1555
        )
        req.tokenizer = self.tokenizer

1556
1557
        # Handle multimodal inputs
        if recv_req.image_inputs is not None:
Mick's avatar
Mick committed
1558
            image_inputs = MultimodalInputs.from_dict(recv_req.image_inputs)
1559
1560
1561
1562
1563
1564
1565
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
            req.origin_input_ids = self.pad_input_ids_func(
                req.origin_input_ids, image_inputs
            )
            req.extend_image_inputs(image_inputs)

            if len(req.origin_input_ids) >= self.max_req_input_len:
1566
1567
1568
1569
1570
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1571
                )
1572
                self._add_request_to_queue(req)
1573
1574
                return

1575
        # Validate prompts length
1576
        error_msg = validate_input_length(
1577
1578
1579
1580
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
1581
        if error_msg:
1582
            self._add_request_to_queue(req)
1583
            return
1584

1585
1586
        # Copy more attributes
        req.logprob_start_len = len(req.origin_input_ids) - 1
1587
        self._add_request_to_queue(req)
1588

1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
    def handle_batch_embedding_request(
        self,
        recv_req: BatchTokenizedEmbeddingReqInput,
    ):
        """Handle optimized batch embedding request."""
        logger.debug(
            f"Processing batch embedding request with {len(recv_req)} requests"
        )

        # Process each request in the batch
        for tokenized_req in recv_req:
            self.handle_embedding_request(tokenized_req)

1602
1603
1604
1605
1606
    def self_check_during_idle(self):
        self.check_memory()
        self.check_tree_cache()
        self.new_token_ratio = self.init_new_token_ratio
        self.maybe_sleep_on_idle()
1607

Lianmin Zheng's avatar
Lianmin Zheng committed
1608
    def check_memory(self):
Hanming Lu's avatar
Hanming Lu committed
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
        if self.is_hybrid:
            (
                full_num_used,
                swa_num_used,
                _,
                _,
                full_available_size,
                full_evictable_size,
                swa_available_size,
                swa_evictable_size,
            ) = self._get_swa_token_info()
            memory_leak = full_num_used != 0 or swa_num_used != 0
            token_msg = (
                f"{self.full_tokens_per_layer=}, {full_available_size=}, {full_evictable_size=}, {self.tree_cache.full_protected_size()=}\n"
                f"{self.swa_tokens_per_layer=}, {swa_available_size=}, {swa_evictable_size=}, {self.tree_cache.swa_protected_size()=}\n"
            )
tarinkk's avatar
tarinkk committed
1625
        else:
Hanming Lu's avatar
Hanming Lu committed
1626
1627
1628
            _, _, available_size, evictable_size = self._get_token_info()
            protected_size = self.tree_cache.protected_size()
            memory_leak = (available_size + evictable_size) != (
1629
1630
1631
                # self.max_total_num_tokens
                # if not self.enable_hierarchical_cache
                # else self.max_total_num_tokens - protected_size
Hanming Lu's avatar
Hanming Lu committed
1632
                self.max_total_num_tokens
1633
                - protected_size
Lianmin Zheng's avatar
Lianmin Zheng committed
1634
            )
Hanming Lu's avatar
Hanming Lu committed
1635
1636
1637
1638
            token_msg = f"{self.max_total_num_tokens=}, {available_size=}, {evictable_size=}, {protected_size=}\n"

        if memory_leak:
            msg = "token_to_kv_pool_allocator memory leak detected! " f"{token_msg}"
Lianmin Zheng's avatar
Lianmin Zheng committed
1639
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1640

1641
1642
1643
1644
1645
1646
1647
1648
        if self.disaggregation_mode == DisaggregationMode.DECODE:
            req_total_size = (
                self.req_to_token_pool.size + self.req_to_token_pool.pre_alloc_size
            )
        else:
            req_total_size = self.req_to_token_pool.size

        if len(self.req_to_token_pool.free_slots) != req_total_size:
1649
            msg = (
1650
                "req_to_token_pool memory leak detected!"
1651
1652
                f"available_size={len(self.req_to_token_pool.free_slots)}, "
                f"total_size={self.req_to_token_pool.size}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1653
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1654
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1655

1656
1657
        if (
            self.enable_metrics
1658
            and self.current_scheduler_metrics_enabled()
1659
            and time.perf_counter() > self.metrics_collector.last_log_time + 30
1660
1661
        ):
            # During idle time, also collect metrics every 30 seconds.
Hanming Lu's avatar
Hanming Lu committed
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
            if self.is_hybrid:
                (
                    full_num_used,
                    swa_num_used,
                    full_token_usage,
                    swa_token_usage,
                    _,
                    _,
                    _,
                    _,
                ) = self._get_swa_token_info()
                num_used = max(full_num_used, swa_num_used)
                token_usage = max(full_token_usage, swa_token_usage)
            else:
                num_used, token_usage, _, _ = self._get_token_info()
Lianmin Zheng's avatar
Lianmin Zheng committed
1677
            num_running_reqs = len(self.running_batch.reqs)
1678
1679
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
Hanming Lu's avatar
Hanming Lu committed
1680
            self.stats.token_usage = round(token_usage, 2)
1681
1682
            self.stats.gen_throughput = 0
            self.stats.num_queue_reqs = len(self.waiting_queue)
1683
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
            if self.disaggregation_mode == DisaggregationMode.PREFILL:
                self.stats.num_prefill_prealloc_queue_reqs = len(
                    self.disagg_prefill_bootstrap_queue.queue
                )
                self.stats.num_prefill_inflight_queue_reqs = len(
                    self.disagg_prefill_inflight_queue
                )
            if self.disaggregation_mode == DisaggregationMode.DECODE:
                self.stats.num_decode_prealloc_queue_reqs = len(
                    self.disagg_decode_prealloc_queue.queue
                )
                self.stats.num_decode_transfer_queue_reqs = len(
                    self.disagg_decode_transfer_queue.queue
                )
1698
            self.metrics_collector.log_stats(self.stats)
1699
        self._publish_kv_events()
1700

Hanming Lu's avatar
Hanming Lu committed
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
    def check_tree_cache(self):
        if self.is_hybrid and isinstance(self.tree_cache, SWARadixCache):
            self.tree_cache.sanity_check()

    def _get_token_info(self):
        available_size = self.token_to_kv_pool_allocator.available_size()
        evictable_size = self.tree_cache.evictable_size()
        num_used = self.max_total_num_tokens - (available_size + evictable_size)
        token_usage = num_used / self.max_total_num_tokens
        return num_used, token_usage, available_size, evictable_size

    def _get_swa_token_info(self):
        full_available_size = self.token_to_kv_pool_allocator.full_available_size()
        full_evictable_size = self.tree_cache.full_evictable_size()
        swa_available_size = self.token_to_kv_pool_allocator.swa_available_size()
        swa_evictable_size = self.tree_cache.swa_evictable_size()
        full_num_used = self.full_tokens_per_layer - (
            full_available_size + full_evictable_size
        )
        swa_num_used = self.swa_tokens_per_layer - (
            swa_available_size + swa_evictable_size
        )
        full_token_usage = full_num_used / self.full_tokens_per_layer
        swa_token_usage = swa_num_used / self.swa_tokens_per_layer
        return (
            full_num_used,
            swa_num_used,
            full_token_usage,
            swa_token_usage,
            full_available_size,
            full_evictable_size,
            swa_available_size,
            swa_evictable_size,
        )

1736
    def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
1737
        # Merge the prefill batch into the running batch
1738
1739
1740
1741
1742
        chunked_req_to_exclude = set()
        if self.chunked_req:
            # Move the chunked request out of the batch so that we can merge
            # only finished requests to running_batch.
            chunked_req_to_exclude.add(self.chunked_req)
1743
            self.tree_cache.cache_unfinished_req(self.chunked_req, chunked=True)
1744
            # chunked request keeps its rid but will get a new req_pool_idx
Yi Zhang's avatar
Yi Zhang committed
1745
1746
1747
1748
1749
1750
            if self.tp_worker.worker.model_runner.is_hybrid_gdn:
                self.req_to_token_pool.free(
                    self.chunked_req.req_pool_idx, free_mamba_cache=False
                )
            else:
                self.req_to_token_pool.free(self.chunked_req.req_pool_idx)
Lianmin Zheng's avatar
Lianmin Zheng committed
1751
        if self.last_batch and self.last_batch.forward_mode.is_extend():
1752
1753
1754
1755
            if self.last_batch.chunked_req is not None:
                # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
                # We need to discard it.
                chunked_req_to_exclude.add(self.last_batch.chunked_req)
Lianmin Zheng's avatar
Lianmin Zheng committed
1756

1757
            # Filter batch
1758
            last_bs = self.last_batch.batch_size()
1759
1760
1761
            self.last_batch.filter_batch(
                chunked_req_to_exclude=list(chunked_req_to_exclude)
            )
1762
            if self.last_batch.batch_size() < last_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1763
                self.running_batch.batch_is_full = False
1764

1765
1766
1767
            # Merge the new batch into the running batch.
            # For prefill-only batch, we can avoid going through decoding step.
            if not self.last_batch.is_empty() and not self.last_batch.is_prefill_only:
Lianmin Zheng's avatar
Lianmin Zheng committed
1768
                if self.running_batch.is_empty():
1769
1770
                    self.running_batch = self.last_batch
                else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1771
                    # Merge running_batch with prefill batch
1772
                    self.running_batch.merge_batch(self.last_batch)
1773

1774
        new_batch = self.get_new_batch_prefill()
1775

1776
1777
1778
1779
1780
        need_dp_attn_preparation = require_mlp_sync(self.server_args)

        if need_dp_attn_preparation and not self.spec_algorithm.is_none():
            # In speculative decoding, prefill batches and decode batches cannot be processed in the same DP attention group.
            # We prepare idle batches in advance to skip preparing decode batches when there are prefill batches in the group.
1781
            new_batch = self.prepare_mlp_sync_batch(new_batch)
1782
1783
1784
            need_dp_attn_preparation = new_batch is None

        if new_batch is not None:
1785
1786
1787
1788
            # Run prefill first if possible
            ret = new_batch
        else:
            # Run decode
Lianmin Zheng's avatar
Lianmin Zheng committed
1789
            if not self.running_batch.is_empty():
1790
                self.running_batch = self.update_running_batch(self.running_batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1791
1792
1793
                ret = self.running_batch if not self.running_batch.is_empty() else None
            else:
                ret = None
1794

1795
1796
        # Handle DP attention
        if need_dp_attn_preparation:
1797
            ret = self.prepare_mlp_sync_batch(ret)
1798
1799

        return ret
1800

1801
1802
1803
1804
1805
1806
    def get_num_allocatable_reqs(self, running_bs):
        res = global_server_args_dict["max_micro_batch_size"] - running_bs
        if self.pp_size > 1:
            res = min(res, self.req_to_token_pool.available_size())
        return res

Lianmin Zheng's avatar
Lianmin Zheng committed
1807
    def get_new_batch_prefill(self) -> Optional[ScheduleBatch]:
Lianmin Zheng's avatar
Lianmin Zheng committed
1808
        # Check if the grammar is ready in the grammar queue
1809
        if self.grammar_queue:
1810
            self.move_ready_grammar_requests()
1811

1812
1813
1814
1815
        if self.try_preemption:
            # Reset batch_is_full to try preemption with a prefill adder.
            self.running_batch.batch_is_full = False

Lianmin Zheng's avatar
Lianmin Zheng committed
1816
1817
        # Handle the cases where prefill is not allowed
        if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1818
            self.running_batch.batch_is_full or len(self.waiting_queue) == 0
1819
        ) and self.chunked_req is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
1820
1821
            return None

Lianmin Zheng's avatar
Lianmin Zheng committed
1822
        running_bs = len(self.running_batch.reqs)
1823
        # Ignore the check if self.chunked_req is not None.
1824
1825
1826
1827
        # In the non-PP case, when self.chunked_req is not None, num_allocatable_reqs should always be greater than 0,
        # as the space for the chunked request has just been released.
        # In PP case, a chunked req can start in one microbatch and end in another microbatch, so the max_running_requests per microbatch should not be strict.
        # Instead, we should always allow chunked request to be added, otherwise, there will be a memory leak.
1828
1829
1830
1831
1832
        if (
            self.get_num_allocatable_reqs(running_bs) <= 0
            and not self.chunked_req
            and not self.try_preemption
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1833
            self.running_batch.batch_is_full = True
1834
1835
            return None

1836
        if self.enable_hierarchical_cache:
1837
            self.tree_cache.check_hicache_events()
1838

1839
        # Get priority queue
1840
        self.policy.calc_priority(self.waiting_queue)
1841

Lianmin Zheng's avatar
Lianmin Zheng committed
1842
        # Prefill policy
1843
        adder = PrefillAdder(
1844
            self.page_size,
1845
            self.tree_cache,
1846
            self.token_to_kv_pool_allocator,
1847
1848
1849
1850
            self.running_batch,
            self.new_token_ratio,
            self.max_prefill_tokens,
            self.chunked_prefill_size,
1851
            running_bs if self.is_mixed_chunk else 0,
1852
            self.priority_scheduling_preemption_threshold,
1853
1854
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1855
        if self.chunked_req is not None:
1856
1857
            self.chunked_req.init_next_round_input()
            self.chunked_req = adder.add_chunked_req(self.chunked_req)
1858

1859
        if self.enable_lora:
1860
            lora_set = set([req.lora_id for req in self.running_batch.reqs])
Lianmin Zheng's avatar
Lianmin Zheng committed
1861

1862
        # Get requests from the waiting queue to a new prefill batch
1863
        for req in self.waiting_queue:
1864
1865
1866
1867
1868

            if self.enable_lora and not self.tp_worker.can_run_lora_batch(
                lora_set
                | set([req.lora_id for req in adder.can_run_list])
                | set([req.lora_id])
1869
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1870
                self.running_batch.batch_is_full = True
1871
1872
                break

1873
            running_bs = len(self.running_batch.reqs) - len(adder.preempt_list)
1874
            if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):
Lianmin Zheng's avatar
Lianmin Zheng committed
1875
                self.running_batch.batch_is_full = True
Byron Hsu's avatar
Byron Hsu committed
1876
1877
1878
1879
1880
            if self.disaggregation_mode == DisaggregationMode.PREFILL:
                # In prefill mode, prealloc queue and transfer queue can also take memory,
                # so we need to check if the available size for the actual available size.
                if len(adder.can_run_list) >= self.req_to_token_pool.available_size():
                    self.running_batch.batch_is_full = True
1881
1882
1883
1884
1885

            if self.running_batch.batch_is_full:
                if not self.try_preemption:
                    break
                if not adder.preempt_to_schedule(req, self.server_args):
Byron Hsu's avatar
Byron Hsu committed
1886
1887
                    break

1888
            if self.enable_hicache_storage:
pansicheng's avatar
pansicheng committed
1889
1890
1891
1892
                prefetch_done = self.tree_cache.check_prefetch_progress(req.rid)
                if not prefetch_done:
                    # skip staging requests that are ongoing prefetch
                    continue
1893

1894
            req.init_next_round_input(self.tree_cache)
1895
1896
1897
1898
1899
            res = adder.add_one_req(
                req,
                has_chunked_req=(self.chunked_req is not None),
                truncation_align_size=self.truncation_align_size,
            )
1900

1901
1902
            if res != AddReqResult.CONTINUE:
                if res == AddReqResult.NO_TOKEN:
1903
1904
                    if self.enable_hierarchical_cache:
                        # Set batch_is_full after making sure there are requests that can be served
Lianmin Zheng's avatar
Lianmin Zheng committed
1905
1906
                        self.running_batch.batch_is_full = len(
                            adder.can_run_list
1907
                        ) > 0 or (not self.running_batch.is_empty())
1908
                    else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1909
                        self.running_batch.batch_is_full = True
1910
1911
                break

Lianmin Zheng's avatar
Lianmin Zheng committed
1912
        # Update waiting queue
1913
        can_run_list: List[Req] = adder.can_run_list
Lianmin Zheng's avatar
Lianmin Zheng committed
1914
1915
        if len(can_run_list) == 0:
            return None
1916
1917
1918
1919

        if self.enable_metrics:
            # only record queue time when enable_metrics is True to avoid overhead
            for req in can_run_list:
1920
                req.add_latency(RequestStage.PREFILL_WAITING)
1921

Lianmin Zheng's avatar
Lianmin Zheng committed
1922
1923
1924
        self.waiting_queue = [
            x for x in self.waiting_queue if x not in set(can_run_list)
        ]
1925
        if adder.preempt_list:
1926
1927
            for req in adder.preempt_list:
                self._add_request_to_queue(req)
1928

1929
1930
1931
        if adder.new_chunked_req is not None:
            assert self.chunked_req is None
            self.chunked_req = adder.new_chunked_req
1932

1933
1934
        if self.chunked_req:
            self.chunked_req.is_chunked += 1
Lianmin Zheng's avatar
Lianmin Zheng committed
1935

1936
        # Print stats
1937
        if self.current_scheduler_metrics_enabled():
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
            self.log_prefill_stats(adder, can_run_list, running_bs, 0)

        for req in can_run_list:
            if req.time_stats.forward_entry_time == 0:
                # Avoid update chunked request many times
                req.time_stats.forward_entry_time = time.perf_counter()
                if self.enable_metrics:
                    self.metrics_collector.observe_queue_time(
                        req.time_stats.get_queueing_time(),
                    )
1948

Lianmin Zheng's avatar
Lianmin Zheng committed
1949
        # Create a new batch
1950
1951
1952
        new_batch = ScheduleBatch.init_new(
            can_run_list,
            self.req_to_token_pool,
1953
            self.token_to_kv_pool_allocator,
1954
            self.tree_cache,
1955
            self.model_config,
1956
            self.enable_overlap,
1957
            self.spec_algorithm,
1958
            chunked_req=self.chunked_req,
1959
        )
1960
1961
        if self.enable_hierarchical_cache:
            # todo (zhiqiang): disable cuda graph execution if hicache loading triggered
1962
1963
1964
            new_batch.hicache_consumer_index = (
                self.tree_cache.ready_to_load_host_cache()
            )
1965

1966
        new_batch.prepare_for_extend()
1967

Lianmin Zheng's avatar
Lianmin Zheng committed
1968
        # Mixed-style chunked prefill
1969
1970
        if (
            self.is_mixed_chunk
Lianmin Zheng's avatar
Lianmin Zheng committed
1971
            and not self.running_batch.is_empty()
1972
1973
1974
            and not (new_batch.return_logprob or self.running_batch.return_logprob)
        ):
            # TODO (lianmin): support return_logprob + mixed chunked prefill
1975
1976
            self.running_batch.filter_batch()
            if not self.running_batch.is_empty():
1977
                self.running_batch.prepare_for_decode()
1978
1979
                new_batch.mix_with_running(self.running_batch)
                new_batch.decoding_reqs = self.running_batch.reqs
Lianmin Zheng's avatar
Lianmin Zheng committed
1980
1981
1982
            self.running_batch = ScheduleBatch(
                reqs=[], batch_is_full=self.running_batch.batch_is_full
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1983
1984
        else:
            new_batch.decoding_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1985
1986
1987

        return new_batch

Lianmin Zheng's avatar
Lianmin Zheng committed
1988
    def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
1989
        """Update the current running decoding batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1990
        initial_bs = batch.batch_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1991

1992
1993
        batch.filter_batch()
        if batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1994
1995
            batch.batch_is_full = False
            return batch
1996

Lianmin Zheng's avatar
Lianmin Zheng committed
1997
        # Check if decode out of memory
1998
        if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or (
1999
            TEST_RETRACT and batch.batch_size() > 10
2000
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
2001
            old_ratio = self.new_token_ratio
2002
2003
2004
2005
            retracted_reqs, new_token_ratio, reqs_to_abort = batch.retract_decode(
                self.server_args
            )
            self.num_retracted_reqs = len(retracted_reqs)
Lianmin Zheng's avatar
Lianmin Zheng committed
2006
            self.new_token_ratio = new_token_ratio
2007
2008
            for req in reqs_to_abort:
                self.send_to_tokenizer.send_pyobj(
2009
                    AbortReq(abort_reason=req.to_abort_message, rid=req.rid)
2010
                )
2011

Lianmin Zheng's avatar
Lianmin Zheng committed
2012
            logger.info(
2013
                "KV cache pool is full. Retract requests. "
2014
2015
2016
                f"#retracted_reqs: {len(retracted_reqs)}, "
                f"#aborted_retracted_reqs: {len(reqs_to_abort)}, "
                f"#new_token_ratio: {old_ratio:.4f} -> {new_token_ratio:.4f}"
Lianmin Zheng's avatar
Lianmin Zheng committed
2017
            )
2018

2019
2020
            for req in retracted_reqs:
                self._add_request_to_queue(req, is_retracted=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
2021
2022
        else:
            self.new_token_ratio = max(
2023
                self.new_token_ratio - self.new_token_ratio_decay,
Lianmin Zheng's avatar
Lianmin Zheng committed
2024
2025
2026
                self.min_new_token_ratio,
            )

Lianmin Zheng's avatar
Lianmin Zheng committed
2027
        if batch.batch_size() < initial_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
2028
            batch.batch_is_full = False
Lianmin Zheng's avatar
Lianmin Zheng committed
2029
2030

        # Update batch tensors
2031
        batch.prepare_for_decode()
Lianmin Zheng's avatar
Lianmin Zheng committed
2032
        return batch
Lianmin Zheng's avatar
Lianmin Zheng committed
2033

2034
2035
2036
    def run_batch(
        self, batch: ScheduleBatch
    ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
2037
        """Run a batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
2038
2039
        self.forward_ct += 1

2040
2041
        # Whether to run the profiler
        self._profile_batch_predicate(batch)
2042
2043
2044
2045
        if self.forward_sleep_time is not None:
            logger.info(f"Scheduler.run_batch sleep {self.forward_sleep_time}s")
            time.sleep(self.forward_sleep_time)

2046
        # Run forward
2047
        if self.is_generation:
2048
2049
2050

            batch_or_worker_batch = batch

2051
            if self.spec_algorithm.is_none():
2052
2053
                # FIXME(lsyin): remove this if and finally unify the abstraction
                batch_or_worker_batch = batch.get_model_worker_batch()
2054

2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
            forward_batch_output = self.model_worker.forward_batch_generation(
                batch_or_worker_batch
            )

            if not self.spec_algorithm.is_none():
                # TODO(lsyin): unify this metric-updating logic with non-spec, and move it to decode processing
                self.udpate_spec_metrics(
                    batch.batch_size(), forward_batch_output.num_accepted_tokens
                )

            # update batch's output ids
            batch.output_ids = forward_batch_output.next_token_ids
2067

2068
2069
2070
            # These 2 values are needed for processing the output, but the values can be
            # modified by overlap schedule. So we have to copy them here so that
            # we can use the correct values in output processing.
2071
            if batch.return_logprob or self.spec_algorithm.is_eagle():
2072
                extend_input_len_per_req = [req.extend_input_len for req in batch.reqs]
2073
2074
            else:
                extend_input_len_per_req = None
2075

2076
            if batch.return_logprob:
2077
2078
2079
2080
2081
2082
                extend_logprob_start_len_per_req = [
                    req.extend_logprob_start_len for req in batch.reqs
                ]
            else:
                extend_logprob_start_len_per_req = None

2083
2084
            return GenerationBatchResult.from_forward_batch_output(
                forward_batch_output=forward_batch_output,
2085
2086
                extend_input_len_per_req=extend_input_len_per_req,
                extend_logprob_start_len_per_req=extend_logprob_start_len_per_req,
2087
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
2088
2089
2090
        else:  # embedding or reward model
            model_worker_batch = batch.get_model_worker_batch()
            embeddings = self.tp_worker.forward_batch_embedding(model_worker_batch)
2091
            ret = EmbeddingBatchResult(embeddings=embeddings)
2092
        return ret
Chayenne's avatar
Chayenne committed
2093

2094
2095
2096
2097
    def process_batch_result(
        self,
        batch: ScheduleBatch,
        result: Union[GenerationBatchResult, EmbeddingBatchResult],
2098
        launch_done: Optional[threading.Event] = None,
2099
    ):
Lianmin Zheng's avatar
Lianmin Zheng committed
2100
        if batch.forward_mode.is_decode():
2101
            self.process_batch_result_decode(batch, result, launch_done)
2102
2103
            if self.enable_trace:
                trace_slice_batch("decode loop", batch.reqs)
2104

2105
        elif batch.forward_mode.is_extend():
2106
            self.process_batch_result_prefill(batch, result, launch_done)
2107
2108
2109
            if self.enable_trace:
                trace_slice_batch("prefill", batch.reqs)

2110
2111
        elif batch.forward_mode.is_idle():
            if self.enable_overlap:
2112
                self.tp_worker.resolve_last_batch_result(launch_done)
2113
                self.set_next_batch_sampling_info_done(batch)
2114
        elif batch.forward_mode.is_dummy_first():
2115
            self.set_next_batch_sampling_info_done(batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
2116

2117
2118
2119
        self.maybe_send_health_check_signal()

    def maybe_send_health_check_signal(self):
2120
2121
2122
2123
2124
2125
2126
        if self.return_health_check_ct:
            # Return some signal for the health check.
            # This is used to prevent the health check signal being blocked by long context prefill.
            # However, one minor issue is that this code path does not check the status of detokenizer manager.
            self.return_health_check_ct -= 1
            self.send_to_tokenizer.send_pyobj(HealthCheckOutput())

2127
2128
    def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
        return self.prepare_mlp_sync_batch_raw(
2129
2130
2131
            local_batch,
            dp_size=self.server_args.dp_size,
            attn_tp_size=self.attn_tp_size,
2132
            tp_group=self.tp_group,
2133
2134
2135
2136
            get_idle_batch=self.get_idle_batch,
            disable_cuda_graph=self.server_args.disable_cuda_graph,
            spec_algorithm=self.spec_algorithm,
            speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens,
2137
            require_mlp_tp_gather=require_mlp_tp_gather(self.server_args),
2138
            disable_overlap_schedule=self.server_args.disable_overlap_schedule,
2139
2140
2141
        )

    @staticmethod
2142
    def prepare_mlp_sync_batch_raw(
2143
2144
2145
        local_batch: ScheduleBatch,
        dp_size,
        attn_tp_size: int,
2146
        tp_group,
2147
2148
2149
2150
        get_idle_batch,
        disable_cuda_graph: bool,
        spec_algorithm,
        speculative_num_draft_tokens,
2151
        require_mlp_tp_gather: bool,
2152
        disable_overlap_schedule: bool,
2153
    ):
2154
2155
2156
        # Check if other DP workers have running batches
        if local_batch is None:
            num_tokens = 0
2157
            num_tokens_for_logprob = 0
2158
2159
        elif local_batch.forward_mode.is_decode():
            num_tokens = local_batch.batch_size()
2160
            num_tokens_for_logprob = num_tokens
2161
2162
        else:
            num_tokens = local_batch.extend_num_tokens
2163
            num_tokens_for_logprob = sum(
Lianmin Zheng's avatar
Lianmin Zheng committed
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
                [
                    # We should have at least 1 token for sample in every case.
                    max(extend_len - logprob_start_len, 1)
                    for logprob_start_len, extend_len in zip(
                        local_batch.extend_logprob_start_lens, local_batch.extend_lens
                    )
                ]
            )

        if local_batch is None or local_batch.forward_mode.is_decode_or_idle():
            can_cuda_graph = 1
        else:
            can_cuda_graph = 0

        is_extend_in_batch = (
            local_batch.forward_mode.is_extend() if local_batch else False
        )
2181
2182

        tbo_preparer = TboDPAttentionPreparer()
2183
2184
2185
2186
2187
2188
        if disable_overlap_schedule:
            group = tp_group.device_group
            device = tp_group.device
        else:
            group = tp_group.cpu_group
            device = "cpu"
2189

Lianmin Zheng's avatar
Lianmin Zheng committed
2190
2191
2192
2193
        local_info = torch.tensor(
            [
                num_tokens,
                can_cuda_graph,
2194
                num_tokens_for_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
2195
                is_extend_in_batch,
2196
2197
2198
                *tbo_preparer.prepare_all_gather(
                    local_batch,
                ),
Lianmin Zheng's avatar
Lianmin Zheng committed
2199
2200
            ],
            dtype=torch.int64,
2201
            device=device,
Lianmin Zheng's avatar
Lianmin Zheng committed
2202
2203
        )
        global_info = torch.empty(
2204
            (dp_size, attn_tp_size, 6),
Lianmin Zheng's avatar
Lianmin Zheng committed
2205
            dtype=torch.int64,
2206
            device=device,
Lianmin Zheng's avatar
Lianmin Zheng committed
2207
        )
2208
        torch.distributed.all_gather_into_tensor(
Lianmin Zheng's avatar
Lianmin Zheng committed
2209
2210
            global_info.flatten(),
            local_info,
2211
            group=group,
2212
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
2213
2214
2215
2216
        global_num_tokens = global_info[:, 0, 0].tolist()
        can_cuda_graph = min(global_info[:, 0, 1].tolist())
        global_num_tokens_for_logprob = global_info[:, 0, 2].tolist()
        is_extend_in_batch = global_info[:, 0, 3].tolist()
2217

2218
2219
2220
2221
        tbo_split_seq_index, global_forward_mode = tbo_preparer.compute_output(
            global_info[:, :, 4:6]
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
2222
        if local_batch is None and max(global_num_tokens) > 0:
2223
            local_batch = get_idle_batch()
2224
2225

        if local_batch is not None:
2226
            # TODO: handle the case when moe_dense_tp_size != 1
2227
            if not require_mlp_tp_gather:
2228
2229
2230
2231
2232
2233
2234
                local_batch.global_num_tokens = [num_tokens]
                local_batch.global_num_tokens_for_logprob = [num_tokens_for_logprob]
            else:
                local_batch.global_num_tokens = global_num_tokens
                local_batch.global_num_tokens_for_logprob = (
                    global_num_tokens_for_logprob
                )
2235
            local_batch.is_extend_in_batch = any(is_extend_in_batch)
2236
2237
            local_batch.tbo_split_seq_index = tbo_split_seq_index
            local_batch.global_forward_mode = global_forward_mode
2238

2239
            # Check forward mode for cuda graph
2240
            if not disable_cuda_graph:
Lianmin Zheng's avatar
Lianmin Zheng committed
2241
                local_batch.can_run_dp_cuda_graph = can_cuda_graph
2242

2243
        return local_batch
2244
2245
2246
2247
2248

    def get_idle_batch(self):
        idle_batch = ScheduleBatch.init_new(
            [],
            self.req_to_token_pool,
2249
            self.token_to_kv_pool_allocator,
2250
2251
2252
            self.tree_cache,
            self.model_config,
            self.enable_overlap,
2253
            self.spec_algorithm,
2254
2255
2256
2257
        )
        idle_batch.prepare_for_idle()
        return idle_batch

2258
2259
    def move_ready_grammar_requests(self):
        """Move requests whose grammar objects are ready from grammar_queue to waiting_queue."""
2260

2261
        num_ready_reqs = 0
2262
        num_timeout_reqs = 0
2263
2264
        for req in self.grammar_queue:
            try:
2265
2266
2267
                if req.finished():  # It is aborted by AbortReq
                    num_ready_reqs += 1
                    continue
2268

2269
                req.grammar = req.grammar.result(timeout=0.03)
2270
2271
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
2272
2273
2274
                    error_msg = f"Invalid grammar request: {req.grammar_key=}"
                    req.set_finish_with_abort(error_msg)

2275
2276
                num_ready_reqs += 1
            except futures._base.TimeoutError:
2277
                req.grammar_wait_ct += 1
2278
2279
                # NOTE(lianmin): this timeout is the waiting time of the above line. It is
                # not the waiting time from it enters the grammar queue.
2280
                if req.grammar_wait_ct > GRAMMAR_TIMEOUT / 0.03:
2281
                    num_timeout_reqs = 1
2282
2283
                break

2284
        if self.server_args.enable_dp_attention:
2285
2286
            tp_size = self.attn_tp_size
            tp_group = self.attn_tp_cpu_group
2287
        else:
2288
2289
2290
2291
2292
            tp_size = self.tp_size
            tp_group = self.tp_cpu_group

        if tp_size > 1:
            # Sync across TP ranks to make sure they have the same number of ready requests
2293
            tensor = torch.tensor([num_ready_reqs, num_timeout_reqs], dtype=torch.int32)
2294
2295
2296
            torch.distributed.all_reduce(
                tensor, op=torch.distributed.ReduceOp.MAX, group=tp_group
            )
2297
            num_ready_reqs_max, num_timeout_reqs_max = tensor.tolist()
2298

2299
            for i in range(num_ready_reqs, num_ready_reqs_max):
2300
                req = self.grammar_queue[i]
2301
2302
                if req.finished():  # It is aborted by AbortReq
                    continue
2303
                req.grammar = req.grammar.result()
2304
2305
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
2306
2307
                    error_msg = f"Invalid grammar request: {req.grammar_key=}"
                    req.set_finish_with_abort(error_msg)
2308
2309
2310
        else:
            num_ready_reqs_max = num_ready_reqs
            num_timeout_reqs_max = num_timeout_reqs
2311

2312
2313
2314
        for i in range(num_ready_reqs, num_ready_reqs + num_timeout_reqs_max):
            req = self.grammar_queue[i]
            req.grammar.cancel()
2315
            self.grammar_backend.set_cache(req.grammar_key, INVALID_GRAMMAR_OBJ)
2316
2317
            error_msg = f"Grammar preprocessing timed out for {req.grammar_key=}"
            req.set_finish_with_abort(error_msg)
2318

2319
        num_ready_reqs = num_ready_reqs_max + num_timeout_reqs_max
2320

2321
2322
        for req in self.grammar_queue[:num_ready_reqs]:
            self._add_request_to_queue(req)
2323
2324
        self.grammar_queue = self.grammar_queue[num_ready_reqs:]

2325
2326
2327
    def set_next_batch_sampling_info_done(self, batch: ScheduleBatch):
        if batch.next_batch_sampling_info:
            if batch.next_batch_sampling_info.grammars is not None:
2328
2329
                batch.next_batch_sampling_info.update_regex_vocab_mask()
                self.current_stream.synchronize()
2330
2331
            batch.next_batch_sampling_info.sampling_info_done.set()

2332
2333
2334
    def watchdog_thread(self):
        """A watch dog thread that will try to kill the server itself if one forward batch takes too long."""
        self.watchdog_last_forward_ct = 0
2335
        self.watchdog_last_time = time.perf_counter()
2336
2337

        while True:
2338
            current = time.perf_counter()
2339
2340
2341
2342
2343
2344
2345
2346
2347
            if self.cur_batch is not None:
                if self.watchdog_last_forward_ct == self.forward_ct:
                    if current > self.watchdog_last_time + self.watchdog_timeout:
                        break
                else:
                    self.watchdog_last_forward_ct = self.forward_ct
                    self.watchdog_last_time = current
            time.sleep(self.watchdog_timeout // 2)

Lianmin Zheng's avatar
Lianmin Zheng committed
2348
2349
        if not disable_request_logging():
            # Print batch size and memory pool info to check whether there are de-sync issues.
Hanming Lu's avatar
Hanming Lu committed
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
            if self.is_hybrid:
                (
                    _,
                    _,
                    _,
                    _,
                    full_available_size,
                    full_evictable_size,
                    swa_available_size,
                    swa_evictable_size,
                ) = self._get_swa_token_info()
                info_msg = (
                    f"{full_available_size=}, "
                    f"{full_evictable_size=}, "
                    f"{swa_available_size=}, "
                    f"{swa_evictable_size=}, "
                )
            else:
                _, _, available_size, evictable_size = self._get_token_info()
                info_msg = f"{available_size=}, " f"{evictable_size=}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2370
2371
2372
            logger.error(
                f"{self.cur_batch.batch_size()=}, "
                f"{self.cur_batch.reqs=}, "
Hanming Lu's avatar
Hanming Lu committed
2373
                f"{info_msg}"
Lianmin Zheng's avatar
Lianmin Zheng committed
2374
2375
            )

2376
        pyspy_dump_schedulers()
Lianmin Zheng's avatar
Lianmin Zheng committed
2377
        logger.error(f"Watchdog timeout ({self.watchdog_timeout=})")
2378
2379
        print(file=sys.stderr, flush=True)
        print(file=sys.stdout, flush=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
2380
2381

        # Wait for some time so that the parent process can print the error.
2382
2383
2384
        time.sleep(5)
        self.parent_process.send_signal(signal.SIGQUIT)

2385
2386
2387
    def flush_cache_wrapped(self, recv_req: FlushCacheReqInput):
        success = self.flush_cache()
        return FlushCacheReqOutput(success=success)
2388

2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
    def clear_hicache_storage_wrapped(self, recv_req: ClearHiCacheReqInput):
        if self.enable_hierarchical_cache:
            self.tree_cache.clear_storage_backend()
            logger.info("Hierarchical cache cleared successfully!")
            if_success = True
        else:
            logging.warning("Hierarchical cache is not enabled.")
            if_success = False
        return ClearHiCacheReqOutput(success=if_success)

2399
    def flush_cache(self):
2400
        """Flush the memory pool and cache."""
2401
2402
2403
2404
2405
        if (
            len(self.waiting_queue) == 0
            and self.running_batch.is_empty()
            and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
        ):
2406
2407
            self.cur_batch = None
            self.last_batch = None
2408
            self.tree_cache.reset()
2409
            if self.grammar_backend:
Lianmin Zheng's avatar
Lianmin Zheng committed
2410
                self.grammar_backend.reset()
2411
            self.req_to_token_pool.clear()
2412
            self.token_to_kv_pool_allocator.clear()
2413

2414
2415
            if self.draft_worker:
                self.draft_worker.clear_cache_pool()
2416
2417
2418
2419
2420

            self.num_generated_tokens = 0
            self.forward_ct_decode = 0
            self.spec_num_total_accepted_tokens = 0
            self.spec_num_total_forward_ct = 0
2421
2422
            self.cum_spec_accept_length = 0
            self.cum_spec_accept_count = 0
2423
2424
2425
2426
2427
2428
2429
            torch.cuda.empty_cache()
            logger.info("Cache flushed successfully!")
            if_success = True
        else:
            logging.warning(
                f"Cache not flushed because there are pending requests. "
                f"#queue-req: {len(self.waiting_queue)}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2430
                f"#running-req: {len(self.running_batch.reqs)}"
2431
2432
2433
2434
            )
            if_success = False
        return if_success

2435
    def get_load(self, recv_req: GetLoadReqInput = None) -> GetLoadReqOutput:
Liangsheng Yin's avatar
Liangsheng Yin committed
2436
        # TODO(lsyin): use dynamically maintained num_waiting_tokens
2437

Hanming Lu's avatar
Hanming Lu committed
2438
        if self.is_hybrid:
2439
            num_tokens_full = (
Hanming Lu's avatar
Hanming Lu committed
2440
2441
2442
2443
                self.full_tokens_per_layer
                - self.token_to_kv_pool_allocator.full_available_size()
                - self.tree_cache.full_evictable_size()
            )
2444
            num_tokens_swa = (
Hanming Lu's avatar
Hanming Lu committed
2445
2446
2447
2448
                self.swa_tokens_per_layer
                - self.token_to_kv_pool_allocator.swa_available_size()
                - self.tree_cache.swa_evictable_size()
            )
2449
            num_tokens = max(num_tokens_full, num_tokens_swa)
Hanming Lu's avatar
Hanming Lu committed
2450
        else:
2451
            num_tokens = (
Hanming Lu's avatar
Hanming Lu committed
2452
2453
2454
2455
                self.max_total_num_tokens
                - self.token_to_kv_pool_allocator.available_size()
                - self.tree_cache.evictable_size()
            )
2456
2457
2458
2459

        # Tokens in waiting queue, bootstrap queue, prealloc queue
        num_tokens += sum(len(req.origin_input_ids) for req in self.waiting_queue)
        num_waiting_reqs = len(self.waiting_queue)
Liangsheng Yin's avatar
Liangsheng Yin committed
2460
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
2461
            num_tokens += sum(
Liangsheng Yin's avatar
Liangsheng Yin committed
2462
2463
2464
                len(req.origin_input_ids)
                for req in self.disagg_prefill_bootstrap_queue.queue
            )
2465
            num_waiting_reqs += len(self.disagg_prefill_bootstrap_queue.queue)
Liangsheng Yin's avatar
Liangsheng Yin committed
2466
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
2467
            num_tokens += sum(
Liangsheng Yin's avatar
Liangsheng Yin committed
2468
2469
2470
                len(req.req.origin_input_ids)
                for req in self.disagg_decode_prealloc_queue.queue
            )
2471
            num_waiting_reqs += len(self.disagg_decode_prealloc_queue.queue)
Liangsheng Yin's avatar
Liangsheng Yin committed
2472

2473
2474
2475
2476
2477
2478
        return GetLoadReqOutput(
            dp_rank=self.dp_rank,
            num_reqs=len(self.running_batch.reqs) + num_waiting_reqs,
            num_waiting_reqs=num_waiting_reqs,
            num_tokens=num_tokens,
        )
Liangsheng Yin's avatar
Liangsheng Yin committed
2479

2480
2481
2482
    def get_internal_state(self, recv_req: GetInternalStateReq):
        ret = dict(global_server_args_dict)
        ret["last_gen_throughput"] = self.last_gen_throughput
2483
2484
2485
2486
2487
2488
2489
2490
2491
        ret["memory_usage"] = {
            "weight": round(
                self.tp_worker.worker.model_runner.weight_load_mem_usage, 2
            ),
            "kvcache": round(
                self.token_to_kv_pool_allocator.get_kvcache().mem_usage, 2
            ),
            "token_capacity": int(self.max_total_num_tokens),
        }
2492

2493
2494
2495
        ret["memory_usage"]["graph"] = round(
            self.tp_worker.worker.model_runner.graph_mem_usage, 2
        )
2496

2497
2498
2499
2500
2501
2502
        if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
            ret["avg_spec_accept_length"] = (
                self.cum_spec_accept_length / self.cum_spec_accept_count
            )
        if RECORD_STEP_TIME:
            ret["step_time_dict"] = self.step_time_dict
Liangsheng Yin's avatar
Liangsheng Yin committed
2503
2504

        return GetInternalStateReqOutput(internal_state=ret)
2505
2506
2507
2508
2509

    def set_internal_state(self, recv_req: SetInternalStateReq):
        server_args_dict = recv_req.server_args
        args_allow_update = set(
            [
2510
                "max_micro_batch_size",
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
                "speculative_accept_threshold_single",
                "speculative_accept_threshold_acc",
            ]
        )
        if_success = True
        for k, v in server_args_dict.items():
            if k not in args_allow_update:
                logging.warning(f"Updating {k} is not supported.")
                if_success = False
                break
2521
2522
2523
2524
2525
2526
2527
2528
            elif k == "max_micro_batch_size" and (
                v > self.max_running_requests // self.pp_size or v < 1
            ):
                logging.warning(
                    f"Updating {k} to {v} is rejected because it is out of the valid range [1, {self.max_running_requests // self.pp_size}]."
                )
                if_success = False
                break
2529
2530
2531
2532
2533
2534
2535
2536
2537
        if if_success:
            if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
                avg_spec_accept_length = (
                    self.cum_spec_accept_length / self.cum_spec_accept_count
                )
                logger.info(f"{avg_spec_accept_length=}")
            self.cum_spec_accept_length = self.cum_spec_accept_count = 0
            for k, v in server_args_dict.items():
                global_server_args_dict[k] = v
2538
            logger.info(f"Global server args updated! {global_server_args_dict=}")
2539
2540
2541
2542
2543
        return SetInternalStateReqOutput(
            updated=True,
            server_args=global_server_args_dict,
        )

2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
    def handle_rpc_request(self, recv_req: RpcReqInput):
        # Handle RPC requests
        logger.info(
            f"handle_rpc_request: {recv_req.method}, param: {recv_req.parameters}"
        )

        success = True
        exec = None
        try:
            func = getattr(self, recv_req.method)
            func(recv_req.parameters)
        except Exception as e:
            success = False
            exec = e
            logger.error(f"Failed to call rpc {recv_req.method}: {str(e)}")

        barrier()
        return RpcReqOutput(success, "" if not exec else str(exec))

2563
2564
    def abort_request(self, recv_req: AbortReq):
        # Delete requests in the waiting queue
Lianmin Zheng's avatar
Lianmin Zheng committed
2565
        to_del = []
2566
        for i, req in enumerate(self.waiting_queue):
2567
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
Lianmin Zheng's avatar
Lianmin Zheng committed
2568
                to_del.append(i)
2569

Lianmin Zheng's avatar
Lianmin Zheng committed
2570
        # Sort in reverse order to avoid index issues when deleting
Lianmin Zheng's avatar
Lianmin Zheng committed
2571
        for i in reversed(to_del):
2572
2573
2574
            # Abort method 1: directly pop from the queue
            # This only works for requests that have not started anything.
            # We still need to send something back to TokenizerManager to clean up the state.
Lianmin Zheng's avatar
Lianmin Zheng committed
2575
            req = self.waiting_queue.pop(i)
2576
2577
2578
            if self.enable_hicache_storage:
                # to release prefetch events associated with the request
                self.tree_cache.release_aborted_request(req.rid)
2579
            self.send_to_tokenizer.send_pyobj(AbortReq(rid=req.rid))
2580
2581
2582
2583
            # For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
            if self.disaggregation_mode == DisaggregationMode.DECODE:
                self.tree_cache.cache_finished_req(req)

2584
            logger.debug(f"Abort queued request. {req.rid=}")
2585

2586
2587
2588
2589
2590
        # Delete the requests in the grammar queue
        for req in self.grammar_queue:
            # Abort method 2: call `set_finish_with_abort`
            # The request will still run one prefill forward pass.
            # In this case, we change the input_ids to be only one token to make this prefill cheap.
2591
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2592
                logger.debug(f"Abort grammar queue request. {req.rid=}")
2593
2594
                if req.grammar:
                    req.grammar.cancel()
2595
2596
                req.set_finish_with_abort("Aborted by AbortReq.")

2597
2598
2599
        # Delete requests not in the waiting queue when PD disaggregation is enabled
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            # Abort requests that have not yet been bootstrapped
2600
            for req in self.disagg_prefill_bootstrap_queue.queue:
2601
                if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2602
                    logger.debug(f"Abort bootstrap queue request. {req.rid=}")
2603
2604
2605
2606
                    if hasattr(req.disagg_kv_sender, "abort"):
                        req.disagg_kv_sender.abort()

            # Abort in-flight requests
2607
            for req in self.disagg_prefill_inflight_queue:
2608
                if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2609
                    logger.debug(f"Abort inflight queue request. {req.rid=}")
2610
2611
2612
2613
2614
                    if hasattr(req.disagg_kv_sender, "abort"):
                        req.disagg_kv_sender.abort()

        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            # Abort requests that have not yet finished preallocation
2615
            for decode_req in self.disagg_decode_prealloc_queue.queue:
2616
                if recv_req.abort_all or decode_req.req.rid.startswith(recv_req.rid):
2617
                    logger.debug(f"Abort prealloc queue request. {decode_req.req.rid=}")
2618
2619
2620
2621
                    if hasattr(decode_req.kv_receiver, "abort"):
                        decode_req.kv_receiver.abort()

            # Abort requests waiting for kvcache to release tree cache
2622
            for decode_req in self.disagg_decode_transfer_queue.queue:
2623
                if recv_req.abort_all or decode_req.req.rid.startswith(recv_req.rid):
2624
                    logger.debug(f"Abort transfer queue request. {decode_req.req.rid=}")
2625
2626
2627
                    if hasattr(decode_req.kv_receiver, "abort"):
                        decode_req.kv_receiver.abort()

2628
        # Delete requests in the running batch
Lianmin Zheng's avatar
Lianmin Zheng committed
2629
2630
2631
2632
2633
2634
        if self.cur_batch is self.running_batch or self.cur_batch is None:
            reqs = self.running_batch.reqs
        else:
            reqs = self.running_batch.reqs + self.cur_batch.reqs

        for req in reqs:
2635
2636
2637
            if not req.finished() and (
                recv_req.abort_all or req.rid.startswith(recv_req.rid)
            ):
2638
2639
2640
                # Abort method 3: set `to_abort=True`
                # The request will still run one decode forward pass.
                # Then we reuse all existing code to clean up the KV cache allocation.
Lianmin Zheng's avatar
Lianmin Zheng committed
2641
2642
                logger.debug(f"Abort running request. {req.rid=}")
                req.to_abort = True
2643

2644
2645
2646
    def _pause_engine(self) -> Tuple[List[Req], int]:
        raise NotImplementedError()

2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
    def load_lora_adapter(
        self, recv_req: LoadLoRAAdapterReqInput
    ) -> LoadLoRAAdapterReqOutput:
        """In-place loading a new lora adapter from disk or huggingface."""

        result = self.tp_worker.load_lora_adapter(recv_req)
        return result

    def unload_lora_adapter(
        self, recv_req: UnloadLoRAAdapterReqInput
    ) -> UnloadLoRAAdapterReqOutput:
        """Unload the lora adapter."""

        result = self.tp_worker.unload_lora_adapter(recv_req)
        return result

2663
2664
2665
2666
    def register_multi_tokenizer(self, recv_req: MultiTokenizerRegisterReq):
        self.send_to_detokenizer.send_pyobj(recv_req)
        return recv_req

2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
    def init_weights_send_group_for_remote_instance(
        self, recv_req: InitWeightsSendGroupForRemoteInstanceReqInput
    ):
        """Init the seed and client instance communication group."""
        success, message = self.tp_worker.init_weights_send_group_for_remote_instance(
            recv_req
        )
        return InitWeightsSendGroupForRemoteInstanceReqOutput(success, message)

    def send_weights_to_remote_instance(
        self, recv_req: SendWeightsToRemoteInstanceReqInput
    ):
        """Send the seed instance weights to the destination instance."""
        success, message = self.tp_worker.send_weights_to_remote_instance(recv_req)
        return SendWeightsToRemoteInstanceReqOutput(success, message)

2683
2684
2685
2686
2687
2688
2689
    def slow_down(self, recv_req: SlowDownReqInput):
        t = recv_req.forward_sleep_time
        if t is not None and t <= 0:
            t = None
        self.forward_sleep_time = t
        return SlowDownReqOutput()

2690
    def expert_distribution_handle(self, recv_req: ExpertDistributionReq):
2691
2692
        action = recv_req.action
        if action == ExpertDistributionReqType.START_RECORD:
2693
            get_global_expert_distribution_recorder().start_record()
2694
        elif action == ExpertDistributionReqType.STOP_RECORD:
2695
            get_global_expert_distribution_recorder().stop_record()
2696
        elif action == ExpertDistributionReqType.DUMP_RECORD:
2697
            get_global_expert_distribution_recorder().dump_record()
2698
        else:
2699
            raise ValueError(f"Unrecognized ExpertDistributionReq value: {recv_req=}")
2700
        return ExpertDistributionReqOutput()
2701

2702
    def open_session(self, recv_req: OpenSessionReqInput):
2703
2704
2705
2706
        # handle error
        session_id = recv_req.session_id
        if session_id in self.sessions:
            logger.warning(f"session id {session_id} already exist, cannot open.")
2707
            return OpenSessionReqOutput(session_id, False)
2708
        elif session_id is None:
2709
            logger.warning("session id is None, cannot open.")
2710
            return OpenSessionReqOutput(session_id, False)
2711
2712
2713
2714
        else:
            self.sessions[session_id] = Session(
                recv_req.capacity_of_str_len, session_id
            )
2715
            return OpenSessionReqOutput(session_id, True)
2716
2717
2718
2719
2720
2721
2722
2723
2724

    def close_session(self, recv_req: CloseSessionReqInput):
        # handle error
        session_id = recv_req.session_id
        if session_id not in self.sessions:
            logger.warning(f"session id {session_id} does not exist, cannot delete.")
        else:
            del self.sessions[session_id]

2725
2726
    def get_print_prefix(self):
        prefix = ""
2727
2728
        if self.attn_dp_rank is not None:
            prefix += f" DP{self.attn_dp_rank}"
2729
2730
2731
2732
2733
2734
        if self.server_args.tp_size > 1:
            prefix += f" TP{self.tp_rank}"
        if self.pp_size > 1:
            prefix += f" PP{self.pp_rank}"
        return prefix

2735
2736
    def current_scheduler_metrics_enabled(self):
        return self.attn_tp_rank == 0 or self.enable_metrics_for_all_schedulers
2737

2738
2739
2740
    def maybe_sleep_on_idle(self):
        if self.idle_sleeper is not None:
            self.idle_sleeper.maybe_sleep()
2741

2742
2743
2744
2745
2746
2747
    def handle_freeze_gc(self, recv_req: FreezeGCReq):
        """Handle freeze_gc request: freeze scheduler's GC and forward to detokenizer."""
        freeze_gc("Scheduler")
        self.send_to_detokenizer.send_pyobj(recv_req)
        return None

2748

2749
2750
2751
2752
2753
2754
2755
class IdleSleeper:
    """
    In setups which have long inactivity periods it is desirable to reduce
    system power consumption when sglang does nothing. This would lead not only
    to power savings, but also to more CPU thermal headroom when a request
    eventually comes. This is important in cases when multiple GPUs are connected
    as each GPU would otherwise pin one thread at 100% CPU usage.
2756

2757
2758
2759
    The simplest solution is to use zmq.Poller on all sockets that may receive
    data that needs handling immediately.
    """
2760

2761
2762
    def __init__(self, sockets):
        self.poller = zmq.Poller()
2763
        self.last_empty_time = time.time()
2764
2765
2766
2767
2768
        for s in sockets:
            self.poller.register(s, zmq.POLLIN)

    def maybe_sleep(self):
        self.poller.poll(1000)
2769
2770
2771
2772
2773
2774
2775
        if (
            global_config.torch_empty_cache_interval > 0
            and time.time() - self.last_empty_time
            > global_config.torch_empty_cache_interval
        ):
            self.last_empty_time = time.time()
            torch.cuda.empty_cache()
2776

2777

2778
def is_health_check_generate_req(recv_req):
2779
2780
    rid = getattr(recv_req, "rid", None)
    return rid is not None and rid.startswith("HEALTH_CHECK")
2781

2782
2783

def is_work_request(recv_req):
2784
2785
2786
2787
2788
2789
2790
2791
2792
    return isinstance(
        recv_req,
        (
            TokenizedGenerateReqInput,
            TokenizedEmbeddingReqInput,
            BatchTokenizedGenerateReqInput,
            BatchTokenizedEmbeddingReqInput,
        ),
    )
2793
2794


2795
2796
2797
2798
2799
def run_scheduler_process(
    server_args: ServerArgs,
    port_args: PortArgs,
    gpu_id: int,
    tp_rank: int,
Cheng Wan's avatar
Cheng Wan committed
2800
    moe_ep_rank: int,
2801
    pp_rank: int,
2802
    dp_rank: Optional[int],
2803
    pipe_writer,
2804
):
2805
    # Generate the logger prefix
2806
    prefix = ""
2807
2808
2809
    if dp_rank is None and "SGLANG_DP_RANK" in os.environ:
        # [For Router] if env var "SGLANG_DP_RANK" exist, set dp_rank to the value of the env var
        dp_rank = int(os.environ["SGLANG_DP_RANK"])
2810
2811
2812
2813
    if dp_rank is not None:
        prefix += f" DP{dp_rank}"
    if server_args.tp_size > 1:
        prefix += f" TP{tp_rank}"
Cheng Wan's avatar
Cheng Wan committed
2814
2815
    if server_args.ep_size > 1:
        prefix += f" EP{moe_ep_rank}"
2816
2817
    if server_args.pp_size > 1:
        prefix += f" PP{pp_rank}"
2818

2819
    # Config the process
2820
    setproctitle.setproctitle(f"sglang::scheduler{prefix.replace(' ', '_')}")
2821
    faulthandler.enable()
2822
    kill_itself_when_parent_died()
2823
    parent_process = psutil.Process().parent()
2824

Wang Ran (汪然)'s avatar
Wang Ran (汪然) committed
2825
    # Configure the logger
2826
    configure_logger(server_args, prefix=prefix)
2827
    suppress_other_loggers()
2828

2829
    # Set cpu affinity to this gpu process
2830
2831
    if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
        set_gpu_proc_affinity(server_args.tp_size, server_args.nnodes, gpu_id)
2832
2833
2834
2835
2836
2837
2838
2839
2840
    if (numa_node := server_args.numa_node) is not None:
        numa_bind_to_node(numa_node[gpu_id])

    # Set up tracing
    if server_args.enable_trace:
        process_tracing_init(server_args.oltp_traces_endpoint, "sglang")
        if server_args.disaggregation_mode == "null":
            thread_label = "Scheduler"
            trace_set_thread_info(thread_label, tp_rank, dp_rank)
2841

2842
    # Create a scheduler and run the event loop
2843
    try:
Cheng Wan's avatar
Cheng Wan committed
2844
        scheduler = Scheduler(
2845
2846
2847
2848
2849
2850
2851
            server_args,
            port_args,
            gpu_id,
            tp_rank,
            moe_ep_rank,
            pp_rank,
            dp_rank,
Cheng Wan's avatar
Cheng Wan committed
2852
        )
2853
        pipe_writer.send(
Mick's avatar
Mick committed
2854
2855
2856
2857
2858
            {
                "status": "ready",
                "max_total_num_tokens": scheduler.max_total_num_tokens,
                "max_req_input_len": scheduler.max_req_input_len,
            }
2859
        )
Byron Hsu's avatar
Byron Hsu committed
2860

2861
        disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
Byron Hsu's avatar
Byron Hsu committed
2862
        if disaggregation_mode == DisaggregationMode.NULL:
2863
2864
2865
            if server_args.pp_size > 1:
                scheduler.event_loop_pp()
            elif scheduler.enable_overlap:
Byron Hsu's avatar
Byron Hsu committed
2866
2867
2868
2869
                scheduler.event_loop_overlap()
            else:
                scheduler.event_loop_normal()
        elif disaggregation_mode == DisaggregationMode.PREFILL:
2870
2871
2872
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_prefill()
            else:
2873
2874
2875
2876
                if server_args.pp_size > 1:
                    scheduler.event_loop_pp_disagg_prefill()
                else:
                    scheduler.event_loop_normal_disagg_prefill()
2877

Byron Hsu's avatar
Byron Hsu committed
2878
        elif disaggregation_mode == DisaggregationMode.DECODE:
2879
2880
2881
2882
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_decode()
            else:
                scheduler.event_loop_normal_disagg_decode()
Byron Hsu's avatar
Byron Hsu committed
2883

2884
    except Exception:
2885
2886
2887
        traceback = get_exception_traceback()
        logger.error(f"Scheduler hit an exception: {traceback}")
        parent_process.send_signal(signal.SIGQUIT)