".github/vscode:/vscode.git/clone" did not exist on "7638f5e44ef4866f23a86b6ec1a9098189423d10"
scheduler.py 118 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
14
15
"""A scheduler that manages a tensor parallel GPU worker."""

16
import datetime
17
import faulthandler
18
import logging
19
import os
20
import signal
21
import sys
Lianmin Zheng's avatar
Lianmin Zheng committed
22
import threading
23
import time
24
from collections import defaultdict, deque
Lianmin Zheng's avatar
Lianmin Zheng committed
25
from concurrent import futures
26
from dataclasses import dataclass
27
from pathlib import Path
28
from types import SimpleNamespace
29
from typing import Dict, List, Optional, Tuple, Union
30

31
import psutil
32
import setproctitle
33
import torch
34
import zmq
35
from torch.distributed import barrier
36

37
from sglang.global_config import global_config
Lianmin Zheng's avatar
Lianmin Zheng committed
38
from sglang.srt.configs.model_config import ModelConfig
39
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE, GPU_MEMORY_TYPE_WEIGHTS
40
41
42
43
from sglang.srt.constrained.base_grammar_backend import (
    INVALID_GRAMMAR_OBJ,
    create_grammar_backend,
)
Byron Hsu's avatar
Byron Hsu committed
44
45
46
47
48
from sglang.srt.disaggregation.decode import (
    DecodePreallocQueue,
    DecodeTransferQueue,
    SchedulerDisaggregationDecodeMixin,
)
49
from sglang.srt.disaggregation.kv_events import EventPublisherFactory, KVEventBatch
Byron Hsu's avatar
Byron Hsu committed
50
51
52
53
54
55
from sglang.srt.disaggregation.prefill import (
    PrefillBootstrapQueue,
    SchedulerDisaggregationPrefillMixin,
)
from sglang.srt.disaggregation.utils import (
    DisaggregationMode,
56
    MetadataBuffers,
Byron Hsu's avatar
Byron Hsu committed
57
    ReqToMetadataIdxAllocator,
58
    TransferBackend,
59
    prepare_abort,
Byron Hsu's avatar
Byron Hsu committed
60
)
61
from sglang.srt.distributed import get_pp_group, get_world_group
fzyzcjy's avatar
fzyzcjy committed
62
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
xm:D's avatar
xm:D committed
63
64
65
66
67
from sglang.srt.hf_transformers_utils import (
    get_processor,
    get_tokenizer,
    get_tokenizer_from_processor,
)
68
from sglang.srt.layers.dp_attention import compute_dp_attention_world_info
69
70
71
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.io_struct import (
    AbortReq,
72
    CloseSessionReqInput,
73
    ExpertDistributionReq,
74
    ExpertDistributionReqOutput,
75
76
    FlushCacheReqInput,
    FlushCacheReqOutput,
77
78
    GetInternalStateReq,
    GetInternalStateReqOutput,
79
80
    GetWeightsByNameReqInput,
    GetWeightsByNameReqOutput,
81
    HealthCheckOutput,
82
83
    InitWeightsUpdateGroupReqInput,
    InitWeightsUpdateGroupReqOutput,
84
85
    LoadLoRAAdapterReqInput,
    LoadLoRAAdapterReqOutput,
86
87
    OpenSessionReqInput,
    OpenSessionReqOutput,
88
    ProfileReq,
89
90
    ProfileReqOutput,
    ProfileReqType,
91
92
93
94
    ReleaseMemoryOccupationReqInput,
    ReleaseMemoryOccupationReqOutput,
    ResumeMemoryOccupationReqInput,
    ResumeMemoryOccupationReqOutput,
95
96
    RpcReqInput,
    RpcReqOutput,
97
98
    SetInternalStateReq,
    SetInternalStateReqOutput,
99
100
    SlowDownReqInput,
    SlowDownReqOutput,
101
102
    TokenizedEmbeddingReqInput,
    TokenizedGenerateReqInput,
103
104
    UnloadLoRAAdapterReqInput,
    UnloadLoRAAdapterReqOutput,
Chayenne's avatar
Chayenne committed
105
106
    UpdateWeightFromDiskReqInput,
    UpdateWeightFromDiskReqOutput,
107
108
    UpdateWeightsFromDistributedReqInput,
    UpdateWeightsFromDistributedReqOutput,
109
110
    UpdateWeightsFromTensorReqInput,
    UpdateWeightsFromTensorReqOutput,
111
)
112
from sglang.srt.managers.mm_utils import init_embedding_cache
113
114
from sglang.srt.managers.schedule_batch import (
    FINISH_ABORT,
Mick's avatar
Mick committed
115
    MultimodalInputs,
116
117
    Req,
    ScheduleBatch,
118
    global_server_args_dict,
119
)
120
121
122
123
124
from sglang.srt.managers.schedule_policy import (
    AddReqResult,
    PrefillAdder,
    SchedulePolicy,
)
125
126
127
from sglang.srt.managers.scheduler_output_processor_mixin import (
    SchedulerOutputProcessorMixin,
)
128
from sglang.srt.managers.session_controller import Session
129
from sglang.srt.managers.tp_worker import TpModelWorker
130
from sglang.srt.managers.tp_worker_overlap_thread import TpModelWorkerClient
131
from sglang.srt.managers.utils import validate_input_length
tarinkk's avatar
tarinkk committed
132
from sglang.srt.mem_cache.chunk_cache import ChunkCache, SWAChunkCache
133
from sglang.srt.mem_cache.hiradix_cache import HiRadixCache
134
from sglang.srt.mem_cache.radix_cache import RadixCache
Hanming Lu's avatar
Hanming Lu committed
135
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
136
from sglang.srt.metrics.collector import SchedulerMetricsCollector, SchedulerStats
Lianmin Zheng's avatar
Lianmin Zheng committed
137
from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors
138
from sglang.srt.reasoning_parser import ReasoningParser
139
from sglang.srt.server_args import PortArgs, ServerArgs
140
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
141
from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter
142
from sglang.srt.two_batch_overlap import TboDPAttentionPreparer
143
from sglang.srt.utils import (
144
    DeepEPMode,
145
    DynamicGradMode,
146
    broadcast_pyobj,
fzyzcjy's avatar
fzyzcjy committed
147
    configure_gc_logger,
148
    configure_logger,
Lianmin Zheng's avatar
Lianmin Zheng committed
149
    disable_request_logging,
150
    get_available_gpu_memory,
151
    get_bool_env_var,
152
    get_zmq_socket,
153
    is_cpu,
Lianmin Zheng's avatar
Lianmin Zheng committed
154
    kill_itself_when_parent_died,
155
    point_to_point_pyobj,
156
    pyspy_dump_schedulers,
157
158
    require_mlp_sync,
    require_mlp_tp_gather,
159
    set_gpu_proc_affinity,
160
161
162
    set_random_seed,
    suppress_other_loggers,
)
163
from sglang.utils import TypeBasedDispatcher, get_exception_traceback
164
165
166

logger = logging.getLogger(__name__)

167
# Test retract decode for debugging purposes
168
169
TEST_RETRACT = get_bool_env_var("SGLANG_TEST_RETRACT")
RECORD_STEP_TIME = get_bool_env_var("SGLANG_RECORD_STEP_TIME")
170
GRAMMAR_TIMEOUT = float(os.environ.get("SGLANG_GRAMMAR_TIMEOUT", 300))
171

172
173
_is_cpu = is_cpu()

174

175
176
@dataclass
class GenerationBatchResult:
177
178
179
    logits_output: Optional[LogitsProcessorOutput]
    pp_hidden_states_proxy_tensors: Optional[torch.Tensor]
    next_token_ids: Optional[List[int]]
180
181
    extend_input_len_per_req: List[int]
    extend_logprob_start_len_per_req: List[int]
182
    bid: int
183
    can_run_cuda_graph: bool
184
185
186
187
188
189
190
191


@dataclass
class EmbeddingBatchResult:
    embeddings: torch.Tensor
    bid: int


192
193
194
195
196
197
198
199
200
201
202
203
class KvMetrics:
    def __init__(self):
        self.request_active_slots = None
        self.request_total_slots = None
        self.kv_active_blocks = None
        self.kv_total_blocks = None
        self.num_requests_waiting = None
        self.gpu_cache_usage_perc = None
        self.gpu_prefix_cache_hit_rate = None
        self.data_parallel_rank = None


204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
class IdleSleeper:
    """
    In setups which have long inactivity periods it is desirable to reduce
    system power consumption when sglang does nothing. This would lead not only
    to power savings, but also to more CPU thermal headroom when a request
    eventually comes. This is important in cases when multiple GPUs are connected
    as each GPU would otherwise pin one thread at 100% CPU usage.

    The simplest solution is to use zmq.Poller on all sockets that may receive
    data that needs handling immediately.
    """

    def __init__(self, sockets):
        self.poller = zmq.Poller()
        for s in sockets:
            self.poller.register(s, zmq.POLLIN)

    def maybe_sleep(self):
        self.poller.poll(1000)


Byron Hsu's avatar
Byron Hsu committed
225
226
227
228
229
class Scheduler(
    SchedulerOutputProcessorMixin,
    SchedulerDisaggregationDecodeMixin,
    SchedulerDisaggregationPrefillMixin,
):
230
231
232
233
234
235
236
237
    """A scheduler that manages a tensor parallel GPU worker."""

    def __init__(
        self,
        server_args: ServerArgs,
        port_args: PortArgs,
        gpu_id: int,
        tp_rank: int,
238
        pp_rank: int,
239
        dp_rank: Optional[int],
240
241
    ):
        # Parse args
242
        self.server_args = server_args
243
        self.tp_rank = tp_rank
244
        self.pp_rank = pp_rank
245
        self.dp_rank = dp_rank
246
        self.tp_size = server_args.tp_size
247
248
        self.pp_size = server_args.pp_size
        self.dp_size = server_args.dp_size
249
        self.schedule_policy = server_args.schedule_policy
250
        self.enable_lora = server_args.enable_lora
251
        self.max_loras_per_batch = server_args.max_loras_per_batch
252
        self.enable_overlap = not server_args.disable_overlap_schedule
253
        self.skip_tokenizer_init = server_args.skip_tokenizer_init
254
        self.enable_metrics = server_args.enable_metrics
255
256
257
        self.enable_metrics_for_all_schedulers = (
            server_args.enable_metrics_for_all_schedulers
        )
258
        self.enable_kv_cache_events = server_args.kv_events_config is not None
259
        self.stream_interval = server_args.stream_interval
260
261
262
        self.spec_algorithm = SpeculativeAlgorithm.from_string(
            server_args.speculative_algorithm
        )
263
264
        self.gpu_id = gpu_id
        self.enable_hierarchical_cache = server_args.enable_hierarchical_cache
265
        self.enable_hicache_storage = server_args.hicache_storage_backend is not None
Lianmin Zheng's avatar
Lianmin Zheng committed
266
        self.page_size = server_args.page_size
267
268
        self.dp_size = server_args.dp_size
        self.attn_tp_rank, self.attn_tp_size, self.attn_dp_rank = (
269
270
271
272
273
274
275
276
            compute_dp_attention_world_info(
                server_args.enable_dp_attention,
                self.tp_rank,
                self.tp_size,
                self.dp_size,
            )
        )

277
278
        # Init inter-process communication
        context = zmq.Context(2)
279
280
        self.idle_sleeper = None

281
        if self.pp_rank == 0 and self.attn_tp_rank == 0:
282
            self.recv_from_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
283
                context, zmq.PULL, port_args.scheduler_input_ipc_name, False
284
            )
285
            self.send_to_tokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
286
                context, zmq.PUSH, port_args.tokenizer_ipc_name, False
287
            )
288

289
            if server_args.skip_tokenizer_init:
290
                # Directly send to the TokenizerManager
291
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
292
                    context, zmq.PUSH, port_args.tokenizer_ipc_name, False
293
294
                )
            else:
295
                # Send to the DetokenizerManager
296
                self.send_to_detokenizer = get_zmq_socket(
Lianmin Zheng's avatar
Lianmin Zheng committed
297
                    context, zmq.PUSH, port_args.detokenizer_ipc_name, False
298
                )
299
300
301
302

            self.recv_from_rpc = get_zmq_socket(
                context, zmq.DEALER, port_args.rpc_ipc_name, False
            )
303
304
305
306
307
308
309
            if self.server_args.sleep_on_idle:
                self.idle_sleeper = IdleSleeper(
                    [
                        self.recv_from_tokenizer,
                        self.recv_from_rpc,
                    ]
                )
310
        else:
311
            self.recv_from_tokenizer = None
312
            self.recv_from_rpc = None
313
314
            self.send_to_tokenizer = SimpleNamespace(send_pyobj=lambda x: None)
            self.send_to_detokenizer = SimpleNamespace(send_pyobj=lambda x: None)
315

316
317
318
319
320
        if self.current_scheduler_metrics_enabled():
            self.send_metrics_from_scheduler = get_zmq_socket(
                context, zmq.PUSH, port_args.metrics_ipc_name, False
            )

321
        # Init tokenizer
322
        self.init_tokenizer()
323

324
325
326
327
328
329
330
331
332
        # Set reasoning_parser and think_end_id if --reasoning_parser is enabled
        if self.server_args.reasoning_parser and self.tokenizer:
            reasoning_parser = ReasoningParser(
                model_type=self.server_args.reasoning_parser, stream_reasoning=False
            )
            self.tokenizer.think_end_id = self.tokenizer.encode(
                reasoning_parser.detector.think_end_token, add_special_tokens=False
            )[0]

333
334
335
336
        # Check whether overlap can be enabled
        if not self.is_generation:
            self.enable_overlap = False
            logger.info("Overlap scheduler is disabled for embedding models.")
337

338
        # Launch a tensor parallel worker
339
        if self.enable_overlap:
340
            TpWorkerClass = TpModelWorkerClient
341
342
        else:
            TpWorkerClass = TpModelWorker
343

344
        self.tp_worker = TpWorkerClass(
345
            server_args=server_args,
346
347
            gpu_id=gpu_id,
            tp_rank=tp_rank,
348
            pp_rank=pp_rank,
349
            dp_rank=dp_rank,
350
            nccl_port=port_args.nccl_port,
351
        )
352

353
        # Launch a draft worker for speculative decoding
354
355
356
357
358
359
360
361
362
363
364
365
366
367
        if self.spec_algorithm.is_eagle():
            from sglang.srt.speculative.eagle_worker import EAGLEWorker

            self.draft_worker = EAGLEWorker(
                gpu_id=gpu_id,
                tp_rank=tp_rank,
                server_args=server_args,
                nccl_port=port_args.nccl_port,
                target_worker=self.tp_worker,
                dp_rank=dp_rank,
            )
        else:
            self.draft_worker = None

368
        # Get token and memory info from the model worker
369
370
371
372
        (
            self.max_total_num_tokens,
            self.max_prefill_tokens,
            self.max_running_requests,
373
            self.max_req_len,
374
375
            self.max_req_input_len,
            self.random_seed,
376
            self.device,
377
378
379
380
381
            worker_global_server_args_dict,
            _,
            _,
            _,
        ) = self.tp_worker.get_worker_info()
382
383
384
385
386
387
388
389
        if global_server_args_dict["max_micro_batch_size"] is None:
            global_server_args_dict["max_micro_batch_size"] = max(
                self.max_running_requests // server_args.pp_size, 1
            )

        self.tp_group = self.tp_worker.get_tp_group()
        self.tp_cpu_group = self.tp_group.cpu_group
        self.attn_tp_group = self.tp_worker.get_attention_tp_group()
390
        self.attn_tp_cpu_group = self.tp_worker.get_attention_tp_cpu_group()
391
392
393
        self.pp_group = get_pp_group()
        self.world_group = get_world_group()

394
        self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
395
        global_server_args_dict.update(worker_global_server_args_dict)
396
        set_random_seed(self.random_seed)
397

Hanming Lu's avatar
Hanming Lu committed
398
399
400
401
402
403
404
405
        # Hybrid
        self.is_hybrid = self.tp_worker.is_hybrid
        if self.is_hybrid:
            self.sliding_window_size = self.tp_worker.sliding_window_size
            self.full_tokens_per_layer, self.swa_tokens_per_layer = (
                self.tp_worker.get_tokens_per_layer_info()
            )

406
        # Print debug info
407
        if tp_rank == 0:
408
409
410
            avail_mem = get_available_gpu_memory(
                self.device, self.gpu_id, empty_cache=False
            )
411
412
413
414
415
            logger.info(
                f"max_total_num_tokens={self.max_total_num_tokens}, "
                f"chunked_prefill_size={server_args.chunked_prefill_size}, "
                f"max_prefill_tokens={self.max_prefill_tokens}, "
                f"max_running_requests={self.max_running_requests}, "
416
417
                f"context_len={self.model_config.context_len}, "
                f"available_gpu_mem={avail_mem:.2f} GB"
418
            )
419

Lianmin Zheng's avatar
Lianmin Zheng committed
420
        # Init memory pool and cache
421
        self.init_memory_pool_and_cache()
422
423
424

        # Init running status
        self.waiting_queue: List[Req] = []
425
        # The running decoding batch for continuous batching
Lianmin Zheng's avatar
Lianmin Zheng committed
426
        self.running_batch: ScheduleBatch = ScheduleBatch(reqs=[], batch_is_full=False)
427
        # The current forward batch
Lianmin Zheng's avatar
Lianmin Zheng committed
428
        self.cur_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
429
        # The last forward batch
430
        self.last_batch: Optional[ScheduleBatch] = None
Lianmin Zheng's avatar
Lianmin Zheng committed
431
432
        self.forward_ct = 0
        self.forward_ct_decode = 0
433
        self.num_generated_tokens = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
434
        self.last_prefill_tokens = 0
435
436
        self.last_decode_stats_tic = time.perf_counter()
        self.last_prefill_stats_tic = time.perf_counter()
437
        self.return_health_check_ct = 0
438
439
440
441
442
        self.num_retracted_reqs: int = 0
        self.num_paused_reqs: int = 0
        self.kv_transfer_speed_gb_s: float = 0.0
        self.kv_transfer_latency_ms: float = 0.0
        self.sessions: Dict[str, Session] = {}
443
        self.current_stream = torch.get_device_module(self.device).current_stream()
444
445
        if self.device == "cpu":
            self.current_stream.synchronize = lambda: None  # No-op for CPU
446
        self.forward_sleep_time = None
447

448
449
        # Init chunked prefill
        self.chunked_prefill_size = server_args.chunked_prefill_size
450
451
        if self.chunked_prefill_size <= 0:  # -1 means disable
            self.chunked_prefill_size = None
452
        self.chunked_req = None
453
454
455
456
        self.is_mixed_chunk = (
            self.chunked_prefill_size is not None and server_args.enable_mixed_chunk
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
457
        # Init the grammar backend for constrained generation
458
        self.grammar_queue: List[Req] = []
459
        if not server_args.skip_tokenizer_init:
460
461
462
            self.grammar_backend = create_grammar_backend(
                server_args, self.tokenizer, self.model_config.vocab_size
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
463
464
        else:
            self.grammar_backend = None
465

466
        # Init schedule policy and new token estimation
467
        self.policy = SchedulePolicy(
Lianmin Zheng's avatar
Lianmin Zheng committed
468
469
470
            self.schedule_policy,
            self.tree_cache,
            self.enable_hierarchical_cache,
471
        )
472
473
474
        assert (
            server_args.schedule_conservativeness >= 0
        ), "Invalid schedule_conservativeness"
475
476
        self.init_new_token_ratio = min(
            global_config.default_init_new_token_ratio
477
478
            * server_args.schedule_conservativeness,
            1.0,
479
        )
480
481
482
483
484
485
486
487
488
489
        self.min_new_token_ratio = min(
            self.init_new_token_ratio
            * global_config.default_min_new_token_ratio_factor,
            1.0,
        )
        self.new_token_ratio_decay = (
            self.init_new_token_ratio - self.min_new_token_ratio
        ) / global_config.default_new_token_ratio_decay_steps
        self.new_token_ratio = self.init_new_token_ratio

Lianmin Zheng's avatar
Lianmin Zheng committed
490
491
492
493
        # Init watchdog thread
        self.watchdog_timeout = server_args.watchdog_timeout
        t = threading.Thread(target=self.watchdog_thread, daemon=True)
        t.start()
494
        self.parent_process = psutil.Process().parent()
495
496

        # Init memory saver, profiler and metric stats
497
498
499
        self.memory_saver_adapter = TorchMemorySaverAdapter.create(
            enable=server_args.enable_memory_saver
        )
500
        self.init_profier()
501
502

        # Init metrics stats
503
        self.init_metrics(tp_rank, pp_rank, dp_rank)
504
        self.init_kv_events(server_args.kv_events_config)
505

506
507
        # Init request dispatcher
        self._request_dispatcher = TypeBasedDispatcher(
508
509
510
            [
                (TokenizedGenerateReqInput, self.handle_generate_request),
                (TokenizedEmbeddingReqInput, self.handle_embedding_request),
511
                (FlushCacheReqInput, self.flush_cache_wrapped),
512
                (AbortReq, self.abort_request),
513
514
                (OpenSessionReqInput, self.open_session),
                (CloseSessionReqInput, self.close_session),
515
516
517
518
519
520
521
522
                (UpdateWeightFromDiskReqInput, self.update_weights_from_disk),
                (InitWeightsUpdateGroupReqInput, self.init_weights_update_group),
                (
                    UpdateWeightsFromDistributedReqInput,
                    self.update_weights_from_distributed,
                ),
                (UpdateWeightsFromTensorReqInput, self.update_weights_from_tensor),
                (GetWeightsByNameReqInput, self.get_weights_by_name),
523
524
                (ReleaseMemoryOccupationReqInput, self.release_memory_occupation),
                (ResumeMemoryOccupationReqInput, self.resume_memory_occupation),
525
                (SlowDownReqInput, self.slow_down),
526
                (ProfileReq, self.profile),
527
                (GetInternalStateReq, self.get_internal_state),
528
                (SetInternalStateReq, self.set_internal_state),
529
                (RpcReqInput, self.handle_rpc_request),
530
                (ExpertDistributionReq, self.expert_distribution_handle),
531
532
                (LoadLoRAAdapterReqInput, self.load_lora_adapter),
                (UnloadLoRAAdapterReqInput, self.unload_lora_adapter),
533
534
535
            ]
        )

536
        # Init disaggregation
Byron Hsu's avatar
Byron Hsu committed
537
538
539
540
541
        self.disaggregation_mode = DisaggregationMode(
            self.server_args.disaggregation_mode
        )
        self.init_disaggregation()

fzyzcjy's avatar
fzyzcjy committed
542
543
544
        if get_bool_env_var("SGLANG_GC_LOG"):
            configure_gc_logger()

545
546
547
    def current_scheduler_metrics_enabled(self):
        return self.attn_tp_rank == 0 or self.enable_metrics_for_all_schedulers

548
549
550
551
    def maybe_sleep_on_idle(self):
        if self.idle_sleeper is not None:
            self.idle_sleeper.maybe_sleep()

552
553
    def init_tokenizer(self):
        server_args = self.server_args
Lianmin Zheng's avatar
Lianmin Zheng committed
554

555
        self.model_config = ModelConfig.from_server_args(server_args)
556
        self.is_generation = self.model_config.is_generation
557

558
559
560
561
562
563
564
565
566
        if server_args.skip_tokenizer_init:
            self.tokenizer = self.processor = None
        else:
            if self.model_config.is_multimodal:
                self.processor = get_processor(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
567
                    use_fast=not server_args.disable_fast_image_processor,
568
                )
xm:D's avatar
xm:D committed
569
                self.tokenizer = get_tokenizer_from_processor(self.processor)
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
            else:
                self.tokenizer = get_tokenizer(
                    server_args.tokenizer_path,
                    tokenizer_mode=server_args.tokenizer_mode,
                    trust_remote_code=server_args.trust_remote_code,
                    revision=server_args.revision,
                )

    def init_memory_pool_and_cache(self):
        server_args = self.server_args

        self.req_to_token_pool, self.token_to_kv_pool_allocator = (
            self.tp_worker.get_memory_pool()
        )

        if (
            server_args.chunked_prefill_size is not None
            and server_args.disable_radix_cache
        ):
Hanming Lu's avatar
Hanming Lu committed
589
            if self.is_hybrid:
tarinkk's avatar
tarinkk committed
590
591
592
593
                ChunkCacheClass = SWAChunkCache
            else:
                ChunkCacheClass = ChunkCache
            self.tree_cache = ChunkCacheClass(
594
595
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
596
                page_size=self.page_size,
597
598
599
600
601
602
            )
        else:
            if self.enable_hierarchical_cache:
                self.tree_cache = HiRadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
603
604
605
606
607
                    tp_cache_group=(
                        self.attn_tp_cpu_group
                        if self.server_args.enable_dp_attention
                        else self.tp_cpu_group
                    ),
608
                    page_size=self.page_size,
609
                    hicache_ratio=server_args.hicache_ratio,
Zhiqiang Xie's avatar
Zhiqiang Xie committed
610
611
                    hicache_size=server_args.hicache_size,
                    hicache_write_policy=server_args.hicache_write_policy,
612
613
614
615
616
617
                    hicache_io_backend=(
                        "direct"
                        if server_args.attention_backend
                        == "fa3"  # hot fix for incompatibility
                        else server_args.hicache_io_backend
                    ),
618
                    hicache_storage_backend=server_args.hicache_storage_backend,
619
                )
620
621
622
                self.tp_worker.register_hicache_layer_transfer_counter(
                    self.tree_cache.cache_controller.layer_done_counter
                )
Hanming Lu's avatar
Hanming Lu committed
623
624
625
626
627
628
629
630
631
632
633
            elif self.is_hybrid:
                assert (
                    self.server_args.disaggregation_mode == "null"
                ), "Hybrid mode does not support disaggregation yet"
                self.tree_cache = SWARadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
                    sliding_window_size=self.sliding_window_size,
                    page_size=self.page_size,
                    disable=server_args.disable_radix_cache,
                )
634

635
636
637
638
            else:
                self.tree_cache = RadixCache(
                    req_to_token_pool=self.req_to_token_pool,
                    token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Lianmin Zheng's avatar
Lianmin Zheng committed
639
                    page_size=self.page_size,
640
                    disable=server_args.disable_radix_cache,
641
                    enable_kv_cache_events=self.enable_kv_cache_events,
642
643
644
645
646
647
648
649
650
651
652
653
                )

        self.decode_mem_cache_buf_multiplier = (
            1
            if self.spec_algorithm.is_none()
            else (
                server_args.speculative_num_draft_tokens
                + (
                    server_args.speculative_eagle_topk
                    * server_args.speculative_num_steps
                )
            )
654
        )
655

656
657
658
659
660
    def init_profier(self):
        self.torch_profiler = None
        self.torch_profiler_output_dir: Optional[str] = None
        self.profiler_activities: Optional[List[str]] = None
        self.profile_id: Optional[str] = None
661
        self.profiler_start_forward_ct: Optional[int] = None
662
663
664
665
666
667
668
669
670
671
        self.profiler_target_forward_ct: Optional[int] = None
        self.profiler_target_prefill_ct: Optional[int] = None
        self.profiler_target_decode_ct: Optional[int] = None
        self.profiler_prefill_ct: Optional[int] = None
        self.profiler_decode_ct: Optional[int] = None
        self.profile_by_stage: bool = False
        self.profile_steps: Optional[int] = None
        self.profile_in_progress: bool = False
        self.rpd_profiler = None

672
    def init_metrics(self, tp_rank: int, pp_rank: int, dp_rank: Optional[int]):
673
        self.last_gen_throughput: float = 0.0
Lianmin Zheng's avatar
Lianmin Zheng committed
674
        self.last_input_throughput: float = 0.0
675
676
677
678
679
        self.step_time_dict = defaultdict(list)  # Dict[batch size -> step time]
        self.spec_num_total_accepted_tokens = 0
        self.spec_num_total_forward_ct = 0
        self.cum_spec_accept_length = 0
        self.cum_spec_accept_count = 0
680
        self.total_retracted_reqs = 0
681
682
683
        self.stats = SchedulerStats()
        if self.enable_metrics:
            engine_type = "unified"
684
685
686
687
688
689
690
691
692
            labels = {
                "model_name": self.server_args.served_model_name,
                "engine_type": engine_type,
                "tp_rank": tp_rank,
                "pp_rank": pp_rank,
            }
            if dp_rank is not None:
                labels["dp_rank"] = dp_rank
            self.metrics_collector = SchedulerMetricsCollector(labels=labels)
Lianmin Zheng's avatar
Lianmin Zheng committed
693

694
695
    def init_kv_events(self, kv_events_config: Optional[str]):
        if self.enable_kv_cache_events:
696
697
698
            self.kv_event_publisher = EventPublisherFactory.create(
                kv_events_config, self.attn_dp_rank
            )
699

Byron Hsu's avatar
Byron Hsu committed
700
    def init_disaggregation(self):
701
702
703
704
        self.transfer_backend = TransferBackend(
            self.server_args.disaggregation_transfer_backend
        )

Byron Hsu's avatar
Byron Hsu committed
705
706
707
708
        if (
            self.disaggregation_mode == DisaggregationMode.DECODE
        ):  # *2 for the headroom.
            buffer_size = (self.req_to_token_pool.size) * 2
Byron Hsu's avatar
Byron Hsu committed
709
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
710
711
                buffer_size
            )
712
713
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
714
715
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
716
717
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
718
719
720

            # The decode requests polling kv cache
            self.disagg_decode_transfer_queue = DecodeTransferQueue(
721
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
722
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
723
                tp_rank=self.tp_rank,
724
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
725
726
                scheduler=self,
                tree_cache=self.tree_cache,
Byron Hsu's avatar
Byron Hsu committed
727
728
729
730
731
732
            )

            # The decode requests pending for pre-allocation
            self.disagg_decode_prealloc_queue = DecodePreallocQueue(
                req_to_token_pool=self.req_to_token_pool,
                token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
Byron Hsu's avatar
Byron Hsu committed
733
734
735
736
737
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
738
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
739
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
740
741
742
                scheduler=self,
                transfer_queue=self.disagg_decode_transfer_queue,
                tree_cache=self.tree_cache,
743
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
744
745
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
746
747
                dp_size=self.server_args.dp_size,
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
748
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
749
750
                max_total_num_tokens=self.max_total_num_tokens,
                prefill_pp_size=self.server_args.disaggregation_prefill_pp,
751
                num_reserved_decode_tokens=self.server_args.num_reserved_decode_tokens,
752
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
753
            )
Liangsheng Yin's avatar
Liangsheng Yin committed
754

Byron Hsu's avatar
Byron Hsu committed
755
756
757
        elif self.disaggregation_mode == DisaggregationMode.PREFILL:
            # *2 for the headroom.
            buffer_size = self.max_running_requests * 2
Byron Hsu's avatar
Byron Hsu committed
758
            self.req_to_metadata_buffer_idx_allocator = ReqToMetadataIdxAllocator(
Byron Hsu's avatar
Byron Hsu committed
759
760
                buffer_size
            )
761
762
            self.disagg_metadata_buffers = MetadataBuffers(
                buffer_size,
763
764
                hidden_size=self.model_config.hf_text_config.hidden_size,
                dtype=self.model_config.dtype,
765
766
                custom_mem_pool=self.token_to_kv_pool_allocator.get_kvcache().maybe_get_custom_mem_pool(),
            )
Byron Hsu's avatar
Byron Hsu committed
767

Liangsheng Yin's avatar
Liangsheng Yin committed
768
            self.disagg_prefill_bootstrap_queue = PrefillBootstrapQueue(
Byron Hsu's avatar
Byron Hsu committed
769
                token_to_kv_pool=self.token_to_kv_pool_allocator.get_kvcache(),
Byron Hsu's avatar
Byron Hsu committed
770
771
772
773
774
                draft_token_to_kv_pool=(
                    None
                    if self.draft_worker is None
                    else self.draft_worker.model_runner.token_to_kv_pool
                ),
Byron Hsu's avatar
Byron Hsu committed
775
                req_to_metadata_buffer_idx_allocator=self.req_to_metadata_buffer_idx_allocator,
776
                metadata_buffers=self.disagg_metadata_buffers,
Byron Hsu's avatar
Byron Hsu committed
777
778
                tp_rank=self.tp_rank,
                tp_size=self.tp_size,
Byron Hsu's avatar
Byron Hsu committed
779
                gpu_id=self.gpu_id,
Byron Hsu's avatar
Byron Hsu committed
780
                bootstrap_port=self.server_args.disaggregation_bootstrap_port,
781
                gloo_group=self.attn_tp_cpu_group,
Byron Hsu's avatar
Byron Hsu committed
782
783
784
                max_total_num_tokens=self.max_total_num_tokens,
                decode_tp_size=self.server_args.disaggregation_decode_tp,
                decode_dp_size=self.server_args.disaggregation_decode_dp,
785
                scheduler=self,
Byron Hsu's avatar
Byron Hsu committed
786
787
788
                pp_rank=self.pp_rank,
                pp_size=self.pp_size,
                transfer_backend=self.transfer_backend,
Byron Hsu's avatar
Byron Hsu committed
789
790
            )
            # The prefill requests that are in the middle of kv sending
791
            self.disagg_prefill_inflight_queue: List[Req] = []
Byron Hsu's avatar
Byron Hsu committed
792

793
    @DynamicGradMode()
794
    def event_loop_normal(self):
795
        """A normal scheduler loop."""
796
        while True:
Lianmin Zheng's avatar
Lianmin Zheng committed
797
798
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)
799

800
            batch = self.get_next_batch_to_run()
Lianmin Zheng's avatar
Lianmin Zheng committed
801
            self.cur_batch = batch
802
803
804
805

            if batch:
                result = self.run_batch(batch)
                self.process_batch_result(batch, result)
Lianmin Zheng's avatar
Lianmin Zheng committed
806
            else:
Lianmin Zheng's avatar
Lianmin Zheng committed
807
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
808
                self.check_memory()
Hanming Lu's avatar
Hanming Lu committed
809
                self.check_tree_cache()
810
                self.new_token_ratio = self.init_new_token_ratio
811
                self.maybe_sleep_on_idle()
812
813

            self.last_batch = batch
814

815
    @DynamicGradMode()
Lianmin Zheng's avatar
Lianmin Zheng committed
816
    def event_loop_overlap(self):
817
        """A scheduler loop that overlaps the CPU processing and GPU computation."""
818
        self.result_queue = deque()
Lianmin Zheng's avatar
Lianmin Zheng committed
819
820
821
822
823
824
825

        while True:
            recv_reqs = self.recv_requests()
            self.process_input_requests(recv_reqs)

            batch = self.get_next_batch_to_run()
            self.cur_batch = batch
826

Lianmin Zheng's avatar
Lianmin Zheng committed
827
            if batch:
828
                batch.launch_done = threading.Event()
Lianmin Zheng's avatar
Lianmin Zheng committed
829
                result = self.run_batch(batch)
830
                self.result_queue.append((batch.copy(), result))
Lianmin Zheng's avatar
Lianmin Zheng committed
831

832
                if self.last_batch is None:
833
                    # Create a dummy first batch to start the pipeline for overlap schedule.
834
835
836
837
838
839
                    # It is now used for triggering the sampling_info_done event.
                    tmp_batch = ScheduleBatch(
                        reqs=None,
                        forward_mode=ForwardMode.DUMMY_FIRST,
                        next_batch_sampling_info=self.tp_worker.cur_sampling_info,
                    )
840
                    self.process_batch_result(tmp_batch, None, batch.launch_done)
841

Lianmin Zheng's avatar
Lianmin Zheng committed
842
            if self.last_batch:
843
                # Process the results of the last batch
844
                tmp_batch, tmp_result = self.result_queue.popleft()
845
846
847
                tmp_batch.next_batch_sampling_info = (
                    self.tp_worker.cur_sampling_info if batch else None
                )
848
849
850
851
                # NOTE: we should use current launched batch's launch_done event Instead of the last batch's
                self.process_batch_result(
                    tmp_batch, tmp_result, batch.launch_done if batch else None
                )
Lianmin Zheng's avatar
Lianmin Zheng committed
852
            elif batch is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
853
                # When the server is idle, do self-check and re-init some states
Lianmin Zheng's avatar
Lianmin Zheng committed
854
                self.check_memory()
Hanming Lu's avatar
Hanming Lu committed
855
                self.check_tree_cache()
856
                self.new_token_ratio = self.init_new_token_ratio
857
                self.maybe_sleep_on_idle()
Lianmin Zheng's avatar
Lianmin Zheng committed
858
859
860

            self.last_batch = batch

861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
    @DynamicGradMode()
    def event_loop_pp(self):
        """A non-overlap scheduler loop for pipeline parallelism."""
        mbs = [None] * self.pp_size
        last_mbs = [None] * self.pp_size
        self.running_mbs = [
            ScheduleBatch(reqs=[], batch_is_full=False) for _ in range(self.pp_size)
        ]
        bids = [None] * self.pp_size
        pp_outputs: Optional[PPProxyTensors] = None
        while True:
            server_is_idle = True
            for mb_id in range(self.pp_size):
                self.running_batch = self.running_mbs[mb_id]
                self.last_batch = last_mbs[mb_id]

                recv_reqs = self.recv_requests()
                self.process_input_requests(recv_reqs)
                mbs[mb_id] = self.get_next_batch_to_run()
                self.running_mbs[mb_id] = self.running_batch

                self.cur_batch = mbs[mb_id]
                if self.cur_batch:
                    server_is_idle = False
                    result = self.run_batch(self.cur_batch)

887
                # (last rank) send the outputs to the next step
888
889
890
891
892
893
                if self.pp_group.is_last_rank:
                    if self.cur_batch:
                        next_token_ids, bids[mb_id] = (
                            result.next_token_ids,
                            result.bid,
                        )
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
                        if self.cur_batch.return_logprob:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                    "extend_input_len_per_req": result.extend_input_len_per_req,
                                    "extend_logprob_start_len_per_req": result.extend_logprob_start_len_per_req,
                                }
                                | (
                                    {
                                        f"logits_output.{k}": v
                                        for k, v in result.logits_output.__dict__.items()
                                    }
                                    if result.logits_output is not None
                                    else {}
                                )
                            )
                        else:
                            pp_outputs = PPProxyTensors(
                                {
                                    "next_token_ids": next_token_ids,
                                }
                            )
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
                        # send the output from the last round to let the next stage worker run post processing
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                # receive outputs and post-process (filter finished reqs) the coming microbatch
                next_mb_id = (mb_id + 1) % self.pp_size
                next_pp_outputs = None
                if mbs[next_mb_id] is not None:
                    next_pp_outputs: Optional[PPProxyTensors] = PPProxyTensors(
                        self.pp_group.recv_tensor_dict(
                            all_gather_group=self.attn_tp_group
                        )
                    )
                    mbs[next_mb_id].output_ids = next_pp_outputs["next_token_ids"]
932
933
934
935
936
937
938
939
940
                    logits_output_args = {
                        k[len("logits_output.") :]: v
                        for k, v in next_pp_outputs.tensors.items()
                        if k.startswith("logits_output.")
                    }
                    if len(logits_output_args) > 0:
                        logits_output = LogitsProcessorOutput(**logits_output_args)
                    else:
                        logits_output = None
941
                    output_result = GenerationBatchResult(
942
                        logits_output=logits_output,
943
944
                        pp_hidden_states_proxy_tensors=None,
                        next_token_ids=next_pp_outputs["next_token_ids"],
945
946
947
948
949
950
                        extend_input_len_per_req=next_pp_outputs.tensors.get(
                            "extend_input_len_per_req", None
                        ),
                        extend_logprob_start_len_per_req=next_pp_outputs.tensors.get(
                            "extend_logprob_start_len_per_req", None
                        ),
951
                        bid=bids[next_mb_id],
952
                        can_run_cuda_graph=result.can_run_cuda_graph,
953
954
955
956
                    )
                    self.process_batch_result(mbs[next_mb_id], output_result)
                    last_mbs[next_mb_id] = mbs[next_mb_id]

957
                # (not last rank)
958
959
960
                if not self.pp_group.is_last_rank:
                    if self.cur_batch:
                        bids[mb_id] = result.bid
961
962
                    # carry the outputs to the next stage
                    # send the outputs from the last round to let the next stage worker run post processing
963
964
965
966
967
968
969
                    if pp_outputs:
                        self.pp_group.send_tensor_dict(
                            pp_outputs.tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                    # send out reqs to the next stage
970
                    dp_offset = self.attn_dp_rank * self.attn_tp_size
971
972
973
974
                    if self.attn_tp_rank == 0:
                        point_to_point_pyobj(
                            recv_reqs,
                            self.pp_rank * self.tp_size + dp_offset,
975
                            self.world_group.device_group,
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
                            self.pp_rank * self.tp_size + dp_offset,
                            (self.pp_rank + 1) * self.tp_size + dp_offset,
                        )

                    # send out proxy tensors to the next stage
                    if self.cur_batch:
                        self.pp_group.send_tensor_dict(
                            result.pp_hidden_states_proxy_tensors,
                            all_gather_group=self.attn_tp_group,
                        )

                pp_outputs = next_pp_outputs

            # When the server is idle, self-check and re-init some states
            if server_is_idle:
                self.check_memory()
Hanming Lu's avatar
Hanming Lu committed
992
                self.check_tree_cache()
993
                self.new_token_ratio = self.init_new_token_ratio
994
                self.maybe_sleep_on_idle()
995

996
997
    def recv_requests(self) -> List[Req]:
        """Receive results at tp_rank = 0 and broadcast it to all other TP ranks."""
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
        if self.pp_rank == 0:
            if self.attn_tp_rank == 0:
                recv_reqs = []

                while True:
                    try:
                        recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_req)

                while True:
                    try:
                        recv_rpc = self.recv_from_rpc.recv_pyobj(zmq.NOBLOCK)
                    except zmq.ZMQError:
                        break
                    recv_reqs.append(recv_rpc)
            else:
                recv_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1017
        else:
1018
            if self.attn_tp_rank == 0:
1019
                dp_offset = self.attn_dp_rank * self.attn_tp_size
1020
1021
1022
                recv_reqs = point_to_point_pyobj(
                    [],
                    self.pp_rank * self.tp_size + dp_offset,
1023
                    self.world_group.device_group,
1024
1025
1026
1027
1028
                    (self.pp_rank - 1) * self.tp_size + dp_offset,
                    self.pp_rank * self.tp_size + dp_offset,
                )
            else:
                recv_reqs = None
1029

1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
        if self.server_args.enable_dp_attention:
            if self.attn_tp_rank == 0:
                work_reqs = [
                    req
                    for req in recv_reqs
                    if isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
                control_reqs = [
                    req
                    for req in recv_reqs
                    if not isinstance(
                        req, (TokenizedGenerateReqInput, TokenizedEmbeddingReqInput)
                    )
                ]
            else:
                work_reqs = None
                control_reqs = None

            if self.attn_tp_size != 1:
                work_reqs = broadcast_pyobj(
                    work_reqs,
1053
                    self.attn_tp_group.rank,
1054
                    self.attn_tp_cpu_group,
1055
                    src=self.attn_tp_group.ranks[0],
1056
1057
1058
                )
            if self.tp_size != 1:
                control_reqs = broadcast_pyobj(
1059
1060
1061
1062
                    control_reqs,
                    self.tp_group.rank,
                    self.tp_cpu_group,
                    src=self.tp_group.ranks[0],
1063
1064
1065
                )
            recv_reqs = work_reqs + control_reqs
        elif self.tp_size != 1:
1066
1067
1068
1069
1070
1071
            recv_reqs = broadcast_pyobj(
                recv_reqs,
                self.tp_group.rank,
                self.tp_cpu_group,
                src=self.tp_group.ranks[0],
            )
1072
1073
        return recv_reqs

Lianmin Zheng's avatar
Lianmin Zheng committed
1074
    def process_input_requests(self, recv_reqs: List):
1075
        for recv_req in recv_reqs:
1076
1077
            # If it is a health check generation request and there are running requests, ignore it.
            if is_health_check_generate_req(recv_req) and (
Lianmin Zheng's avatar
Lianmin Zheng committed
1078
                self.chunked_req is not None or not self.running_batch.is_empty()
1079
1080
1081
1082
            ):
                self.return_health_check_ct += 1
                continue

1083
            output = self._request_dispatcher(recv_req)
1084
            if output is not None:
1085
1086
1087
1088
1089
                if isinstance(output, RpcReqOutput):
                    if self.recv_from_rpc is not None:
                        self.recv_from_rpc.send_pyobj(output)
                else:
                    self.send_to_tokenizer.send_pyobj(output)
1090
1091
1092
1093
1094

    def handle_generate_request(
        self,
        recv_req: TokenizedGenerateReqInput,
    ):
1095
        # Create a new request
1096
1097
1098
1099
1100
        if (
            recv_req.session_params is None
            or recv_req.session_params.id is None
            or recv_req.session_params.id not in self.sessions
        ):
Rin Intachuen's avatar
Rin Intachuen committed
1101
1102
1103
1104
1105
1106
            if recv_req.input_embeds is not None:
                # Generate fake input_ids based on the length of input_embeds
                seq_length = len(recv_req.input_embeds)
                fake_input_ids = [1] * seq_length
                recv_req.input_ids = fake_input_ids

1107
1108
1109
1110
            if recv_req.bootstrap_port is None:
                # Use default bootstrap port
                recv_req.bootstrap_port = self.server_args.disaggregation_bootstrap_port

1111
1112
1113
1114
1115
            req = Req(
                recv_req.rid,
                recv_req.input_text,
                recv_req.input_ids,
                recv_req.sampling_params,
Lianmin Zheng's avatar
Lianmin Zheng committed
1116
1117
                return_logprob=recv_req.return_logprob,
                top_logprobs_num=recv_req.top_logprobs_num,
1118
                token_ids_logprob=recv_req.token_ids_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
1119
                stream=recv_req.stream,
1120
                lora_path=recv_req.lora_path,
Rin Intachuen's avatar
Rin Intachuen committed
1121
                input_embeds=recv_req.input_embeds,
Lianmin Zheng's avatar
Lianmin Zheng committed
1122
                custom_logit_processor=recv_req.custom_logit_processor,
1123
                return_hidden_states=recv_req.return_hidden_states,
1124
                eos_token_ids=self.model_config.hf_eos_token_id,
1125
                bootstrap_host=recv_req.bootstrap_host,
1126
                bootstrap_port=recv_req.bootstrap_port,
1127
                bootstrap_room=recv_req.bootstrap_room,
1128
                data_parallel_rank=recv_req.data_parallel_rank,
1129
1130
            )
            req.tokenizer = self.tokenizer
Lianmin Zheng's avatar
Lianmin Zheng committed
1131

1132
1133
1134
            if self.disaggregation_mode != DisaggregationMode.NULL:
                # Invalid request for disaggregated mode
                if recv_req.bootstrap_room is None:
1135
                    error_msg = (
1136
1137
1138
                        f"Invalid request: Disaggregated request received without "
                        f"boostrap room id. {req.rid=}"
                    )
1139
1140
                    logger.error(error_msg)
                    prepare_abort(req, error_msg)
1141
1142
1143
                    self.stream_output([req], req.return_logprob)
                    return

1144
1145
1146
1147
            if (
                recv_req.session_params is not None
                and recv_req.session_params.id is not None
            ):
1148
                req.set_finish_with_abort(
1149
                    f"Invalid request: session id {recv_req.session_params.id} does not exist"
1150
                )
1151
                self._add_request_to_queue(req)
1152
1153
                return
        else:
1154
1155
            # Create a new request from a previous session
            session = self.sessions[recv_req.session_params.id]
1156
            req = session.create_req(recv_req, self.tokenizer)
1157
            if isinstance(req.finished_reason, FINISH_ABORT):
1158
                self._add_request_to_queue(req)
1159
                return
1160

1161
        # Handle multimodal inputs
Mick's avatar
Mick committed
1162
1163
        if recv_req.mm_inputs is not None:
            image_inputs = MultimodalInputs.from_dict(recv_req.mm_inputs)
1164
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
1165
            req.origin_input_ids = self.pad_input_ids_func(
1166
                req.origin_input_ids, image_inputs
1167
            )
1168
            req.extend_image_inputs(image_inputs)
1169

1170
            if len(req.origin_input_ids) >= self.max_req_input_len:
1171
1172
1173
1174
1175
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1176
                )
1177
                self._add_request_to_queue(req)
1178
1179
                return

1180
        # Validate prompt length
1181
1182
1183
1184
1185
1186
        error_msg = validate_input_length(
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
        if error_msg:
1187
            req.set_finish_with_abort(error_msg)
1188
            self._add_request_to_queue(req)
1189
            return
1190

1191
        # Copy more attributes
1192
        if recv_req.logprob_start_len == -1 or not recv_req.return_logprob:
1193
1194
1195
1196
1197
            # By default, only return the logprobs for output tokens
            req.logprob_start_len = len(req.origin_input_ids) - 1
        else:
            req.logprob_start_len = recv_req.logprob_start_len

1198
        if req.logprob_start_len >= len(req.origin_input_ids):
1199
            error_msg = f"{req.logprob_start_len=} is higher than the number of input tokens {len(req.origin_input_ids)=}. Please use a smaller logprob_start_len."
1200
            req.logprob_start_len = len(req.origin_input_ids) - 1
1201
            req.set_finish_with_abort(error_msg)
1202
1203
1204
            self._add_request_to_queue(req)
            return

1205
1206
1207
1208
1209
1210
        req.sampling_params.max_new_tokens = min(
            (
                req.sampling_params.max_new_tokens
                if req.sampling_params.max_new_tokens is not None
                else 1 << 30
            ),
1211
            self.max_req_len - len(req.origin_input_ids) - 1,
1212
1213
        )

1214
1215
1216
1217
1218
        # Init grammar cache for this request
        add_to_grammar_queue = False
        if (
            req.sampling_params.json_schema is not None
            or req.sampling_params.regex is not None
1219
            or req.sampling_params.ebnf is not None
1220
            or req.sampling_params.structural_tag is not None
1221
1222
1223
1224
1225
1226
        ):
            assert self.grammar_backend is not None
            if req.sampling_params.json_schema is not None:
                key = ("json", req.sampling_params.json_schema)
            elif req.sampling_params.regex is not None:
                key = ("regex", req.sampling_params.regex)
1227
1228
            elif req.sampling_params.ebnf is not None:
                key = ("ebnf", req.sampling_params.ebnf)
1229
1230
            elif req.sampling_params.structural_tag:
                key = ("structural_tag", req.sampling_params.structural_tag)
1231

1232
1233
1234
1235
1236
            value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
            req.grammar = value

            if not cache_hit:
                req.grammar_key = key
1237
                add_to_grammar_queue = True
1238
1239
1240
1241
            else:
                if value is INVALID_GRAMMAR_OBJ:  # We hit a cached invalid grammar.
                    error_msg = f"Invalid grammar request with cache hit: {key=}"
                    req.set_finish_with_abort(error_msg)
1242
1243

        if add_to_grammar_queue:
1244
            req.queue_time_start = time.perf_counter()
1245
1246
            self.grammar_queue.append(req)
        else:
1247
1248
1249
            self._add_request_to_queue(req)

    def _add_request_to_queue(self, req: Req):
1250
        req.queue_time_start = time.perf_counter()
Byron Hsu's avatar
Byron Hsu committed
1251
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1252
1253
1254
            self.disagg_prefill_bootstrap_queue.add(
                req, self.model_config.num_key_value_heads
            )
Byron Hsu's avatar
Byron Hsu committed
1255
1256
1257
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            self.disagg_decode_prealloc_queue.add(req)
        else:
1258
1259
1260
1261
1262
1263
1264
1265
1266
            if self.enable_hicache_storage:
                req.init_next_round_input(self.tree_cache)
                last_hash = req.last_host_node.get_last_hash_value()
                matched_len = len(req.prefix_indices) + req.host_hit_length
                if (matched_len > 0 and last_hash is not None) or matched_len == 0:
                    new_input_tokens = req.fill_ids[matched_len:]
                    self.tree_cache.prefetch_from_storage(
                        req.rid, req.last_host_node, new_input_tokens, last_hash
                    )
Byron Hsu's avatar
Byron Hsu committed
1267
1268
            self.waiting_queue.append(req)

1269
    def _extend_requests_to_queue(self, reqs: List[Req], is_retracted: bool = False):
1270
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
Byron Hsu's avatar
Byron Hsu committed
1271
1272
1273
            self.disagg_prefill_bootstrap_queue.extend(
                reqs, self.model_config.num_key_value_heads
            )
1274
1275
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            # If this is a decode server, we put the request to the decode pending prealloc queue
1276
            self.disagg_decode_prealloc_queue.extend(reqs, is_retracted)
Byron Hsu's avatar
Byron Hsu committed
1277
1278
        else:
            self.waiting_queue.extend(reqs)
1279
1280
1281

    def handle_embedding_request(
        self,
1282
        recv_req: TokenizedEmbeddingReqInput,
1283
1284
1285
1286
1287
1288
    ):
        req = Req(
            recv_req.rid,
            recv_req.input_text,
            recv_req.input_ids,
            recv_req.sampling_params,
woodx's avatar
woodx committed
1289
            token_type_ids=recv_req.token_type_ids,
1290
1291
1292
        )
        req.tokenizer = self.tokenizer

1293
1294
        # Handle multimodal inputs
        if recv_req.image_inputs is not None:
Mick's avatar
Mick committed
1295
            image_inputs = MultimodalInputs.from_dict(recv_req.image_inputs)
1296
1297
1298
1299
1300
1301
1302
            # Expand a single image token into multiple dummy tokens for receiving image embeddings
            req.origin_input_ids = self.pad_input_ids_func(
                req.origin_input_ids, image_inputs
            )
            req.extend_image_inputs(image_inputs)

            if len(req.origin_input_ids) >= self.max_req_input_len:
1303
1304
1305
1306
1307
                req.set_finish_with_abort(
                    error_msg=(
                        "Multimodal prompt is too long after expanding multimodal tokens. "
                        f"After expanding {len(req.origin_input_ids_unpadded)=} => {len(req.origin_input_ids)} >= {self.max_req_input_len}."
                    )
1308
                )
1309
                self._add_request_to_queue(req)
1310
1311
                return

1312
        # Validate prompts length
1313
        error_msg = validate_input_length(
1314
1315
1316
1317
            req,
            self.max_req_input_len,
            self.server_args.allow_auto_truncate,
        )
1318
        if error_msg:
1319
            self._add_request_to_queue(req)
1320
            return
1321

1322
1323
        # Copy more attributes
        req.logprob_start_len = len(req.origin_input_ids) - 1
1324
        self._add_request_to_queue(req)
1325

1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
    def _emit_kv_metrics(self):
        kv_metrics = KvMetrics()
        kv_metrics.request_active_slots = self.stats.num_running_reqs
        kv_metrics.request_total_slots = self.max_running_requests
        kv_metrics.kv_active_blocks = int(
            self.stats.token_usage * self.max_total_num_tokens
        )
        kv_metrics.kv_total_blocks = self.max_total_num_tokens
        kv_metrics.num_requests_waiting = self.stats.num_queue_reqs
        kv_metrics.gpu_cache_usage_perc = self.stats.token_usage
        kv_metrics.gpu_prefix_cache_hit_rate = self.stats.cache_hit_rate
        kv_metrics.data_parallel_rank = self.dp_rank if self.dp_rank is not None else 0

        if not self.send_metrics_from_scheduler.closed:
            self.send_metrics_from_scheduler.send_pyobj(kv_metrics)

1342
1343
1344
1345
    def log_prefill_stats(
        self,
        adder: PrefillAdder,
        can_run_list: List[Req],
1346
        running_bs: int,
1347
    ):
1348
1349
        gap_latency = time.perf_counter() - self.last_prefill_stats_tic
        self.last_prefill_stats_tic = time.perf_counter()
Liangsheng Yin's avatar
Liangsheng Yin committed
1350
1351
        self.last_input_throughput = self.last_prefill_tokens / gap_latency
        self.last_prefill_tokens = adder.log_input_tokens
Lianmin Zheng's avatar
Lianmin Zheng committed
1352

Hanming Lu's avatar
Hanming Lu committed
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
        if self.is_hybrid:
            (
                full_num_used,
                swa_num_used,
                full_token_usage,
                swa_token_usage,
                _,
                _,
                _,
                _,
            ) = self._get_swa_token_info()
            num_used = max(full_num_used, swa_num_used)
            token_usage = max(full_token_usage, swa_token_usage)
            token_msg = (
                f"full token usage: {full_token_usage:.2f}, "
                f"swa token usage: {swa_token_usage:.2f}, "
            )
        else:
            num_used, token_usage, _, _ = self._get_token_info()
            token_msg = f"token usage: {token_usage:.2f}, "
1373

1374
        num_new_seq = len(can_run_list)
1375
        f = (
1376
            f"Prefill batch. "
1377
            f"#new-seq: {num_new_seq}, "
1378
1379
            f"#new-token: {adder.log_input_tokens}, "
            f"#cached-token: {adder.log_hit_tokens}, "
Hanming Lu's avatar
Hanming Lu committed
1380
            f"{token_msg}"
1381
        )
Liangsheng Yin's avatar
Liangsheng Yin committed
1382
1383
1384
1385

        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            f += f"#unbootstrapped-req: {len(self.disagg_prefill_bootstrap_queue.queue)}, "
            f += f"#queue-req: {len(self.waiting_queue)}, "
fzyzcjy's avatar
fzyzcjy committed
1386
            f += f"#transferring-req: {len(self.disagg_prefill_inflight_queue)}, "
1387
            f += f"input throughput (token/s): {self.last_input_throughput:.2f}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1388
        else:
Liangsheng Yin's avatar
Liangsheng Yin committed
1389
            f += f"#running-req: {running_bs}, "
1390
1391
            f += f"#queue-req: {len(self.waiting_queue)}, "

1392
        logger.info(f)
1393
1394

        if self.enable_metrics:
1395
1396
1397
            cache_hit_rate = adder.log_hit_tokens / (
                adder.log_input_tokens + adder.log_hit_tokens
            )
1398
1399
            self.stats.num_running_reqs = running_bs
            self.stats.num_used_tokens = num_used
Hanming Lu's avatar
Hanming Lu committed
1400
            self.stats.token_usage = round(token_usage, 2)
1401
1402
            self.stats.num_queue_reqs = len(self.waiting_queue)
            self.stats.cache_hit_rate = cache_hit_rate
1403
1404
1405
1406
1407
1408

            total_queue_latency = 0
            for req in can_run_list:
                total_queue_latency += req.queue_time_end - req.queue_time_start
            self.stats.avg_request_queue_latency = total_queue_latency / num_new_seq

1409
            self.metrics_collector.log_stats(self.stats)
1410
            self._emit_kv_metrics()
1411
        self._publish_kv_events()
1412

1413
1414
1415
    def log_decode_stats(
        self, can_run_cuda_graph: bool, running_batch: ScheduleBatch = None
    ):
1416
1417
        batch = running_batch or self.running_batch

1418
1419
        gap_latency = time.perf_counter() - self.last_decode_stats_tic
        self.last_decode_stats_tic = time.perf_counter()
1420
1421
        self.last_gen_throughput = self.num_generated_tokens / gap_latency
        self.num_generated_tokens = 0
1422
        num_running_reqs = len(batch.reqs)
Hanming Lu's avatar
Hanming Lu committed
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
        if self.is_hybrid:
            (
                full_num_used,
                swa_num_used,
                full_token_usage,
                swa_token_usage,
                _,
                _,
                _,
                _,
            ) = self._get_swa_token_info()
            num_used = max(full_num_used, swa_num_used)
            token_usage = max(full_token_usage, swa_token_usage)
            token_msg = (
                f"#full token: {full_num_used}, "
                f"full token usage: {full_token_usage:.2f}, "
                f"#swa token: {swa_num_used}, "
                f"swa token usage: {swa_token_usage:.2f}, "
            )
        else:
            num_used, token_usage, _, _ = self._get_token_info()
            token_msg = f"#token: {num_used}, " f"token usage: {token_usage:.2f}, "
1445
1446
1447
1448
1449

        if RECORD_STEP_TIME:
            self.step_time_dict[num_running_reqs].append(
                gap_latency / self.server_args.decode_log_interval
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1450

Hanming Lu's avatar
Hanming Lu committed
1451
        msg = f"Decode batch. #running-req: {num_running_reqs}, {token_msg}"
Liangsheng Yin's avatar
Liangsheng Yin committed
1452

1453
        if self.spec_algorithm.is_none():
1454
            spec_accept_length = 0
1455
        else:
1456
            spec_accept_length = (
1457
1458
                self.spec_num_total_accepted_tokens / self.spec_num_total_forward_ct
            )
1459
1460
            self.cum_spec_accept_length += self.spec_num_total_accepted_tokens
            self.cum_spec_accept_count += self.spec_num_total_forward_ct
1461
            self.spec_num_total_accepted_tokens = self.spec_num_total_forward_ct = 0
Liangsheng Yin's avatar
Liangsheng Yin committed
1462
1463
1464
            msg += f"accept len: {spec_accept_length:.2f}, "

        if self.disaggregation_mode == DisaggregationMode.DECODE:
1465
            msg += f"pre-allocated usage: {self.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.max_total_num_tokens:.2f}, "
1466
            msg += f"#retracted-req: {len(self.disagg_decode_prealloc_queue.retracted_queue)}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1467
1468

        msg += (
1469
            f"cuda graph: {can_run_cuda_graph}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1470
            f"gen throughput (token/s): {self.last_gen_throughput:.2f}, "
1471
            f"#queue-req: {len(self.waiting_queue)}, "
Liangsheng Yin's avatar
Liangsheng Yin committed
1472
        )
1473
1474

        logger.info(msg)
1475
1476
1477
        if self.enable_metrics:
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
Hanming Lu's avatar
Hanming Lu committed
1478
            self.stats.token_usage = round(token_usage, 2)
1479
1480
            self.stats.cache_hit_rate = 0.0
            self.stats.gen_throughput = self.last_gen_throughput
1481
            self.stats.num_queue_reqs = len(self.waiting_queue)
1482
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1483
            self.stats.spec_accept_length = spec_accept_length
1484
            self.stats.total_retracted_reqs = self.total_retracted_reqs
1485
            self.metrics_collector.log_stats(self.stats)
1486
            self._emit_kv_metrics()
1487
        self._publish_kv_events()
1488

Lianmin Zheng's avatar
Lianmin Zheng committed
1489
    def check_memory(self):
Hanming Lu's avatar
Hanming Lu committed
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
        if self.is_hybrid:
            (
                full_num_used,
                swa_num_used,
                _,
                _,
                full_available_size,
                full_evictable_size,
                swa_available_size,
                swa_evictable_size,
            ) = self._get_swa_token_info()
            memory_leak = full_num_used != 0 or swa_num_used != 0
            token_msg = (
                f"{self.full_tokens_per_layer=}, {full_available_size=}, {full_evictable_size=}, {self.tree_cache.full_protected_size()=}\n"
                f"{self.swa_tokens_per_layer=}, {swa_available_size=}, {swa_evictable_size=}, {self.tree_cache.swa_protected_size()=}\n"
            )
tarinkk's avatar
tarinkk committed
1506
        else:
Hanming Lu's avatar
Hanming Lu committed
1507
1508
1509
1510
1511
1512
            _, _, available_size, evictable_size = self._get_token_info()
            protected_size = self.tree_cache.protected_size()
            memory_leak = (available_size + evictable_size) != (
                self.max_total_num_tokens
                if not self.enable_hierarchical_cache
                else self.max_total_num_tokens - protected_size
Lianmin Zheng's avatar
Lianmin Zheng committed
1513
            )
Hanming Lu's avatar
Hanming Lu committed
1514
1515
1516
1517
            token_msg = f"{self.max_total_num_tokens=}, {available_size=}, {evictable_size=}, {protected_size=}\n"

        if memory_leak:
            msg = "token_to_kv_pool_allocator memory leak detected! " f"{token_msg}"
Lianmin Zheng's avatar
Lianmin Zheng committed
1518
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1519

1520
1521
1522
1523
1524
1525
1526
1527
        if self.disaggregation_mode == DisaggregationMode.DECODE:
            req_total_size = (
                self.req_to_token_pool.size + self.req_to_token_pool.pre_alloc_size
            )
        else:
            req_total_size = self.req_to_token_pool.size

        if len(self.req_to_token_pool.free_slots) != req_total_size:
1528
            msg = (
1529
                "req_to_token_pool memory leak detected!"
1530
1531
                f"available_size={len(self.req_to_token_pool.free_slots)}, "
                f"total_size={self.req_to_token_pool.size}\n"
Lianmin Zheng's avatar
Lianmin Zheng committed
1532
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1533
            raise ValueError(msg)
Lianmin Zheng's avatar
Lianmin Zheng committed
1534

1535
1536
        if (
            self.enable_metrics
1537
            and self.current_scheduler_metrics_enabled()
1538
            and time.perf_counter() > self.metrics_collector.last_log_time + 30
1539
1540
        ):
            # During idle time, also collect metrics every 30 seconds.
Hanming Lu's avatar
Hanming Lu committed
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
            if self.is_hybrid:
                (
                    full_num_used,
                    swa_num_used,
                    full_token_usage,
                    swa_token_usage,
                    _,
                    _,
                    _,
                    _,
                ) = self._get_swa_token_info()
                num_used = max(full_num_used, swa_num_used)
                token_usage = max(full_token_usage, swa_token_usage)
            else:
                num_used, token_usage, _, _ = self._get_token_info()
Lianmin Zheng's avatar
Lianmin Zheng committed
1556
            num_running_reqs = len(self.running_batch.reqs)
1557
1558
            self.stats.num_running_reqs = num_running_reqs
            self.stats.num_used_tokens = num_used
Hanming Lu's avatar
Hanming Lu committed
1559
            self.stats.token_usage = round(token_usage, 2)
1560
1561
            self.stats.gen_throughput = 0
            self.stats.num_queue_reqs = len(self.waiting_queue)
1562
            self.stats.num_grammar_queue_reqs = len(self.grammar_queue)
1563
            self.metrics_collector.log_stats(self.stats)
1564
        self._publish_kv_events()
1565

Hanming Lu's avatar
Hanming Lu committed
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
    def check_tree_cache(self):
        if self.is_hybrid and isinstance(self.tree_cache, SWARadixCache):
            self.tree_cache.sanity_check()

    def _get_token_info(self):
        available_size = self.token_to_kv_pool_allocator.available_size()
        evictable_size = self.tree_cache.evictable_size()
        num_used = self.max_total_num_tokens - (available_size + evictable_size)
        token_usage = num_used / self.max_total_num_tokens
        return num_used, token_usage, available_size, evictable_size

    def _get_swa_token_info(self):
        full_available_size = self.token_to_kv_pool_allocator.full_available_size()
        full_evictable_size = self.tree_cache.full_evictable_size()
        swa_available_size = self.token_to_kv_pool_allocator.swa_available_size()
        swa_evictable_size = self.tree_cache.swa_evictable_size()
        full_num_used = self.full_tokens_per_layer - (
            full_available_size + full_evictable_size
        )
        swa_num_used = self.swa_tokens_per_layer - (
            swa_available_size + swa_evictable_size
        )
        full_token_usage = full_num_used / self.full_tokens_per_layer
        swa_token_usage = swa_num_used / self.swa_tokens_per_layer
        return (
            full_num_used,
            swa_num_used,
            full_token_usage,
            swa_token_usage,
            full_available_size,
            full_evictable_size,
            swa_available_size,
            swa_evictable_size,
        )

1601
    def get_next_batch_to_run(self) -> Optional[ScheduleBatch]:
1602
        # Merge the prefill batch into the running batch
1603
1604
1605
1606
1607
1608
1609
1610
        chunked_req_to_exclude = set()
        if self.chunked_req:
            # Move the chunked request out of the batch so that we can merge
            # only finished requests to running_batch.
            chunked_req_to_exclude.add(self.chunked_req)
            self.tree_cache.cache_unfinished_req(self.chunked_req)
            # chunked request keeps its rid but will get a new req_pool_idx
            self.req_to_token_pool.free(self.chunked_req.req_pool_idx)
Lianmin Zheng's avatar
Lianmin Zheng committed
1611
        if self.last_batch and self.last_batch.forward_mode.is_extend():
1612
1613
1614
1615
            if self.last_batch.chunked_req is not None:
                # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
                # We need to discard it.
                chunked_req_to_exclude.add(self.last_batch.chunked_req)
Lianmin Zheng's avatar
Lianmin Zheng committed
1616

1617
            # Filter batch
1618
            last_bs = self.last_batch.batch_size()
1619
1620
1621
            self.last_batch.filter_batch(
                chunked_req_to_exclude=list(chunked_req_to_exclude)
            )
1622
            if self.last_batch.batch_size() < last_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1623
                self.running_batch.batch_is_full = False
1624

1625
            # Merge the new batch into the running batch
1626
            if not self.last_batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1627
                if self.running_batch.is_empty():
1628
1629
                    self.running_batch = self.last_batch
                else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1630
                    # Merge running_batch with prefill batch
1631
                    self.running_batch.merge_batch(self.last_batch)
1632

1633
        new_batch = self.get_new_batch_prefill()
1634

1635
1636
1637
1638
1639
        need_dp_attn_preparation = require_mlp_sync(self.server_args)

        if need_dp_attn_preparation and not self.spec_algorithm.is_none():
            # In speculative decoding, prefill batches and decode batches cannot be processed in the same DP attention group.
            # We prepare idle batches in advance to skip preparing decode batches when there are prefill batches in the group.
1640
            new_batch = self.prepare_mlp_sync_batch(new_batch)
1641
1642
1643
            need_dp_attn_preparation = new_batch is None

        if new_batch is not None:
1644
1645
1646
1647
            # Run prefill first if possible
            ret = new_batch
        else:
            # Run decode
Lianmin Zheng's avatar
Lianmin Zheng committed
1648
            if not self.running_batch.is_empty():
1649
                self.running_batch = self.update_running_batch(self.running_batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1650
1651
1652
                ret = self.running_batch if not self.running_batch.is_empty() else None
            else:
                ret = None
1653

1654
1655
        # Handle DP attention
        if need_dp_attn_preparation:
1656
            ret = self.prepare_mlp_sync_batch(ret)
1657
1658

        return ret
1659

1660
1661
1662
1663
1664
1665
    def get_num_allocatable_reqs(self, running_bs):
        res = global_server_args_dict["max_micro_batch_size"] - running_bs
        if self.pp_size > 1:
            res = min(res, self.req_to_token_pool.available_size())
        return res

Lianmin Zheng's avatar
Lianmin Zheng committed
1666
    def get_new_batch_prefill(self) -> Optional[ScheduleBatch]:
Lianmin Zheng's avatar
Lianmin Zheng committed
1667
        # Check if the grammar is ready in the grammar queue
1668
        if self.grammar_queue:
1669
            self.move_ready_grammar_requests()
1670

Lianmin Zheng's avatar
Lianmin Zheng committed
1671
1672
        # Handle the cases where prefill is not allowed
        if (
Lianmin Zheng's avatar
Lianmin Zheng committed
1673
            self.running_batch.batch_is_full or len(self.waiting_queue) == 0
1674
        ) and self.chunked_req is None:
Lianmin Zheng's avatar
Lianmin Zheng committed
1675
1676
            return None

Lianmin Zheng's avatar
Lianmin Zheng committed
1677
        running_bs = len(self.running_batch.reqs)
1678
        # Ignore the check if self.chunked_req is not None.
1679
1680
1681
1682
1683
        # In the non-PP case, when self.chunked_req is not None, num_allocatable_reqs should always be greater than 0,
        # as the space for the chunked request has just been released.
        # In PP case, a chunked req can start in one microbatch and end in another microbatch, so the max_running_requests per microbatch should not be strict.
        # Instead, we should always allow chunked request to be added, otherwise, there will be a memory leak.
        if self.get_num_allocatable_reqs(running_bs) <= 0 and not self.chunked_req:
Lianmin Zheng's avatar
Lianmin Zheng committed
1684
            self.running_batch.batch_is_full = True
1685
1686
            return None

1687
        if self.enable_hierarchical_cache:
1688
            self.tree_cache.check_hicache_events()
1689

1690
        # Get priority queue
1691
        self.policy.calc_priority(self.waiting_queue)
1692

Lianmin Zheng's avatar
Lianmin Zheng committed
1693
        # Prefill policy
1694
        adder = PrefillAdder(
1695
            self.page_size,
1696
            self.tree_cache,
1697
            self.token_to_kv_pool_allocator,
1698
1699
1700
1701
            self.running_batch,
            self.new_token_ratio,
            self.max_prefill_tokens,
            self.chunked_prefill_size,
1702
            running_bs if self.is_mixed_chunk else 0,
1703
1704
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
1705
        if self.chunked_req is not None:
1706
1707
            self.chunked_req.init_next_round_input()
            self.chunked_req = adder.add_chunked_req(self.chunked_req)
1708

1709
        if self.enable_lora:
Lianmin Zheng's avatar
Lianmin Zheng committed
1710
1711
            lora_set = set([req.lora_path for req in self.running_batch.reqs])

1712
        # Get requests from the waiting queue to a new prefill batch
1713
1714
        for req in self.waiting_queue:
            if (
1715
                self.enable_lora
1716
1717
1718
1719
1720
1721
1722
                and len(
                    lora_set
                    | set([req.lora_path for req in adder.can_run_list])
                    | set([req.lora_path])
                )
                > self.max_loras_per_batch
            ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1723
                self.running_batch.batch_is_full = True
1724
1725
                break

1726
            if len(adder.can_run_list) >= self.get_num_allocatable_reqs(running_bs):
Lianmin Zheng's avatar
Lianmin Zheng committed
1727
                self.running_batch.batch_is_full = True
1728
                break
1729

Byron Hsu's avatar
Byron Hsu committed
1730
1731
1732
1733
1734
1735
1736
            if self.disaggregation_mode == DisaggregationMode.PREFILL:
                # In prefill mode, prealloc queue and transfer queue can also take memory,
                # so we need to check if the available size for the actual available size.
                if len(adder.can_run_list) >= self.req_to_token_pool.available_size():
                    self.running_batch.batch_is_full = True
                    break

1737
1738
1739
            if self.enable_hicache_storage:
                self.tree_cache.check_prefetch_progress(req.rid)

1740
1741
            req.init_next_round_input(self.tree_cache)
            res = adder.add_one_req(req, has_chunked_req=(self.chunked_req is not None))
1742

1743
1744
            if res != AddReqResult.CONTINUE:
                if res == AddReqResult.NO_TOKEN:
1745
1746
                    if self.enable_hierarchical_cache:
                        # Set batch_is_full after making sure there are requests that can be served
Lianmin Zheng's avatar
Lianmin Zheng committed
1747
1748
                        self.running_batch.batch_is_full = len(
                            adder.can_run_list
1749
                        ) > 0 or (not self.running_batch.is_empty())
1750
                    else:
Lianmin Zheng's avatar
Lianmin Zheng committed
1751
                        self.running_batch.batch_is_full = True
1752
1753
                break

Lianmin Zheng's avatar
Lianmin Zheng committed
1754
        # Update waiting queue
1755
        can_run_list: List[Req] = adder.can_run_list
Lianmin Zheng's avatar
Lianmin Zheng committed
1756
1757
        if len(can_run_list) == 0:
            return None
1758
1759
1760
1761

        if self.enable_metrics:
            # only record queue time when enable_metrics is True to avoid overhead
            for req in can_run_list:
1762
                req.queue_time_end = time.perf_counter()
1763

Lianmin Zheng's avatar
Lianmin Zheng committed
1764
1765
1766
        self.waiting_queue = [
            x for x in self.waiting_queue if x not in set(can_run_list)
        ]
1767

1768
1769
1770
        if adder.new_chunked_req is not None:
            assert self.chunked_req is None
            self.chunked_req = adder.new_chunked_req
1771

1772
1773
        if self.chunked_req:
            self.chunked_req.is_chunked += 1
Lianmin Zheng's avatar
Lianmin Zheng committed
1774

1775
        # Print stats
1776
        if self.current_scheduler_metrics_enabled():
1777
            self.log_prefill_stats(adder, can_run_list, running_bs)
1778

Lianmin Zheng's avatar
Lianmin Zheng committed
1779
        # Create a new batch
1780
1781
1782
        new_batch = ScheduleBatch.init_new(
            can_run_list,
            self.req_to_token_pool,
1783
            self.token_to_kv_pool_allocator,
1784
            self.tree_cache,
1785
            self.model_config,
1786
            self.enable_overlap,
1787
            self.spec_algorithm,
1788
            self.server_args.enable_custom_logit_processor,
1789
            chunked_req=self.chunked_req,
1790
        )
1791
1792
        if self.enable_hierarchical_cache:
            # todo (zhiqiang): disable cuda graph execution if hicache loading triggered
1793
1794
1795
            new_batch.hicache_consumer_index = (
                self.tree_cache.ready_to_load_host_cache()
            )
1796

1797
        new_batch.prepare_for_extend()
1798

Lianmin Zheng's avatar
Lianmin Zheng committed
1799
        # Mixed-style chunked prefill
1800
1801
        if (
            self.is_mixed_chunk
Lianmin Zheng's avatar
Lianmin Zheng committed
1802
            and not self.running_batch.is_empty()
1803
1804
1805
            and not (new_batch.return_logprob or self.running_batch.return_logprob)
        ):
            # TODO (lianmin): support return_logprob + mixed chunked prefill
1806
1807
            self.running_batch.filter_batch()
            if not self.running_batch.is_empty():
1808
                self.running_batch.prepare_for_decode()
1809
1810
                new_batch.mix_with_running(self.running_batch)
                new_batch.decoding_reqs = self.running_batch.reqs
Lianmin Zheng's avatar
Lianmin Zheng committed
1811
1812
1813
            self.running_batch = ScheduleBatch(
                reqs=[], batch_is_full=self.running_batch.batch_is_full
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1814
1815
        else:
            new_batch.decoding_reqs = None
Lianmin Zheng's avatar
Lianmin Zheng committed
1816
1817
1818

        return new_batch

Lianmin Zheng's avatar
Lianmin Zheng committed
1819
    def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
1820
        """Update the current running decoding batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1821
        initial_bs = batch.batch_size()
Lianmin Zheng's avatar
Lianmin Zheng committed
1822

1823
1824
        batch.filter_batch()
        if batch.is_empty():
Lianmin Zheng's avatar
Lianmin Zheng committed
1825
1826
            batch.batch_is_full = False
            return batch
1827

Lianmin Zheng's avatar
Lianmin Zheng committed
1828
        # Check if decode out of memory
1829
        if not batch.check_decode_mem(self.decode_mem_cache_buf_multiplier) or (
1830
            TEST_RETRACT and batch.batch_size() > 10
1831
        ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1832
1833
            old_ratio = self.new_token_ratio

1834
            retracted_reqs, new_token_ratio = batch.retract_decode(self.server_args)
1835
            num_retracted_reqs = len(retracted_reqs)
Lianmin Zheng's avatar
Lianmin Zheng committed
1836
            self.new_token_ratio = new_token_ratio
1837

Lianmin Zheng's avatar
Lianmin Zheng committed
1838
            logger.info(
1839
                "KV cache pool is full. Retract requests. "
1840
                f"#retracted_reqs: {num_retracted_reqs}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
1841
1842
                f"#new_token_ratio: {old_ratio:.4f} -> {self.new_token_ratio:.4f}"
            )
1843

1844
            self._extend_requests_to_queue(retracted_reqs, is_retracted=True)
1845
            self.total_retracted_reqs += num_retracted_reqs
Lianmin Zheng's avatar
Lianmin Zheng committed
1846
1847
        else:
            self.new_token_ratio = max(
1848
                self.new_token_ratio - self.new_token_ratio_decay,
Lianmin Zheng's avatar
Lianmin Zheng committed
1849
1850
1851
                self.min_new_token_ratio,
            )

Lianmin Zheng's avatar
Lianmin Zheng committed
1852
        if batch.batch_size() < initial_bs:
Lianmin Zheng's avatar
Lianmin Zheng committed
1853
            batch.batch_is_full = False
Lianmin Zheng's avatar
Lianmin Zheng committed
1854
1855

        # Update batch tensors
1856
        batch.prepare_for_decode()
Lianmin Zheng's avatar
Lianmin Zheng committed
1857
        return batch
Lianmin Zheng's avatar
Lianmin Zheng committed
1858

1859
1860
1861
    def run_batch(
        self, batch: ScheduleBatch
    ) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
1862
        """Run a batch."""
Lianmin Zheng's avatar
Lianmin Zheng committed
1863
1864
        self.forward_ct += 1

1865
1866
        # Whether to run the profiler
        self._profile_batch_predicate(batch)
1867
1868
1869
1870
        if self.forward_sleep_time is not None:
            logger.info(f"Scheduler.run_batch sleep {self.forward_sleep_time}s")
            time.sleep(self.forward_sleep_time)

1871
        # Run forward
1872
        if self.is_generation:
1873
1874
            if self.spec_algorithm.is_none():
                model_worker_batch = batch.get_model_worker_batch()
1875
1876
1877
1878
1879

                # update the consumer index of hicache to the running batch
                self.tp_worker.set_hicache_consumer(
                    model_worker_batch.hicache_consumer_index
                )
1880
                if self.pp_group.is_last_rank:
1881
                    logits_output, next_token_ids, can_run_cuda_graph = (
1882
1883
1884
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
                else:
1885
                    pp_hidden_states_proxy_tensors, _, can_run_cuda_graph = (
1886
1887
                        self.tp_worker.forward_batch_generation(model_worker_batch)
                    )
1888
                bid = model_worker_batch.bid
Lianmin Zheng's avatar
Lianmin Zheng committed
1889
            else:
1890
1891
1892
                (
                    logits_output,
                    next_token_ids,
1893
                    bid,
1894
                    num_accepted_tokens,
1895
                    can_run_cuda_graph,
1896
                ) = self.draft_worker.forward_batch_speculative_generation(batch)
1897
1898
1899
                bs = batch.batch_size()
                self.spec_num_total_accepted_tokens += num_accepted_tokens + bs
                self.spec_num_total_forward_ct += bs
1900
                self.num_generated_tokens += num_accepted_tokens
1901
1902
1903

            if self.pp_group.is_last_rank:
                batch.output_ids = next_token_ids
1904

1905
1906
1907
            # These 2 values are needed for processing the output, but the values can be
            # modified by overlap schedule. So we have to copy them here so that
            # we can use the correct values in output processing.
1908
            if batch.return_logprob or self.spec_algorithm.is_eagle():
1909
                extend_input_len_per_req = [req.extend_input_len for req in batch.reqs]
1910
1911
1912
            else:
                extend_input_len_per_req = None
            if batch.return_logprob:
1913
1914
1915
1916
1917
1918
                extend_logprob_start_len_per_req = [
                    req.extend_logprob_start_len for req in batch.reqs
                ]
            else:
                extend_logprob_start_len_per_req = None

1919
            ret = GenerationBatchResult(
1920
1921
1922
1923
1924
1925
1926
                logits_output=logits_output if self.pp_group.is_last_rank else None,
                pp_hidden_states_proxy_tensors=(
                    pp_hidden_states_proxy_tensors
                    if not self.pp_group.is_last_rank
                    else None
                ),
                next_token_ids=next_token_ids if self.pp_group.is_last_rank else None,
1927
1928
                extend_input_len_per_req=extend_input_len_per_req,
                extend_logprob_start_len_per_req=extend_logprob_start_len_per_req,
1929
                bid=bid,
1930
                can_run_cuda_graph=can_run_cuda_graph,
1931
            )
Lianmin Zheng's avatar
Lianmin Zheng committed
1932
1933
1934
        else:  # embedding or reward model
            model_worker_batch = batch.get_model_worker_batch()
            embeddings = self.tp_worker.forward_batch_embedding(model_worker_batch)
1935
1936
1937
            ret = EmbeddingBatchResult(
                embeddings=embeddings, bid=model_worker_batch.bid
            )
1938
        return ret
Chayenne's avatar
Chayenne committed
1939

1940
1941
1942
1943
    def process_batch_result(
        self,
        batch: ScheduleBatch,
        result: Union[GenerationBatchResult, EmbeddingBatchResult],
1944
        launch_done: Optional[threading.Event] = None,
1945
    ):
Lianmin Zheng's avatar
Lianmin Zheng committed
1946
        if batch.forward_mode.is_decode():
1947
            self.process_batch_result_decode(batch, result, launch_done)
1948
        elif batch.forward_mode.is_extend():
1949
            self.process_batch_result_prefill(batch, result, launch_done)
1950
1951
        elif batch.forward_mode.is_idle():
            if self.enable_overlap:
1952
                self.tp_worker.resolve_last_batch_result(launch_done)
1953
                self.set_next_batch_sampling_info_done(batch)
1954
        elif batch.forward_mode.is_dummy_first():
1955
            self.set_next_batch_sampling_info_done(batch)
Lianmin Zheng's avatar
Lianmin Zheng committed
1956

1957
1958
1959
1960
1961
1962
1963
        if self.return_health_check_ct:
            # Return some signal for the health check.
            # This is used to prevent the health check signal being blocked by long context prefill.
            # However, one minor issue is that this code path does not check the status of detokenizer manager.
            self.return_health_check_ct -= 1
            self.send_to_tokenizer.send_pyobj(HealthCheckOutput())

1964
1965
    def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
        return self.prepare_mlp_sync_batch_raw(
1966
1967
1968
            local_batch,
            dp_size=self.server_args.dp_size,
            attn_tp_size=self.attn_tp_size,
1969
            tp_group=self.tp_group,
1970
1971
1972
1973
            get_idle_batch=self.get_idle_batch,
            disable_cuda_graph=self.server_args.disable_cuda_graph,
            spec_algorithm=self.spec_algorithm,
            speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens,
1974
1975
1976
            enable_two_batch_overlap=self.server_args.enable_two_batch_overlap,
            enable_deepep_moe=self.server_args.enable_deepep_moe,
            deepep_mode=DeepEPMode[self.server_args.deepep_mode],
1977
            require_mlp_tp_gather=require_mlp_tp_gather(self.server_args),
1978
            disable_overlap_schedule=self.server_args.disable_overlap_schedule,
1979
1980
1981
        )

    @staticmethod
1982
    def prepare_mlp_sync_batch_raw(
1983
1984
1985
        local_batch: ScheduleBatch,
        dp_size,
        attn_tp_size: int,
1986
        tp_group,
1987
1988
1989
1990
        get_idle_batch,
        disable_cuda_graph: bool,
        spec_algorithm,
        speculative_num_draft_tokens,
1991
1992
1993
        enable_two_batch_overlap: bool,
        enable_deepep_moe: bool,
        deepep_mode: DeepEPMode,
1994
        require_mlp_tp_gather: bool,
1995
        disable_overlap_schedule: bool,
1996
    ):
1997
1998
1999
        # Check if other DP workers have running batches
        if local_batch is None:
            num_tokens = 0
2000
            num_tokens_for_logprob = 0
2001
2002
        elif local_batch.forward_mode.is_decode():
            num_tokens = local_batch.batch_size()
2003
            num_tokens_for_logprob = num_tokens
2004
2005
        else:
            num_tokens = local_batch.extend_num_tokens
2006
            num_tokens_for_logprob = sum(
Lianmin Zheng's avatar
Lianmin Zheng committed
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
                [
                    # We should have at least 1 token for sample in every case.
                    max(extend_len - logprob_start_len, 1)
                    for logprob_start_len, extend_len in zip(
                        local_batch.extend_logprob_start_lens, local_batch.extend_lens
                    )
                ]
            )

        if local_batch is None or local_batch.forward_mode.is_decode_or_idle():
            can_cuda_graph = 1
        else:
            can_cuda_graph = 0

        is_extend_in_batch = (
            local_batch.forward_mode.is_extend() if local_batch else False
        )
2024
2025

        tbo_preparer = TboDPAttentionPreparer()
2026
2027
2028
2029
2030
2031
        if disable_overlap_schedule:
            group = tp_group.device_group
            device = tp_group.device
        else:
            group = tp_group.cpu_group
            device = "cpu"
2032

Lianmin Zheng's avatar
Lianmin Zheng committed
2033
2034
2035
2036
        local_info = torch.tensor(
            [
                num_tokens,
                can_cuda_graph,
2037
                num_tokens_for_logprob,
Lianmin Zheng's avatar
Lianmin Zheng committed
2038
                is_extend_in_batch,
2039
2040
2041
2042
2043
2044
                *tbo_preparer.prepare_all_gather(
                    local_batch,
                    deepep_mode,
                    enable_deepep_moe,
                    enable_two_batch_overlap,
                ),
Lianmin Zheng's avatar
Lianmin Zheng committed
2045
2046
            ],
            dtype=torch.int64,
2047
            device=device,
Lianmin Zheng's avatar
Lianmin Zheng committed
2048
2049
        )
        global_info = torch.empty(
2050
            (dp_size, attn_tp_size, 6),
Lianmin Zheng's avatar
Lianmin Zheng committed
2051
            dtype=torch.int64,
2052
            device=device,
Lianmin Zheng's avatar
Lianmin Zheng committed
2053
        )
2054
        torch.distributed.all_gather_into_tensor(
Lianmin Zheng's avatar
Lianmin Zheng committed
2055
2056
            global_info.flatten(),
            local_info,
2057
            group=group,
2058
        )
Lianmin Zheng's avatar
Lianmin Zheng committed
2059
2060
2061
2062
        global_num_tokens = global_info[:, 0, 0].tolist()
        can_cuda_graph = min(global_info[:, 0, 1].tolist())
        global_num_tokens_for_logprob = global_info[:, 0, 2].tolist()
        is_extend_in_batch = global_info[:, 0, 3].tolist()
2063

2064
2065
2066
2067
        tbo_split_seq_index, global_forward_mode = tbo_preparer.compute_output(
            global_info[:, :, 4:6]
        )

Lianmin Zheng's avatar
Lianmin Zheng committed
2068
        if local_batch is None and max(global_num_tokens) > 0:
2069
            local_batch = get_idle_batch()
2070
2071

        if local_batch is not None:
2072
            # TODO: handle the case when moe_dense_tp_size != 1
2073
            if not require_mlp_tp_gather:
2074
2075
2076
2077
2078
2079
2080
                local_batch.global_num_tokens = [num_tokens]
                local_batch.global_num_tokens_for_logprob = [num_tokens_for_logprob]
            else:
                local_batch.global_num_tokens = global_num_tokens
                local_batch.global_num_tokens_for_logprob = (
                    global_num_tokens_for_logprob
                )
2081
            local_batch.is_extend_in_batch = any(is_extend_in_batch)
2082
2083
            local_batch.tbo_split_seq_index = tbo_split_seq_index
            local_batch.global_forward_mode = global_forward_mode
2084

2085
            # Check forward mode for cuda graph
2086
            if not disable_cuda_graph:
Lianmin Zheng's avatar
Lianmin Zheng committed
2087
                local_batch.can_run_dp_cuda_graph = can_cuda_graph
2088

2089
        return local_batch
2090
2091
2092
2093
2094

    def get_idle_batch(self):
        idle_batch = ScheduleBatch.init_new(
            [],
            self.req_to_token_pool,
2095
            self.token_to_kv_pool_allocator,
2096
2097
2098
            self.tree_cache,
            self.model_config,
            self.enable_overlap,
2099
            self.spec_algorithm,
2100
            self.server_args.enable_custom_logit_processor,
2101
2102
2103
2104
        )
        idle_batch.prepare_for_idle()
        return idle_batch

2105
2106
    def move_ready_grammar_requests(self):
        """Move requests whose grammar objects are ready from grammar_queue to waiting_queue."""
2107

2108
        num_ready_reqs = 0
2109
        num_timeout_reqs = 0
2110
2111
        for req in self.grammar_queue:
            try:
2112
2113
2114
                if req.finished():  # It is aborted by AbortReq
                    num_ready_reqs += 1
                    continue
2115
                req.grammar = req.grammar.result(timeout=0.03)
2116
2117
2118
2119
2120
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
2121
2122
                num_ready_reqs += 1
            except futures._base.TimeoutError:
2123
                req.grammar_wait_ct += 1
2124
2125
                # NOTE(lianmin): this timeout is the waiting time of the above line. It is
                # not the waiting time from it enters the grammar queue.
2126
                if req.grammar_wait_ct > GRAMMAR_TIMEOUT / 0.03:
2127
                    num_timeout_reqs = 1
2128
2129
                break

2130
        if self.server_args.enable_dp_attention:
2131
2132
            tp_size = self.attn_tp_size
            tp_group = self.attn_tp_cpu_group
2133
        else:
2134
2135
2136
2137
2138
            tp_size = self.tp_size
            tp_group = self.tp_cpu_group

        if tp_size > 1:
            # Sync across TP ranks to make sure they have the same number of ready requests
2139
            tensor = torch.tensor([num_ready_reqs, num_timeout_reqs], dtype=torch.int32)
2140
2141
2142
            torch.distributed.all_reduce(
                tensor, op=torch.distributed.ReduceOp.MAX, group=tp_group
            )
2143
            num_ready_reqs_max, num_timeout_reqs_max = tensor.tolist()
2144

2145
            for i in range(num_ready_reqs, num_ready_reqs_max):
2146
                req = self.grammar_queue[i]
2147
2148
                if req.finished():  # It is aborted by AbortReq
                    continue
2149
                req.grammar = req.grammar.result()
2150
2151
2152
2153
2154
2155
2156
2157
                self.grammar_backend.set_cache(req.grammar_key, req.grammar.copy())
                if req.grammar is INVALID_GRAMMAR_OBJ:
                    req.set_finish_with_abort(
                        f"Invalid grammar request: {req.grammar_key=}"
                    )
        else:
            num_ready_reqs_max = num_ready_reqs
            num_timeout_reqs_max = num_timeout_reqs
2158

2159
2160
2161
2162
2163
2164
2165
        for i in range(num_ready_reqs, num_ready_reqs + num_timeout_reqs_max):
            req = self.grammar_queue[i]
            req.grammar.cancel()
            error_msg = f"Grammar preprocessing timed out for {req.grammar_key=}"
            req.set_finish_with_abort(error_msg)
            self.grammar_backend.set_cache(req.grammar_key, INVALID_GRAMMAR_OBJ)
        num_ready_reqs = num_ready_reqs_max + num_timeout_reqs_max
2166

2167
        self._extend_requests_to_queue(self.grammar_queue[:num_ready_reqs])
2168
2169
        self.grammar_queue = self.grammar_queue[num_ready_reqs:]

2170
2171
2172
2173
2174
2175
2176
    def set_next_batch_sampling_info_done(self, batch: ScheduleBatch):
        if batch.next_batch_sampling_info:
            if batch.next_batch_sampling_info.grammars is not None:
                batch.next_batch_sampling_info.update_regex_vocab_mask()
                self.current_stream.synchronize()
            batch.next_batch_sampling_info.sampling_info_done.set()

2177
2178
2179
    def watchdog_thread(self):
        """A watch dog thread that will try to kill the server itself if one forward batch takes too long."""
        self.watchdog_last_forward_ct = 0
2180
        self.watchdog_last_time = time.perf_counter()
2181
2182

        while True:
2183
            current = time.perf_counter()
2184
2185
2186
2187
2188
2189
2190
2191
2192
            if self.cur_batch is not None:
                if self.watchdog_last_forward_ct == self.forward_ct:
                    if current > self.watchdog_last_time + self.watchdog_timeout:
                        break
                else:
                    self.watchdog_last_forward_ct = self.forward_ct
                    self.watchdog_last_time = current
            time.sleep(self.watchdog_timeout // 2)

Lianmin Zheng's avatar
Lianmin Zheng committed
2193
2194
        if not disable_request_logging():
            # Print batch size and memory pool info to check whether there are de-sync issues.
Hanming Lu's avatar
Hanming Lu committed
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
            if self.is_hybrid:
                (
                    _,
                    _,
                    _,
                    _,
                    full_available_size,
                    full_evictable_size,
                    swa_available_size,
                    swa_evictable_size,
                ) = self._get_swa_token_info()
                info_msg = (
                    f"{full_available_size=}, "
                    f"{full_evictable_size=}, "
                    f"{swa_available_size=}, "
                    f"{swa_evictable_size=}, "
                )
            else:
                _, _, available_size, evictable_size = self._get_token_info()
                info_msg = f"{available_size=}, " f"{evictable_size=}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2215
2216
2217
            logger.error(
                f"{self.cur_batch.batch_size()=}, "
                f"{self.cur_batch.reqs=}, "
Hanming Lu's avatar
Hanming Lu committed
2218
                f"{info_msg}"
Lianmin Zheng's avatar
Lianmin Zheng committed
2219
2220
            )

2221
        pyspy_dump_schedulers()
Lianmin Zheng's avatar
Lianmin Zheng committed
2222
        logger.error(f"Watchdog timeout ({self.watchdog_timeout=})")
2223
2224
        print(file=sys.stderr, flush=True)
        print(file=sys.stdout, flush=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
2225
2226

        # Wait for some time so that the parent process can print the error.
2227
2228
2229
        time.sleep(5)
        self.parent_process.send_signal(signal.SIGQUIT)

2230
2231
2232
    def flush_cache_wrapped(self, recv_req: FlushCacheReqInput):
        success = self.flush_cache()
        return FlushCacheReqOutput(success=success)
2233

2234
    def flush_cache(self):
2235
        """Flush the memory pool and cache."""
2236
2237
2238
2239
2240
        if (
            len(self.waiting_queue) == 0
            and self.running_batch.is_empty()
            and (self.pp_size == 1 or all(x.is_empty() for x in self.running_mbs))
        ):
2241
2242
            self.cur_batch = None
            self.last_batch = None
2243
            self.tree_cache.reset()
2244
            if self.grammar_backend:
Lianmin Zheng's avatar
Lianmin Zheng committed
2245
                self.grammar_backend.reset()
2246
            self.req_to_token_pool.clear()
2247
            self.token_to_kv_pool_allocator.clear()
2248
2249
2250

            if not self.spec_algorithm.is_none():
                self.draft_worker.model_runner.req_to_token_pool.clear()
2251
                self.draft_worker.model_runner.token_to_kv_pool_allocator.clear()
2252
2253
2254
2255
2256

            self.num_generated_tokens = 0
            self.forward_ct_decode = 0
            self.spec_num_total_accepted_tokens = 0
            self.spec_num_total_forward_ct = 0
2257
2258
            self.cum_spec_accept_length = 0
            self.cum_spec_accept_count = 0
2259
2260
2261
2262
2263
2264
2265
            torch.cuda.empty_cache()
            logger.info("Cache flushed successfully!")
            if_success = True
        else:
            logging.warning(
                f"Cache not flushed because there are pending requests. "
                f"#queue-req: {len(self.waiting_queue)}, "
Lianmin Zheng's avatar
Lianmin Zheng committed
2266
                f"#running-req: {len(self.running_batch.reqs)}"
2267
2268
2269
2270
            )
            if_success = False
        return if_success

Liangsheng Yin's avatar
Liangsheng Yin committed
2271
2272
    def get_load(self):
        # TODO(lsyin): use dynamically maintained num_waiting_tokens
Hanming Lu's avatar
Hanming Lu committed
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
        if self.is_hybrid:
            load_full = (
                self.full_tokens_per_layer
                - self.token_to_kv_pool_allocator.full_available_size()
                - self.tree_cache.full_evictable_size()
            )
            load_swa = (
                self.swa_tokens_per_layer
                - self.token_to_kv_pool_allocator.swa_available_size()
                - self.tree_cache.swa_evictable_size()
            )
            load = max(load_full, load_swa)
        else:
            load = (
                self.max_total_num_tokens
                - self.token_to_kv_pool_allocator.available_size()
                - self.tree_cache.evictable_size()
            )
Liangsheng Yin's avatar
Liangsheng Yin committed
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
        load += sum(len(req.origin_input_ids) for req in self.waiting_queue)
        if self.disaggregation_mode == DisaggregationMode.PREFILL:
            load += sum(
                len(req.origin_input_ids)
                for req in self.disagg_prefill_bootstrap_queue.queue
            )
        elif self.disaggregation_mode == DisaggregationMode.DECODE:
            load += sum(
                len(req.req.origin_input_ids)
                for req in self.disagg_decode_prealloc_queue.queue
            )

        return load

2305
2306
2307
    def get_internal_state(self, recv_req: GetInternalStateReq):
        ret = dict(global_server_args_dict)
        ret["last_gen_throughput"] = self.last_gen_throughput
2308
2309
2310
2311
2312
2313
2314
2315
2316
        ret["memory_usage"] = {
            "weight": round(
                self.tp_worker.worker.model_runner.weight_load_mem_usage, 2
            ),
            "kvcache": round(
                self.token_to_kv_pool_allocator.get_kvcache().mem_usage, 2
            ),
            "token_capacity": int(self.max_total_num_tokens),
        }
2317
2318
2319
2320
2321
2322

        if not _is_cpu:
            ret["memory_usage"]["cuda_graph"] = round(
                self.tp_worker.worker.model_runner.cuda_graph_mem_usage, 2
            )

2323
2324
2325
2326
2327
2328
        if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
            ret["avg_spec_accept_length"] = (
                self.cum_spec_accept_length / self.cum_spec_accept_count
            )
        if RECORD_STEP_TIME:
            ret["step_time_dict"] = self.step_time_dict
Liangsheng Yin's avatar
Liangsheng Yin committed
2329
2330
2331
2332

        ret["load"] = self.get_load()

        return GetInternalStateReqOutput(internal_state=ret)
2333
2334
2335
2336
2337

    def set_internal_state(self, recv_req: SetInternalStateReq):
        server_args_dict = recv_req.server_args
        args_allow_update = set(
            [
2338
                "max_micro_batch_size",
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
                "speculative_accept_threshold_single",
                "speculative_accept_threshold_acc",
            ]
        )
        if_success = True
        for k, v in server_args_dict.items():
            if k not in args_allow_update:
                logging.warning(f"Updating {k} is not supported.")
                if_success = False
                break
2349
2350
2351
2352
2353
2354
2355
2356
            elif k == "max_micro_batch_size" and (
                v > self.max_running_requests // self.pp_size or v < 1
            ):
                logging.warning(
                    f"Updating {k} to {v} is rejected because it is out of the valid range [1, {self.max_running_requests // self.pp_size}]."
                )
                if_success = False
                break
2357
2358
2359
2360
2361
2362
2363
2364
2365
        if if_success:
            if not self.spec_algorithm.is_none() and self.cum_spec_accept_count > 0:
                avg_spec_accept_length = (
                    self.cum_spec_accept_length / self.cum_spec_accept_count
                )
                logger.info(f"{avg_spec_accept_length=}")
            self.cum_spec_accept_length = self.cum_spec_accept_count = 0
            for k, v in server_args_dict.items():
                global_server_args_dict[k] = v
2366
            logger.info(f"Global server args updated! {global_server_args_dict=}")
2367
2368
2369
2370
2371
        return SetInternalStateReqOutput(
            updated=True,
            server_args=global_server_args_dict,
        )

2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
    def handle_rpc_request(self, recv_req: RpcReqInput):
        # Handle RPC requests
        logger.info(
            f"handle_rpc_request: {recv_req.method}, param: {recv_req.parameters}"
        )

        success = True
        exec = None
        try:
            func = getattr(self, recv_req.method)
            func(recv_req.parameters)
        except Exception as e:
            success = False
            exec = e
            logger.error(f"Failed to call rpc {recv_req.method}: {str(e)}")

        barrier()
        return RpcReqOutput(success, "" if not exec else str(exec))

    def save_remote_model(self, params):
        url = params["url"]

2394
        worker = self.tp_worker.worker
2395
2396
2397
2398

        worker.model_runner.save_remote_model(url)

    def save_sharded_model(self, params):
2399
        worker = self.tp_worker.worker
2400
2401
2402
2403
2404
2405
2406

        worker.model_runner.save_sharded_model(
            path=params["path"],
            pattern=params["pattern"],
            max_size=params["max_size"],
        )

2407
2408
    def abort_request(self, recv_req: AbortReq):
        # Delete requests in the waiting queue
Lianmin Zheng's avatar
Lianmin Zheng committed
2409
        to_del = []
2410
        for i, req in enumerate(self.waiting_queue):
2411
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
Lianmin Zheng's avatar
Lianmin Zheng committed
2412
                to_del.append(i)
2413

Lianmin Zheng's avatar
Lianmin Zheng committed
2414
        # Sort in reverse order to avoid index issues when deleting
Lianmin Zheng's avatar
Lianmin Zheng committed
2415
        for i in reversed(to_del):
2416
2417
2418
            # Abort method 1: directly pop from the queue
            # This only works for requests that have not started anything.
            # We still need to send something back to TokenizerManager to clean up the state.
Lianmin Zheng's avatar
Lianmin Zheng committed
2419
            req = self.waiting_queue.pop(i)
Lianmin Zheng's avatar
Lianmin Zheng committed
2420
            self.send_to_tokenizer.send_pyobj(AbortReq(req.rid))
2421
            logger.debug(f"Abort queued request. {req.rid=}")
2422

2423
2424
2425
2426
2427
        # Delete the requests in the grammar queue
        for req in self.grammar_queue:
            # Abort method 2: call `set_finish_with_abort`
            # The request will still run one prefill forward pass.
            # In this case, we change the input_ids to be only one token to make this prefill cheap.
2428
            if recv_req.abort_all or req.rid.startswith(recv_req.rid):
2429
                logger.debug(f"Abort grammar queue request. {req.rid=}")
2430
2431
                if req.grammar:
                    req.grammar.cancel()
2432
2433
                req.set_finish_with_abort("Aborted by AbortReq.")

2434
        # Delete requests in the running batch
Lianmin Zheng's avatar
Lianmin Zheng committed
2435
2436
2437
2438
2439
2440
        if self.cur_batch is self.running_batch or self.cur_batch is None:
            reqs = self.running_batch.reqs
        else:
            reqs = self.running_batch.reqs + self.cur_batch.reqs

        for req in reqs:
2441
2442
2443
            if not req.finished() and (
                recv_req.abort_all or req.rid.startswith(recv_req.rid)
            ):
2444
2445
2446
                # Abort method 3: set `to_abort=True`
                # The request will still run one decode forward pass.
                # Then we reuse all existing code to clean up the KV cache allocation.
Lianmin Zheng's avatar
Lianmin Zheng committed
2447
2448
                logger.debug(f"Abort running request. {req.rid=}")
                req.to_abort = True
2449

2450
2451
2452
    def _pause_engine(self) -> Tuple[List[Req], int]:
        raise NotImplementedError()

Chayenne's avatar
Chayenne committed
2453
2454
2455
    def update_weights_from_disk(self, recv_req: UpdateWeightFromDiskReqInput):
        """In-place update of the weights from disk."""
        success, message = self.tp_worker.update_weights_from_disk(recv_req)
2456
        if success:
Stefan He's avatar
Stefan He committed
2457
2458
            flush_cache_success = self.flush_cache()
            assert flush_cache_success, "Cache flush failed after updating weights"
2459
2460
        else:
            logger.error(message)
2461
        return UpdateWeightFromDiskReqOutput(success, message, 0)
2462

2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
    def load_lora_adapter(
        self, recv_req: LoadLoRAAdapterReqInput
    ) -> LoadLoRAAdapterReqOutput:
        """In-place loading a new lora adapter from disk or huggingface."""

        result = self.tp_worker.load_lora_adapter(recv_req)
        return result

    def unload_lora_adapter(
        self, recv_req: UnloadLoRAAdapterReqInput
    ) -> UnloadLoRAAdapterReqOutput:
        """Unload the lora adapter."""

        result = self.tp_worker.unload_lora_adapter(recv_req)
        return result

2479
2480
2481
    def init_weights_update_group(self, recv_req: InitWeightsUpdateGroupReqInput):
        """Initialize the online model parameter update group."""
        success, message = self.tp_worker.init_weights_update_group(recv_req)
2482
        return InitWeightsUpdateGroupReqOutput(success, message)
2483
2484

    def update_weights_from_distributed(
2485
2486
2487
        self,
        recv_req: UpdateWeightsFromDistributedReqInput,
    ) -> Tuple[bool, str]:
2488
2489
2490
        """Update the online model parameter."""
        success, message = self.tp_worker.update_weights_from_distributed(recv_req)
        if success:
2491
2492
2493
            if recv_req.flush_cache:
                flush_cache_success = self.flush_cache()
                assert flush_cache_success, "Cache flush failed after updating weights"
2494
2495
        else:
            logger.error(message)
2496
        return UpdateWeightsFromDistributedReqOutput(success, message)
2497

2498
2499
2500
2501
2502
    def update_weights_from_tensor(self, recv_req: UpdateWeightsFromTensorReqInput):
        """Update the online model parameter from tensors."""
        success, message = self.tp_worker.update_weights_from_tensor(recv_req)
        # TODO extract common code b/t update_weights_from_distributed and update_weights_from_tensor later
        if success:
2503
            if recv_req.flush_cache:
Stefan He's avatar
Stefan He committed
2504
2505
                flush_cache_success = self.flush_cache()
                assert flush_cache_success, "Cache flush failed after updating weights"
2506
2507
        else:
            logger.error(message)
2508
        barrier(group=self.tp_cpu_group)
2509
        return UpdateWeightsFromTensorReqOutput(success, message)
2510

2511
2512
    def get_weights_by_name(self, recv_req: GetWeightsByNameReqInput):
        parameter = self.tp_worker.get_weights_by_name(recv_req)
2513
        return GetWeightsByNameReqOutput(parameter)
2514

2515
    def release_memory_occupation(self, recv_req: ReleaseMemoryOccupationReqInput):
2516
2517
        tags = recv_req.tags

2518
        if tags is None or len(tags) == 0:
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_KV_CACHE)
            self.flush_cache()

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.stashed_model_static_state = _export_static_state(
                self.tp_worker.worker.model_runner.model
            )
2529
            torch.distributed.barrier(self.tp_cpu_group)
2530
2531
            self.memory_saver_adapter.pause(GPU_MEMORY_TYPE_WEIGHTS)

2532
        return ReleaseMemoryOccupationReqOutput()
2533

2534
    def resume_memory_occupation(self, recv_req: ResumeMemoryOccupationReqInput):
2535
        tags = recv_req.tags
2536

2537
2538
2539
2540
2541
        if tags is None or len(tags) == 0:
            tags = [GPU_MEMORY_TYPE_WEIGHTS, GPU_MEMORY_TYPE_KV_CACHE]

        if GPU_MEMORY_TYPE_WEIGHTS in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_WEIGHTS)
2542
            torch.distributed.barrier(self.tp_cpu_group)
2543
2544
2545
2546
2547
2548
2549
2550
2551
            _import_static_state(
                self.tp_worker.worker.model_runner.model,
                self.stashed_model_static_state,
            )
            del self.stashed_model_static_state

        if GPU_MEMORY_TYPE_KV_CACHE in tags:
            self.memory_saver_adapter.resume(GPU_MEMORY_TYPE_KV_CACHE)

2552
2553
        return ResumeMemoryOccupationReqOutput()

2554
2555
2556
2557
2558
2559
2560
    def slow_down(self, recv_req: SlowDownReqInput):
        t = recv_req.forward_sleep_time
        if t is not None and t <= 0:
            t = None
        self.forward_sleep_time = t
        return SlowDownReqOutput()

2561
    def profile(self, recv_req: ProfileReq):
2562
        if recv_req.type == ProfileReqType.START_PROFILE:
2563
            if recv_req.profile_by_stage or recv_req.start_step:
2564
2565
                return self.init_profile(
                    recv_req.output_dir,
2566
                    recv_req.start_step,
2567
2568
2569
2570
2571
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2572
                    recv_req.profile_id,
2573
2574
2575
2576
                )
            else:
                self.init_profile(
                    recv_req.output_dir,
2577
                    recv_req.start_step,
2578
2579
2580
2581
2582
                    recv_req.num_steps,
                    recv_req.activities,
                    recv_req.with_stack,
                    recv_req.record_shapes,
                    recv_req.profile_by_stage,
2583
                    recv_req.profile_id,
2584
2585
                )
                return self.start_profile(True)
2586
        else:
2587
2588
            return self.stop_profile()

2589
    def init_profile(
2590
2591
        self,
        output_dir: Optional[str],
2592
        start_step: Optional[int],
2593
2594
        num_steps: Optional[int],
        activities: Optional[List[str]],
2595
2596
        with_stack: Optional[bool],
        record_shapes: Optional[bool],
2597
        profile_by_stage: bool,
2598
        profile_id: str,
2599
2600
    ) -> ProfileReqOutput:
        if self.profile_in_progress:
2601
2602
2603
2604
2605
            return ProfileReqOutput(
                success=False,
                message="Profiling is already in progress. Call /stop_profile first.",
            )

2606
2607
        self.profile_by_stage = profile_by_stage

2608
2609
2610
2611
2612
2613
        if output_dir is None:
            output_dir = os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp")
        if activities is None:
            activities = ["CPU", "GPU"]

        self.torch_profiler_output_dir = output_dir
2614
2615
        self.torch_profiler_with_stack = with_stack
        self.torch_profiler_record_shapes = record_shapes
2616
        self.profiler_activities = activities
2617
        self.profile_id = profile_id
2618

2619
2620
2621
        if start_step:
            self.profiler_start_forward_ct = max(start_step, self.forward_ct + 1)

2622
2623
2624
2625
2626
2627
2628
        if num_steps:
            self.profile_steps = num_steps
            if self.profile_by_stage:
                self.profiler_target_prefill_ct = num_steps
                self.profiler_target_decode_ct = num_steps
                self.profiler_prefill_ct = 0
                self.profiler_decode_ct = 0
2629
2630
2631
2632
            elif start_step:
                self.profiler_target_forward_ct = (
                    self.profiler_start_forward_ct + num_steps
                )
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
            else:
                self.profiler_target_forward_ct = self.forward_ct + num_steps
            # The caller will be notified when reaching profiler_target_forward_ct
        else:
            self.profiler_target_forward_ct = None

        return ProfileReqOutput(success=True, message="Succeeded")

    def start_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        stage_str = f" for {stage.__str__()}" if stage else ""
2645
        logger.info(
2646
            f"Profiling starts{stage_str}. Traces will be saved to: {self.torch_profiler_output_dir} (with profile id: {self.profile_id})",
2647
2648
        )

2649
2650
2651
2652
        activities = self.profiler_activities
        with_stack = self.torch_profiler_with_stack
        record_shapes = self.torch_profiler_record_shapes

2653
2654
2655
2656
2657
2658
2659
2660
        activity_map = {
            "CPU": torch.profiler.ProfilerActivity.CPU,
            "GPU": torch.profiler.ProfilerActivity.CUDA,
        }
        torchprof_activities = [
            activity_map[a] for a in activities if a in activity_map
        ]

2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
        if "RPD" in activities:
            from rpdTracerControl import rpdTracerControl

            rpdTracerControl.skipCreate()

            self.rpd_profile_path = os.path.join(
                self.torch_profiler_output_dir,
                "rpd-" + str(time.time()) + f"-TP-{self.tp_rank}" + ".trace.json.gz",
            )

            if self.tp_rank == 0:
                import sqlite3

                from rocpd.schema import RocpdSchema

                if os.path.exists("trace.rpd"):
                    os.unlink("trace.rpd")
                schema = RocpdSchema()
                connection = sqlite3.connect("trace.rpd")
                schema.writeSchema(connection)
                connection.commit()
                del connection
            torch.distributed.barrier(self.tp_cpu_group)

            self.rpd_profiler = rpdTracerControl()
            self.rpd_profiler.setPythonTrace(True)
            self.rpd_profiler.start()
            self.rpd_profiler.rangePush("", "rpd profile range", "")
            self.profile_in_progress = True
        elif torchprof_activities:
2691
2692
            self.torch_profiler = torch.profiler.profile(
                activities=torchprof_activities,
2693
2694
                with_stack=with_stack if with_stack is not None else True,
                record_shapes=record_shapes if record_shapes is not None else False,
2695
2696
            )
            self.torch_profiler.start()
2697
            self.profile_in_progress = True
2698
2699
2700

        if "MEM" in activities:
            torch.cuda.memory._record_memory_history(max_entries=100000)
2701
            self.profile_in_progress = True
2702

2703
2704
        if "CUDA_PROFILER" in activities:
            torch.cuda.cudart().cudaProfilerStart()
2705
            self.profile_in_progress = True
2706

2707
        return ProfileReqOutput(success=True, message="Succeeded")
2708

2709
2710
2711
2712
    def stop_profile(
        self, stage: Optional[ForwardMode] = None
    ) -> ProfileReqOutput | None:
        if not self.profile_in_progress:
2713
2714
2715
2716
            return ProfileReqOutput(
                success=False,
                message="Profiling is not in progress. Call /start_profile first.",
            )
2717

2718
2719
2720
        if not Path(self.torch_profiler_output_dir).exists():
            Path(self.torch_profiler_output_dir).mkdir(parents=True, exist_ok=True)

2721
2722
        stage_suffix = f"-{stage.__str__()}" if stage else ""
        logger.info("Stop profiling" + stage_suffix + "...")
2723
2724
2725
2726
2727
        if self.torch_profiler is not None:
            self.torch_profiler.stop()
            self.torch_profiler.export_chrome_trace(
                os.path.join(
                    self.torch_profiler_output_dir,
2728
                    self.profile_id
2729
2730
2731
                    + f"-TP-{self.tp_rank}"
                    + stage_suffix
                    + ".trace.json.gz",
2732
2733
                )
            )
2734
2735
2736
2737
2738
2739
            torch.distributed.barrier(self.tp_cpu_group)

        if self.rpd_profiler is not None:
            self.rpd_profiler.rangePop()
            self.rpd_profiler.stop()
            self.rpd_profiler.flush()
2740

2741
2742
2743
2744
2745
2746
2747
2748
2749
            torch.distributed.barrier(self.tp_cpu_group)
            if self.tp_rank == 0:
                from sglang.srt.utils import rpd_to_chrome_trace

                rpd_to_chrome_trace("trace.rpd", self.rpd_profile_path)
            self.rpd_profiler = None
            self.rpd_profiler_path = None

        if self.profiler_activities is not None and "MEM" in self.profiler_activities:
2750
            memory_profile_path = os.path.join(
2751
                self.torch_profiler_output_dir,
2752
2753
2754
2755
                str(time.time())
                + f"-TP-{self.tp_rank}-memory"
                + stage_suffix
                + ".pickle",
2756
2757
2758
2759
            )
            torch.cuda.memory._dump_snapshot(memory_profile_path)
            torch.cuda.memory._record_memory_history(enabled=None)

2760
2761
2762
        if "CUDA_PROFILER" in self.profiler_activities:
            torch.cuda.cudart().cudaProfilerStop()

2763
2764
2765
        logger.info(
            "Profiling done. Traces are saved to: %s",
            self.torch_profiler_output_dir,
2766
        )
2767
        self.torch_profiler = None
2768
        self.profile_in_progress = False
2769
        self.profiler_start_forward_ct = None
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791

        return ProfileReqOutput(success=True, message="Succeeded.")

    def _profile_batch_predicate(self, batch):
        if self.profile_by_stage:
            if batch.forward_mode.is_prefill():
                if self.profiler_prefill_ct == 0:
                    self.start_profile(batch.forward_mode)
                self.profiler_prefill_ct += 1
                if self.profiler_prefill_ct > self.profiler_target_prefill_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.EXTEND)
            elif batch.forward_mode.is_decode():
                if self.profiler_decode_ct == 0:
                    if self.profile_in_progress:
                        # force trace flush
                        self.stop_profile(ForwardMode.EXTEND)
                    self.start_profile(batch.forward_mode)
                self.profiler_decode_ct += 1
                if self.profiler_decode_ct > self.profiler_target_decode_ct:
                    if self.profile_in_progress:
                        self.stop_profile(stage=ForwardMode.DECODE)
2792
2793
            elif batch.forward_mode.is_idle():
                pass
2794
            else:
2795
                raise RuntimeError(f"unsupported profile stage: {batch.forward_mode}")
2796
2797
2798
2799
2800
2801
2802
        else:
            # Check profiler
            if (
                self.profiler_target_forward_ct
                and self.profiler_target_forward_ct <= self.forward_ct
            ):
                self.stop_profile()
2803
2804
2805
2806
2807
            if (
                self.profiler_start_forward_ct
                and self.profiler_start_forward_ct == self.forward_ct
            ):
                self.start_profile()
2808

2809
2810
    def expert_distribution_handle(self, recv_req: ExpertDistributionReq):
        if recv_req == ExpertDistributionReq.START_RECORD:
2811
            get_global_expert_distribution_recorder().start_record()
2812
        elif recv_req == ExpertDistributionReq.STOP_RECORD:
2813
            get_global_expert_distribution_recorder().stop_record()
2814
        elif recv_req == ExpertDistributionReq.DUMP_RECORD:
2815
            get_global_expert_distribution_recorder().dump_record()
2816
2817
        else:
            raise ValueError("Unrecognized ExpertDistributionReq value")
2818
        return ExpertDistributionReqOutput()
2819

2820
    def open_session(self, recv_req: OpenSessionReqInput):
2821
2822
2823
2824
        # handle error
        session_id = recv_req.session_id
        if session_id in self.sessions:
            logger.warning(f"session id {session_id} already exist, cannot open.")
2825
            return OpenSessionReqOutput(session_id, False)
2826
        elif session_id is None:
2827
            logger.warning("session id is None, cannot open.")
2828
            return OpenSessionReqOutput(session_id, False)
2829
2830
2831
2832
        else:
            self.sessions[session_id] = Session(
                recv_req.capacity_of_str_len, session_id
            )
2833
            return OpenSessionReqOutput(session_id, True)
2834
2835
2836
2837
2838
2839
2840
2841
2842

    def close_session(self, recv_req: CloseSessionReqInput):
        # handle error
        session_id = recv_req.session_id
        if session_id not in self.sessions:
            logger.warning(f"session id {session_id} does not exist, cannot delete.")
        else:
            del self.sessions[session_id]

2843
2844
    def get_print_prefix(self):
        prefix = ""
2845
2846
        if self.attn_dp_rank is not None:
            prefix += f" DP{self.attn_dp_rank}"
2847
2848
2849
2850
2851
2852
        if self.server_args.tp_size > 1:
            prefix += f" TP{self.tp_rank}"
        if self.pp_size > 1:
            prefix += f" PP{self.pp_rank}"
        return prefix

2853
2854
2855
2856
2857
2858
2859
    def _publish_kv_events(self):
        if self.enable_kv_cache_events:
            events = self.tree_cache.take_events()
            if events:
                batch = KVEventBatch(ts=time.time(), events=events)
                self.kv_event_publisher.publish(batch)

2860

2861
2862
2863
2864
def is_health_check_generate_req(recv_req):
    return getattr(recv_req, "rid", "").startswith("HEALTH_CHECK")


2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
def _export_static_state(model):
    return dict(
        buffers=[
            (name, buffer.detach().clone()) for name, buffer in model.named_buffers()
        ]
    )


def _import_static_state(model, static_params):
    self_named_buffers = dict(model.named_buffers())
    for name, tensor in static_params["buffers"]:
        self_named_buffers[name][...] = tensor


2879
2880
2881
2882
2883
def run_scheduler_process(
    server_args: ServerArgs,
    port_args: PortArgs,
    gpu_id: int,
    tp_rank: int,
2884
    pp_rank: int,
2885
    dp_rank: Optional[int],
2886
    pipe_writer,
2887
):
2888
    # Generate the prefix
2889
2890
2891
2892
2893
2894
2895
    prefix = ""
    if dp_rank is not None:
        prefix += f" DP{dp_rank}"
    if server_args.tp_size > 1:
        prefix += f" TP{tp_rank}"
    if server_args.pp_size > 1:
        prefix += f" PP{pp_rank}"
2896

2897
    # Config the process
2898
    kill_itself_when_parent_died()
2899
    setproctitle.setproctitle(f"sglang::scheduler{prefix.replace(' ', '_')}")
2900
    faulthandler.enable()
2901
    parent_process = psutil.Process().parent()
2902

2903
2904
2905
    # [For Router] if env var "SGLANG_DP_RANK" exist, set dp_rank to the value of the env var
    if dp_rank is None and "SGLANG_DP_RANK" in os.environ:
        dp_rank = int(os.environ["SGLANG_DP_RANK"])
2906

Wang Ran (汪然)'s avatar
Wang Ran (汪然) committed
2907
    # Configure the logger
2908
    configure_logger(server_args, prefix=prefix)
2909
    suppress_other_loggers()
2910

2911
    # Set cpu affinity to this gpu process
2912
2913
2914
    if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
        set_gpu_proc_affinity(server_args.tp_size, server_args.nnodes, gpu_id)

2915
2916
2917
2918
    embedding_cache_size = 100
    if "SGLANG_VLM_CACHE_SIZE_MB" in os.environ:
        embedding_cache_size = int(os.environ["SGLANG_VLM_CACHE_SIZE_MB"])
    init_embedding_cache(embedding_cache_size * 1024 * 1024)
2919
    # Create a scheduler and run the event loop
2920
    try:
2921
        scheduler = Scheduler(server_args, port_args, gpu_id, tp_rank, pp_rank, dp_rank)
2922
        pipe_writer.send(
Mick's avatar
Mick committed
2923
2924
2925
2926
2927
            {
                "status": "ready",
                "max_total_num_tokens": scheduler.max_total_num_tokens,
                "max_req_input_len": scheduler.max_req_input_len,
            }
2928
        )
Byron Hsu's avatar
Byron Hsu committed
2929
2930
2931
        disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode

        if disaggregation_mode == DisaggregationMode.NULL:
2932
2933
2934
            if server_args.pp_size > 1:
                scheduler.event_loop_pp()
            elif scheduler.enable_overlap:
Byron Hsu's avatar
Byron Hsu committed
2935
2936
2937
2938
                scheduler.event_loop_overlap()
            else:
                scheduler.event_loop_normal()
        elif disaggregation_mode == DisaggregationMode.PREFILL:
2939
2940
2941
2942
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_prefill()
            else:
                scheduler.event_loop_normal_disagg_prefill()
2943

Byron Hsu's avatar
Byron Hsu committed
2944
        elif disaggregation_mode == DisaggregationMode.DECODE:
2945
2946
2947
2948
            if scheduler.enable_overlap:
                scheduler.event_loop_overlap_disagg_decode()
            else:
                scheduler.event_loop_normal_disagg_decode()
Byron Hsu's avatar
Byron Hsu committed
2949

2950
    except Exception:
2951
2952
2953
        traceback = get_exception_traceback()
        logger.error(f"Scheduler hit an exception: {traceback}")
        parent_process.send_signal(signal.SIGQUIT)