tpu_model_runner.py 91.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import bisect
4
import gc
5
import time
6
from typing import TYPE_CHECKING, Any, cast
7
8
9
10
11
from unittest.mock import patch

import numpy as np
import torch
import torch.nn as nn
12

13
# TPU XLA related
14
import torch_xla
15
import torch_xla.core.xla_model as xm
16
import torch_xla.distributed.spmd as xs
17
18
import torch_xla.runtime as xr

19
import vllm.envs as envs
20
from vllm.attention import Attention
21
from vllm.attention.backends.abstract import AttentionType
22
from vllm.attention.layer import MLAAttention
23
from vllm.attention.layers.chunked_local_attention import ChunkedLocalAttention
24
from vllm.compilation.wrapper import TorchCompileWithNoGuardsWrapper
25
26
27
28
29
30
31
from vllm.config import (
    ParallelConfig,
    VllmConfig,
    get_layers_from_vllm_config,
    update_config,
)
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group
32
from vllm.distributed.kv_transfer.kv_connector.utils import copy_kv_blocks
33
from vllm.forward_context import set_forward_context
34
from vllm.logger import init_logger
35
from vllm.lora.layers import BaseLayerWithLoRA
36
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
37
from vllm.model_executor.model_loader import get_model_loader
38
from vllm.model_executor.model_loader.tpu import TPUModelLoader
39
40
41
42
from vllm.model_executor.models.interfaces import (
    SupportsMultiModal,
    supports_transcription,
)
43
from vllm.model_executor.models.interfaces_base import (
44
45
46
    is_pooling_model,
    is_text_generation_model,
)
47
from vllm.multimodal import MULTIMODAL_REGISTRY
48
49
50
51
52
from vllm.multimodal.inputs import (
    BatchedTensorInputs,
    MultiModalKwargsItem,
    PlaceholderRange,
)
53
from vllm.multimodal.utils import group_mm_kwargs_by_modality
54
from vllm.sequence import IntermediateTensors
55
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
56
from vllm.utils.math_utils import cdiv, prev_power_of_2
57
from vllm.utils.platform_utils import is_pin_memory_available
58
59
60
61
62
63
64
65
66
67
68
from vllm.v1.attention.backends.pallas import (
    TPU_STR_DTYPE_TO_TORCH_DTYPE,
    PallasAttentionBackend,
    PallasMetadata,
    get_page_size_bytes,
)
from vllm.v1.kv_cache_interface import (
    AttentionSpec,
    FullAttentionSpec,
    KVCacheConfig,
    KVCacheSpec,
69
    MLAAttentionSpec,
70
71
72
73
74
75
76
77
    SlidingWindowSpec,
)
from vllm.v1.outputs import (
    EMPTY_MODEL_RUNNER_OUTPUT,
    LogprobsLists,
    LogprobsTensors,
    ModelRunnerOutput,
)
78
79
from vllm.v1.sample.tpu.metadata import TPUSupportedSamplingMetadata
from vllm.v1.sample.tpu.sampler import Sampler as TPUSampler
80
from vllm.v1.worker.kv_connector_model_runner_mixin import (
81
82
83
    KVConnectorModelRunnerMixin,
    KVConnectorOutput,
)
84
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
85
from vllm.v1.worker.tpu_input_batch import CachedRequestState, InputBatch
86

87
88
89
90
91
92
from .utils import (
    MultiModalBudget,
    add_kv_sharing_layers_to_kv_cache_groups,
    bind_kv_cache,
    sanity_check_mm_encoder_outputs,
)
93

94
if TYPE_CHECKING:
95
    from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
96
97
98

logger = init_logger(__name__)

99
INVALID_TOKEN_ID = -1
100
101
# Smallest output size
MIN_NUM_SEQS = 8
102
103


104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#########################################################
# Ways to avoid recompilation
#########################################################
#
# The model executor has two primary components:
# 1. preparing the model and sampler inputs
# 2. executing the model and sampler.
# The core idea is to avoid any TPU computation during input preparation. For
# better compilation tracking and increased flexibility, the model execution and
# sampler are divided into several distinct components.
#
# Below are the detailed steps:
#
# Step 1
# It is recommended to avoid TPU operations when preparing the model and sampler
# inputs. CPU tensors can be prepared and transferred to the XLA device using
# cpu_tensor.to(xla_device), which only triggers CPU to TPU transfers and avoids
# compilation.
#
# Step 2
# The TPU execution should be decomposed into subgraphs (4 at the moment):
# 1. the main model
# 2. selecting hidden states for each request
# 3. sampler
# 4. encoder.
# Each subgraph should be decorated in a torch.compile. This is used to make
# sure that we have the same subgraph topology in both dummy_run and
# xecute_model. The results from these subgraphs should either be passed to
# other subgraphs, or transferred from TPU to CPU using xla_tensor.cpu() for
# subsequent processing on the CPU.
#
# Step 3
# The dummy_run should be comprehensive, ensuring all potential input shapes and
# branch predictions are included as subgraph inputs to facilitate
# pre-compilation.
139
class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
140
141
142
143
    def __init__(
        self,
        vllm_config: VllmConfig,
        device: torch.device,
144
        original_parallel_config: ParallelConfig | None = None,
145
146
147
148
149
150
151
    ):
        self.vllm_config = vllm_config
        self.model_config = vllm_config.model_config
        self.cache_config = vllm_config.cache_config
        self.lora_config = vllm_config.lora_config
        self.load_config = vllm_config.load_config
        self.parallel_config = vllm_config.parallel_config
152
        self.original_parallel_config = original_parallel_config
153
154
155
156
157
158
159
160
161
162
        self.scheduler_config = vllm_config.scheduler_config
        self.speculative_config = vllm_config.speculative_config
        self.observability_config = vllm_config.observability_config
        self.device_config = vllm_config.device_config

        model_config = self.model_config
        cache_config = self.cache_config
        scheduler_config = self.scheduler_config
        parallel_config = self.parallel_config
        self.device = device
163
        self.check_recompilation = envs.VLLM_XLA_CHECK_RECOMPILATION
164

165
166
167
168
169
170
        # SPMD Related
        self.use_spmd = envs.VLLM_XLA_USE_SPMD
        if self.use_spmd:
            num_devices = xr.global_runtime_device_count()
            mesh_shape = (num_devices, 1)
            device_ids = np.array(range(num_devices))
171
            self.mesh = xs.Mesh(device_ids, mesh_shape, ("x", "y"))
172

173
        self.enforce_eager = model_config.enforce_eager
174
175
176
177

        self.num_xla_graphs = 0
        self._update_num_xla_graphs("init")

178
179
        self.pin_memory = is_pin_memory_available()
        self.dtype = self.model_config.dtype
180
        if cache_config.cache_dtype == "auto":
181
182
            model_dtype = self.dtype
            if isinstance(model_dtype, str):
183
                self.kv_cache_dtype = TPU_STR_DTYPE_TO_TORCH_DTYPE[model_dtype]
184
185
            else:
                self.kv_cache_dtype = model_dtype
186
        else:
187
            self.kv_cache_dtype = TPU_STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
188
        self._hidden_states_dtype = self.dtype
189
190
191
192

        self.sliding_window = model_config.get_sliding_window()
        self.block_size = cache_config.block_size
        self.max_model_len = model_config.max_model_len
193
        self.most_model_len = envs.VLLM_TPU_MOST_MODEL_LEN
194
        self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
195
196
197
198
199
        self.num_blocks_per_most_len_req = (
            cdiv(self.most_model_len, self.block_size)
            if self.most_model_len is not None
            else None
        )
200
201
        # InputBatch needs to work with sampling tensors greater than padding
        # to avoid dynamic shapes. Also, avoid suboptimal alignment.
202
        self.max_num_reqs = max(scheduler_config.max_num_seqs, MIN_NUM_SEQS)
203
204
205
        self.num_tokens_paddings = _get_token_paddings(
            min_token_size=16,
            max_token_size=scheduler_config.max_num_batched_tokens,
206
207
            padding_gap=envs.VLLM_TPU_BUCKET_PADDING_GAP,
        )
208
209
210
        # In case `max_num_tokens < max(num_tokens_paddings)` use the actual
        # padded max value to pre-allocate data structures and pre-compile.
        self.max_num_tokens = self.num_tokens_paddings[-1]
211
212
213

        # Model-related.
        self.num_attn_layers = model_config.get_num_layers_by_block_type(
214
            parallel_config, "attention"
215
216
        )
        self.num_query_heads = model_config.get_num_attention_heads(parallel_config)
217
218
219
        self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
        self.head_size = model_config.get_head_size()
        self.hidden_size = model_config.get_hidden_size()
220
        self.vocab_size = model_config.get_vocab_size()
221

222
223
224
        # Multi-modal data support
        self.mm_registry = MULTIMODAL_REGISTRY
        self.uses_mrope = model_config.uses_mrope
225
        self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
226
227
            model_config
        )
228
229
230
        # TODO: Support M-RoPE (e.g, Qwen2-VL)
        assert not self.uses_mrope, "TPU does not support M-RoPE yet."

231
232
233
234
235
236
237
238
239
240
        self._num_slices_per_kv_cache_update_block = (
            _get_num_slices_per_kv_cache_update_block(
                get_page_size_bytes(
                    block_size=self.block_size,
                    num_kv_heads=self.num_kv_heads,
                    head_size=self.head_size,
                    kv_cache_dtype=self.kv_cache_dtype,
                )
            )
        )
241

242
        # Lazy initialization
243
        self.model: nn.Module  # Set after load_model
244
        self.kv_caches: list[torch.Tensor] = []
245
246
        # mm_hash -> encoder_output
        self.encoder_cache: dict[str, torch.Tensor] = {}
247
248
249

        # Request states.
        self.requests: dict[str, CachedRequestState] = {}
250

251
252
253
254
255
256
257
258
        # Initialize input batch early to avoid AttributeError in _update_states
        self.input_batch = InputBatch(
            max_num_reqs=self.max_num_reqs,
            max_model_len=self.max_model_len,
            max_num_batched_tokens=self.max_num_tokens,
            device=self.device,
            pin_memory=self.pin_memory,
            vocab_size=self.model_config.get_vocab_size(),
259
            block_sizes=[self.block_size],
260
            kernel_block_sizes=[self.cache_config.block_size],
261
262
        )

263
264
265
        # Cached torch/numpy tensor
        # The pytorch tensor and numpy array share the same buffer.
        # Sometimes the numpy op is faster so we create both.
266
267
268
        self.input_ids_cpu = torch.zeros(
            self.max_num_tokens, dtype=torch.int32, device="cpu"
        )
269

270
271
272
        self.positions_cpu = torch.zeros(
            self.max_num_tokens, dtype=torch.int32, device="cpu"
        )
273
274
        self.positions_np = self.positions_cpu.numpy()
        self.block_table_cpu = torch.zeros(
275
            (self.max_num_reqs, self.max_num_blocks_per_req),
276
            dtype=torch.int32,
277
278
            device="cpu",
        )
279
        # adjust num_reqs to avoid SMEM OOM.
280
281
282
283
284
285
286
287
288
289
        self.num_reqs_most_model_len = (
            min(
                PallasAttentionBackend.get_max_num_seqs(
                    self.most_model_len, self.block_size
                ),
                self.max_num_reqs,
            )
            if self.most_model_len is not None
            else None
        )
290
        self.num_reqs_max_model_len = min(
291
292
293
294
295
296
297
298
299
300
301
            PallasAttentionBackend.get_max_num_seqs(
                self.max_model_len, self.block_size
            ),
            self.max_num_reqs,
        )
        self.query_start_loc_cpu = torch.zeros(
            self.max_num_tokens + 1,
            dtype=torch.int32,
            device="cpu",
            pin_memory=self.pin_memory,
        )
302
303
        self.query_start_loc_np = self.query_start_loc_cpu.numpy()

304
305
306
307
308
309
        self.seq_lens_cpu = torch.zeros(
            self.max_num_tokens,
            dtype=torch.int32,
            device="cpu",
            pin_memory=self.pin_memory,
        )
310
        self.seq_lens_np = self.seq_lens_cpu.numpy()
311

312
313
        # Only relevant for multimodal models
        if self.supports_mm_inputs:
314
315
316
317
318
319
            self.is_mm_embed_cpu = torch.zeros(
                self.max_num_tokens,
                dtype=torch.bool,
                device="cpu",
                pin_memory=self.pin_memory,
            )
320

321
322
        # Range tensor with values [0 .. self.max_num_tokens - 1].
        # Used to initialize positions / context_lens / seq_lens
323
324
        # Keep in int64 to avoid overflow with long context
        self.arange_np = np.arange(self.max_num_tokens, dtype=np.int64)
325
        self.num_reqs_paddings = _get_req_paddings(
326
327
            min_req_size=MIN_NUM_SEQS, max_req_size=self.max_num_reqs
        )
328

329
330
331
332
333
334
        # Layer pairings for cross-layer KV sharing.
        # If an Attention layer `layer_name` is in the keys of this dict, it
        # means this layer will perform attention using the keys and values
        # from the KV cache of `shared_kv_cache_layers[layer_name]`.
        self.shared_kv_cache_layers: dict[str, str] = {}

335
336
337
338
339
        # tensors for structured decoding
        self.grammar_bitmask_cpu = torch.zeros(
            (self.max_num_reqs, cdiv(self.vocab_size, 32)),
            dtype=torch.int32,
            device="cpu",
340
341
            pin_memory=self.pin_memory,
        )
342
343
344
345
        self.require_structured_out_cpu = torch.zeros(
            (self.max_num_reqs, 1),
            dtype=torch.bool,
            device="cpu",
346
347
            pin_memory=self.pin_memory,
        )
348
        self.structured_decode_arange = torch.arange(
349
350
            0, 32, device="cpu", pin_memory=self.pin_memory
        )
351

352
353
354
355
356
357
358
359
360
        self.mm_budget = (
            MultiModalBudget(
                self.model_config,
                self.scheduler_config,
                self.mm_registry,
            )
            if self.supports_mm_inputs
            else None
        )
361

362
363
364
365
366
        if not self.use_spmd:
            self.sample_from_logits_func = torch.compile(
                self.sample_from_logits,
                backend="openxla",
                fullgraph=True,
367
368
                dynamic=False,
            )
369
370
371
        else:
            self.sample_from_logits_func = self.sample_from_logits

372
373
374
375
376
        # For passing scheduler_output between successive
        # execute_model() and sample_tokens() calls.
        self.scheduler_output: SchedulerOutput | None = None
        self.mm_embed_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None

377
378
379
380
    def reset_mm_cache(self) -> None:
        if self.mm_budget:
            self.mm_budget.reset_cache()

381
382
383
384
385
386
387
388
389
390
    def _update_num_xla_graphs(self, case_str):
        check_comp = self.check_recompilation and not self.enforce_eager
        if not check_comp:
            return

        total_cached_graphs = xr.get_num_cached_compilation_graph()
        new_compiled_graphs = total_cached_graphs - self.num_xla_graphs
        if new_compiled_graphs == 0:
            return

391
392
393
        logger.info(
            "Add new %d compiled XLA graphs due to %s", new_compiled_graphs, case_str
        )
394
395
396
397
398
399
400
401
402
403
404
        self.num_xla_graphs += new_compiled_graphs

    def _verify_num_xla_graphs(self, case_str):
        check_comp = self.check_recompilation and not self.enforce_eager
        if not check_comp:
            return

        curr_cached_graph = xr.get_num_cached_compilation_graph()
        assert self.num_xla_graphs == curr_cached_graph, (
            "Recompilation after warm up is detected during {}."
            " num_xla_graphs = {} curr_cached_graph = {}".format(
405
406
407
                case_str, self.num_xla_graphs, curr_cached_graph
            )
        )
408

409
410
411
412
413
414
415
416
    def _update_states(self, scheduler_output: "SchedulerOutput") -> bool:
        """Update the cached states and the persistent batch with the scheduler
        output.

        The updated states are used by the `_prepare_inputs` function to create
        the input GPU tensors for the model.

        Returns:
417
            True if there is a new/resumed/paused/finished request.
418
419
420
421
422
423
424
425
426
427
428
429
            If False, we can skip copying SamplingMetadata to the GPU.
        """
        # Remove finished requests from the cached states.
        for req_id in scheduler_output.finished_req_ids:
            self.requests.pop(req_id, None)

        # Remove the finished requests from the persistent batch.
        # NOTE(woosuk): There could be an edge case where finished_req_ids and
        # scheduled_req_ids overlap. This happens when a request is aborted and
        # then resubmitted with the same ID. In this case, we treat them as two
        # distinct requests - clearing the cached states for the first request
        # and handling the second as a new request.
430
        removed_req_indices: list[int] = []
431
432
433
434
435
        for req_id in scheduler_output.finished_req_ids:
            req_index = self.input_batch.remove_request(req_id)
            if req_index is not None:
                removed_req_indices.append(req_index)

436
        # Free the cached encoder outputs.
437
438
        for mm_hash in scheduler_output.free_encoder_mm_hashes:
            self.encoder_cache.pop(mm_hash, None)
439

440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
        # Remove the unscheduled requests from the persistent batch.
        # NOTE(woosuk): The unscheduled requests are either preempted requests
        # or running requests that are not scheduled in this step. We remove
        # them from the persistent batch but keep their cached states since
        # they will be scheduled again sometime in the future.
        scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys()
        cached_req_ids = self.input_batch.req_id_to_index.keys()
        unscheduled_req_ids = cached_req_ids - scheduled_req_ids
        # NOTE(woosuk): The persistent batch optimization assumes that
        # consecutive batches contain mostly the same requests. If batches
        # have low request overlap (e.g., alternating between two distinct
        # sets of requests), this optimization becomes very inefficient.
        for req_id in unscheduled_req_ids:
            req_index = self.input_batch.remove_request(req_id)
            assert req_index is not None
            removed_req_indices.append(req_index)

457
        req_ids_to_add: list[str] = []
458
459
        # Add new requests to the cached states.
        for new_req_data in scheduler_output.scheduled_new_reqs:
460
            assert new_req_data.sampling_params is not None, (
461
                "Pooling is not supported in TPU yet"
462
            )
463
464
465
466
467
468
            req_id = new_req_data.req_id
            sampling_params = new_req_data.sampling_params

            self.requests[req_id] = CachedRequestState(
                req_id=req_id,
                prompt_token_ids=new_req_data.prompt_token_ids,
469
                prompt_embeds=new_req_data.prompt_embeds,
470
                mm_features=new_req_data.mm_features,
471
                sampling_params=sampling_params,
472
                pooling_params=None,
473
                generator=None,
474
475
476
477
478
479
480
481
482
                block_ids=new_req_data.block_ids,
                num_computed_tokens=new_req_data.num_computed_tokens,
                output_token_ids=[],
                lora_request=new_req_data.lora_request,
            )

            req_ids_to_add.append(req_id)

        # Update the states of the running/resumed requests.
483
484
        req_data = scheduler_output.scheduled_cached_reqs
        for i, req_id in enumerate(req_data.req_ids):
485
            req_state = self.requests[req_id]
486
487
            num_computed_tokens = req_data.num_computed_tokens[i]
            new_block_ids = req_data.new_block_ids[i]
488
            resumed_from_preemption = req_id in req_data.resumed_req_ids
489
490

            # Update the cached states.
491
492
            req_state.num_computed_tokens = num_computed_tokens
            if not resumed_from_preemption:
493
494
                if new_block_ids is not None:
                    # Append the new blocks to the existing block IDs.
495
                    for block_ids, new_ids in zip(req_state.block_ids, new_block_ids):
496
                        block_ids.extend(new_ids)
497
            else:
498
                assert new_block_ids is not None
499
500
                # The request is resumed from preemption.
                # Replace the existing block IDs with the new ones.
501
                req_state.block_ids = new_block_ids
502
503
504
505
506
507
508
509
510
511

            req_index = self.input_batch.req_id_to_index.get(req_id)
            if req_index is None:
                # The request is not in the persistent batch.
                # The request was either preempted and resumed later, or was not
                # scheduled in the previous step and needs to be added again.
                req_ids_to_add.append(req_id)
                continue

            # Update the persistent batch.
512
            self.input_batch.num_computed_tokens_cpu[req_index] = num_computed_tokens
513
            if new_block_ids is not None:
514
                self.input_batch.block_table.append_row(new_block_ids, req_index)
515
516
517
518
519
520

        # Add the new or resumed requests to the persistent batch.
        # The smaller empty indices are filled first.
        removed_req_indices = sorted(removed_req_indices, reverse=True)
        for req_id in req_ids_to_add:
            req_state = self.requests[req_id]
521
522
            # Fill the empty index or append to the end
            req_index = removed_req_indices.pop() if removed_req_indices else None
523
524
525
526
527
            self.input_batch.add_request(req_state, req_index)

        # Condense the batched states if there are empty indices.
        if removed_req_indices:
            self.input_batch.condense(removed_req_indices)
528

529
530
531
532
533
        return len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0

    def get_model(self) -> nn.Module:
        return self.model

534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
    def get_supported_generation_tasks(self) -> list[GenerationTask]:
        model = self.get_model()
        supported_tasks = list[GenerationTask]()

        if is_text_generation_model(model):
            supported_tasks.append("generate")

        if supports_transcription(model):
            if model.supports_transcription_only:
                return ["transcription"]

            supported_tasks.append("transcription")

        return supported_tasks

549
550
551
552
553
    def get_supported_pooling_tasks(self) -> list[PoolingTask]:
        model = self.get_model()
        if not is_pooling_model(model):
            return []

554
        return list(model.pooler.get_supported_tasks())
555

556
557
558
559
560
561
562
563
564
565
    def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
        tasks = list[SupportedTask]()

        if self.model_config.runner_type == "generate":
            tasks.extend(self.get_supported_generation_tasks())
        if self.model_config.runner_type == "pooling":
            tasks.extend(self.get_supported_pooling_tasks())

        return tuple(tasks)

566
    def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
567
        """
568
        Generates the KVCacheSpec by parsing the kv cache format from each
569
570
        Attention module in the static forward context.
        Returns:
571
            KVCacheSpec: A dictionary mapping layer names to their KV cache
572
573
574
            format. Layers that do not need KV cache are not included.
        """

575
576
577
578
        layers = get_layers_from_vllm_config(
            self.vllm_config,
            AttentionLayerBase,  # type: ignore[type-abstract]
        )
579
        block_size = self.vllm_config.cache_config.block_size
580
581
        cache_dtype_str = self.vllm_config.cache_config.cache_dtype

582
        kv_cache_spec: dict[str, KVCacheSpec] = {}
583
        for layer_name, attn_module in layers.items():
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
            # Classic Attention path
            if isinstance(attn_module, Attention):
                if (
                    kv_tgt_layer := attn_module.kv_sharing_target_layer_name
                ) is not None:
                    # The layer doesn't need its own KV cache and will use that of
                    # the target layer. We skip creating a KVCacheSpec for it, so
                    # that KV cache management logic will act as this layer does
                    # not exist, and doesn't allocate KV cache for the layer. This
                    # enables the memory saving of cross-layer kv sharing, allowing
                    # a given amount of memory to accommodate longer context lengths
                    # or enable more requests to be processed simultaneously.
                    self.shared_kv_cache_layers[layer_name] = kv_tgt_layer
                    continue

                if attn_module.attn_type == AttentionType.DECODER:
                    if isinstance(attn_module, ChunkedLocalAttention):
                        logger.warning_once(
                            "Using irope in Pallas is not supported yet, it "
                            "will fall back to global attention for long context."
                        )
                    if attn_module.sliding_window is not None:
                        kv_cache_spec[layer_name] = SlidingWindowSpec(
                            block_size=block_size,
                            num_kv_heads=attn_module.num_kv_heads,
                            head_size=attn_module.head_size,
                            dtype=self.kv_cache_dtype,
                            sliding_window=attn_module.sliding_window,
                        )
                    else:
                        kv_cache_spec[layer_name] = FullAttentionSpec(
                            block_size=block_size,
                            num_kv_heads=attn_module.num_kv_heads,
                            head_size=attn_module.head_size,
                            dtype=self.kv_cache_dtype,
                        )
                elif attn_module.attn_type in (
                    AttentionType.ENCODER,
                    AttentionType.ENCODER_ONLY,
                ):
                    # encoder-only attention does not need KV cache.
                    continue
                elif attn_module.attn_type == AttentionType.ENCODER_DECODER:
                    raise NotImplementedError
628
                else:
629
630
631
632
633
634
635
636
637
638
639
640
                    raise ValueError(f"Unknown attention type: {attn_module.attn_type}")
            # MLAAttention path
            elif isinstance(attn_module, MLAAttention):
                if layer_name in kv_cache_spec:
                    continue
                kv_cache_spec[layer_name] = MLAAttentionSpec(
                    block_size=block_size,
                    num_kv_heads=1,
                    head_size=attn_module.head_size,
                    dtype=self.kv_cache_dtype,
                    cache_dtype_str=cache_dtype_str,
                )
641
            else:
642
                continue
643
644
645

        return kv_cache_spec

646
647
648
    def _get_slot_mapping_metadata(
        self, num_reqs, num_scheduled_tokens_per_req
    ) -> np.ndarray:
649
650
651
652
653
654
655
656
657
658
659
660
        """
        Computes metadata for mapping slots to blocks in the key-value (KV)
        cache for a batch of requests.

        This function determines, for each request in the batch, how the
        scheduled tokens are distributed across memory blocks, and generates
        metadata needed to map slices of tokens to their corresponding positions
        in the KV cache.

        Args:
            num_reqs (int): Number of requests in the current batch.
            num_scheduled_tokens_per_req (int or np.ndarray): Number of tokens
661
                to be scheduled for each request.
662
663
664

        Returns:
            np.ndarray: A 2D array of shape (total_block_len, 3), where each row
665
                contains:
666
                - kv_cache_start_index (int): The starting index in the KV cache
667
                  for the corresponding slice.
668
                - new_kv_start_index (int): The starting index in the new KV
669
                  cache for the corresponding slice.
670
671
672
                - slice_len (int): The length of the slice.
        """
        slices_start = self.input_batch.num_computed_tokens_cpu[:num_reqs]
673
674
675
676
        slices_end = (
            self.input_batch.num_computed_tokens_cpu[:num_reqs]
            + num_scheduled_tokens_per_req
        )
677
678
679
680
        local_block_start_idx = slices_start // self.block_size
        local_block_end_idx = (slices_end - 1) // self.block_size
        no_repeat_req_indices = self.arange_np[:num_reqs]
        global_block_start_idx = (
681
682
            no_repeat_req_indices * self.max_num_blocks_per_req + local_block_start_idx
        )
683
684
685
686
687
688
689
        block_lens = local_block_end_idx - local_block_start_idx + 1
        global_block_start_idx = np.repeat(global_block_start_idx, block_lens)
        slice_arange = np.concatenate([self.arange_np[:n] for n in block_lens])
        global_block_indices = global_block_start_idx + slice_arange
        block_table_cpu = self.input_batch.block_table[0].get_cpu_tensor()
        block_numbers = block_table_cpu.flatten()[global_block_indices].numpy()
        total_block_len = np.sum(block_lens)
690
691
692
        slot_mapping_slices = np.repeat(
            np.array([[0, self.block_size]], dtype=np.int32), total_block_len, axis=0
        )
693
694
695
        cu_block_lens = np.zeros(len(block_lens) + 1, dtype=np.int32)
        np.cumsum(block_lens, out=cu_block_lens[1:])
        for req_idx in range(num_reqs):
696
697
698
699
700
701
            slot_mapping_slices[cu_block_lens[req_idx]][0] = (
                slices_start[req_idx] % self.block_size
            )
            slot_mapping_slices[cu_block_lens[req_idx + 1] - 1][1] = (
                slices_end[req_idx] - 1
            ) % self.block_size + 1
702
703
704
        slice_lens = slot_mapping_slices[:, 1] - slot_mapping_slices[:, 0]
        cu_slices_lens = np.zeros(len(slice_lens) + 1, dtype=np.int32)
        np.cumsum(slice_lens, out=cu_slices_lens[1:])
705
706
707
        kv_cache_start_indices = slot_mapping_slices[:, 0] + (
            block_numbers * self.block_size
        )
708
709
        new_kv_start_indices = cu_slices_lens[:-1]
        slot_mapping_metadata = np.stack(
710
711
            [kv_cache_start_indices, new_kv_start_indices, slice_lens], axis=1
        )
712
713
        return slot_mapping_metadata

714
    def _prepare_inputs(self, scheduler_output: "SchedulerOutput", start_index: int):
715
        assert scheduler_output.total_num_scheduled_tokens > 0
716
717
        num_reqs = self.input_batch.num_reqs
        assert num_reqs > 0
718
        assert start_index < num_reqs
719

720
        # Get the number of scheduled tokens for each request.
721
        use_max_model_len = self.most_model_len is None
722
723
        num_scheduled_tokens_per_req = []
        max_num_scheduled_tokens_all_reqs = 0
724
725
726
727
728
        end_index = start_index

        # Use either most_model_len or max_model_len depending on request size.
        for i in range(start_index, num_reqs):
            req_id = self.input_batch.req_ids[i]
729
            assert req_id is not None
730
            num_tokens = scheduler_output.num_scheduled_tokens[req_id]
731
732
733
734
735
            if (
                not use_max_model_len
                and self.most_model_len is not None
                and num_tokens > self.most_model_len
            ):
736
                use_max_model_len = True
737
            num_scheduled_tokens_per_req.append(num_tokens)
738
739
        if use_max_model_len:
            if len(num_scheduled_tokens_per_req) > self.num_reqs_max_model_len:
740
741
742
                num_scheduled_tokens_per_req = num_scheduled_tokens_per_req[
                    : self.num_reqs_max_model_len
                ]
743
744
745
746
                end_index = start_index + self.num_reqs_max_model_len
            else:
                end_index = num_reqs
        else:
747
            assert self.num_reqs_most_model_len is not None
748
749
750
751
            if len(num_scheduled_tokens_per_req) > self.num_reqs_most_model_len:
                num_scheduled_tokens_per_req = num_scheduled_tokens_per_req[
                    : self.num_reqs_most_model_len
                ]
752
753
754
755
                end_index = start_index + self.num_reqs_most_model_len
            else:
                end_index = num_reqs
        max_num_scheduled_tokens_all_reqs = max(num_scheduled_tokens_per_req)
756
757
758
        num_scheduled_tokens_per_req = np.array(
            num_scheduled_tokens_per_req, dtype=np.int32
        )
759
        total_num_scheduled_tokens = sum(num_scheduled_tokens_per_req)
760
761
        assert max_num_scheduled_tokens_all_reqs > 0

762
763
        num_reqs = len(num_scheduled_tokens_per_req)

764
765
766
        # Get request indices.
        # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
        # For each scheduled token, what are the corresponding req index.
767
        req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens_per_req)
768
769
770
771
772

        # Get batched arange.
        # E.g., [2, 5, 3] -> [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
        # For each scheduled token, what is its position in corresponding req.
        arange = np.concatenate(
773
774
            [self.arange_np[:n] for n in num_scheduled_tokens_per_req]
        )
775
776
777

        # Get positions.
        positions_np = self.positions_np[:total_num_scheduled_tokens]
778
779
780
781
782
        np.add(
            self.input_batch.num_computed_tokens_cpu[req_indices],
            arange,
            out=positions_np,
        )
783
784
785
786
787

        # Get token indices.
        # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
        # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
        # where M is the max_model_len.
788
789
790
        token_indices = (
            positions_np + req_indices * self.input_batch.token_ids_cpu.shape[1]
        )
791
792
793
794

        # NOTE(woosuk): We use torch.index_select instead of np.take here
        # because torch.index_select is much faster than np.take for large
        # tensors.
795
796
797
798
799
800
        torch.index_select(
            self.input_batch.token_ids_cpu_tensor.flatten(),
            0,
            torch.from_numpy(token_indices),
            out=self.input_ids_cpu[:total_num_scheduled_tokens],
        )
801
802
803

        # Prepare the attention metadata.
        self.query_start_loc_np[0] = 0
804
805
806
807
        np.cumsum(
            num_scheduled_tokens_per_req, out=self.query_start_loc_np[1 : num_reqs + 1]
        )
        self.query_start_loc_np[num_reqs + 1 :] = 1
808
809

        self.seq_lens_np[:num_reqs] = (
810
811
812
            self.input_batch.num_computed_tokens_cpu[:num_reqs]
            + num_scheduled_tokens_per_req
        )
813
814

        # Do the padding and copy the tensors to the TPU.
815
        padded_total_num_scheduled_tokens = _get_padded_token_len(
816
817
            self.num_tokens_paddings, total_num_scheduled_tokens
        )
818
819
        # Zero out to avoid spurious values from prev iteration (last cp chunk)
        self.input_ids_cpu[
820
821
822
823
824
825
826
827
            total_num_scheduled_tokens:padded_total_num_scheduled_tokens
        ] = 0
        self.input_ids = self.input_ids_cpu[:padded_total_num_scheduled_tokens].to(
            self.device
        )
        self.position_ids = self.positions_cpu[:padded_total_num_scheduled_tokens].to(
            self.device
        )
828
        if use_max_model_len:
829
830
831
832
833
834
835
836
837
838
            block_tables = self.block_table_cpu[
                : self.num_reqs_max_model_len, : self.max_num_blocks_per_req
            ]
            block_tables[:num_reqs, : self.max_num_blocks_per_req] = (
                self.input_batch.block_table[0].get_cpu_tensor()[:num_reqs]
            )
            query_start_loc = self.query_start_loc_cpu[
                : self.num_reqs_max_model_len + 1
            ].to(self.device)
            seq_lens = self.seq_lens_cpu[: self.num_reqs_max_model_len].to(self.device)
839
        else:
840
            assert self.num_reqs_most_model_len is not None
841
842
843
844
845
846
847
848
849
850
851
852
            block_tables = self.block_table_cpu[
                : self.num_reqs_most_model_len, : self.num_blocks_per_most_len_req
            ]
            block_tables[:num_reqs, : self.num_blocks_per_most_len_req] = (
                self.input_batch.block_table[0].get_cpu_tensor()[
                    :num_reqs, : self.num_blocks_per_most_len_req
                ]
            )
            query_start_loc = self.query_start_loc_cpu[
                : self.num_reqs_most_model_len + 1
            ].to(self.device)
            seq_lens = self.seq_lens_cpu[: self.num_reqs_most_model_len].to(self.device)
853
        block_tables = block_tables.to(self.device)
854

855
        # Calculate the slot mapping
856
        slot_mapping_metadata = self._get_slot_mapping_metadata(
857
858
            num_reqs, num_scheduled_tokens_per_req
        )
859
        num_kv_update_slices = slot_mapping_metadata.shape[0]
860
        padded_num_slices = _get_padded_num_kv_cache_update_slices(
861
862
            padded_total_num_scheduled_tokens, self.max_num_reqs, self.block_size
        )
863
864
865
        slot_mapping_metadata = np.pad(
            slot_mapping_metadata,
            [[0, padded_num_slices - len(slot_mapping_metadata)], [0, 0]],
866
867
            constant_values=0,
        )
868
        slot_mapping_metadata = np.transpose(slot_mapping_metadata)
869
        slot_mapping_metadata = torch.tensor(slot_mapping_metadata, device=self.device)
870

871
872
873
874
875
        if self.lora_config is not None:
            # We need to respect padding when activating LoRA adapters
            padded_num_scheduled_tokens_per_req = np.copy(
                num_scheduled_tokens_per_req
            )  # Copying to avoid accidental state corruption bugs
876
            padded_num_scheduled_tokens_per_req[-1] += (
877
                padded_total_num_scheduled_tokens - total_num_scheduled_tokens
878
            )
879

880
            self.set_active_loras(self.input_batch, padded_num_scheduled_tokens_per_req)
881

882
        attn_metadata = PallasMetadata(
883
            slot_mapping=slot_mapping_metadata,
884
            block_tables=block_tables,
885
886
            context_lens=seq_lens,
            query_start_loc=query_start_loc,
887
888
889
890
891
            num_seqs=torch.tensor([num_reqs], dtype=torch.int32, device=self.device),
            num_kv_update_slices=torch.tensor(
                [num_kv_update_slices], dtype=torch.int32, device=self.device
            ),
            num_slices_per_kv_cache_update_block=self._num_slices_per_kv_cache_update_block,
892
        )
893
894
895
896
897
        # NOTE(woosuk): Due to chunked prefills, there can be at most 1 partial
        # request in the batch. While we should not sample any token from this
        # partial request, we do so for simplicity. We will ignore the sampled
        # token from the partial request.
        # TODO: Support prompt logprobs.
898
        padded_num_reqs = _get_padded_num_reqs_with_upper_limit(
899
900
            num_reqs, self.max_num_reqs
        )
901
902
        # Indices at which we sample (positions of last token in the sequence).
        # Padded to avoid recompiling when `num_reqs` varies.
903
        logits_indices = self.query_start_loc_cpu[1 : padded_num_reqs + 1] - 1
904
        logits_indices = logits_indices.to(self.device)
905

906
907
908
909
910
        if self.lora_config is not None:
            # We need to respect padding when activating LoRA adapters
            padded_num_scheduled_tokens_per_req = np.copy(
                num_scheduled_tokens_per_req
            )  # Copying to avoid accidental state corruption bugs
911
            padded_num_scheduled_tokens_per_req[-1] += (
912
                padded_total_num_scheduled_tokens - total_num_scheduled_tokens
913
            )
914

915
            self.set_active_loras(self.input_batch, padded_num_scheduled_tokens_per_req)
916

917
        layer_names = get_layers_from_vllm_config(self.vllm_config, Attention).keys()
918
        per_layer_attn_metadata = {
919
            layer_name: attn_metadata for layer_name in layer_names
920
        }
921
922
923
924
925
926
927
        return (
            per_layer_attn_metadata,
            logits_indices,
            padded_num_reqs,
            num_reqs,
            end_index,
        )
928

929
    def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):
930
931
932
933
934
        scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
        if not scheduled_encoder_inputs:
            return

        # Batch the multi-modal inputs.
935
        mm_kwargs = list[MultiModalKwargsItem]()
936
937
        # List of tuple (mm_hash, pos_info)
        mm_hashes_pos = list[tuple[str, PlaceholderRange]]()
938
939
        for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
            req_state = self.requests[req_id]
940
941

            for mm_input_id in encoder_input_ids:
942
                mm_feature = req_state.mm_features[mm_input_id]
943
944
                if mm_feature.data is None:
                    continue
945
946
947
                mm_hash = mm_feature.identifier
                mm_kwargs.append(mm_feature.data)
                mm_hashes_pos.append((mm_hash, mm_feature.mm_position))
948
949
950
951
952
953
954
955

        # Batch mm inputs as much as we can: if a request in the batch has
        # multiple modalities or a different modality than the previous one,
        # we process it separately to preserve item order.
        # FIXME(ywang96): This is a hacky way to deal with multiple modalities
        # in the same batch while still being able to benefit from batching
        # multimodal inputs. The proper solution should be reordering the
        # encoder outputs.
956
        model = cast(SupportsMultiModal, self.model)
957
        encoder_outputs = []
958
        for _, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
959
960
961
962
            mm_kwargs,
            device=self.device,
            pin_memory=self.pin_memory,
            merge_by_field_config=model.merge_by_field_config,
963
            multimodal_cpu_fields=model.multimodal_cpu_fields,
964
        ):
965
966
967
968
969
970
971
            # Run the encoder.
            # `curr_group_outputs` is either of the following:
            # 1. A tensor of shape (num_items, feature_size, hidden_size)
            # in case feature_size is fixed across all multimodal items.
            # 2. A list or tuple (length: num_items) of tensors, each of shape
            # (feature_size, hidden_size) in case the feature size is dynamic
            # depending on the input multimodal items.
972
            torch_xla.sync(wait=False)
973
            curr_group_outputs = model.embed_multimodal(**mm_kwargs_group)
974
            torch_xla.sync(wait=False)
975

976
977
            sanity_check_mm_encoder_outputs(
                curr_group_outputs,
978
                expected_num_items=num_items,
979
980
            )

981
982
983
984
985
986
            if isinstance(curr_group_outputs, torch.Tensor):
                encoder_outputs.append(curr_group_outputs)
            else:
                assert isinstance(curr_group_outputs, (list, tuple))
                for output in curr_group_outputs:
                    encoder_outputs.append(output)
987
988

        # Cache the encoder outputs.
989
990
991
        # NOTE (NickLucche) here we diverge from logic in other runners, as we
        # assume to only have whole mm items to process. Hence we avoid the
        # intrinsic dynamism that `scatter_mm_placeholders` introduces.
992
        for (mm_hash, pos_info), output in zip(mm_hashes_pos, encoder_outputs):
993
994
995
            assert pos_info.is_embed is None, (
                "Expected all positions to be contiguous and embeddings."
            )
996
            self.encoder_cache[mm_hash] = output
997
998

    def _gather_mm_embeddings(
999
1000
        self,
        scheduler_output: "SchedulerOutput",
1001
1002
1003
    ) -> tuple[list[torch.Tensor], torch.Tensor]:
        total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
        padded_total_num_scheduled_tokens = _get_padded_token_len(
1004
1005
            self.num_tokens_paddings, total_num_scheduled_tokens
        )
1006
1007
1008
1009
1010
1011

        is_mm_embed = self.is_mm_embed_cpu
        is_mm_embed[:padded_total_num_scheduled_tokens] = False
        mm_embeds = list[torch.Tensor]()
        req_start_idx = 0

1012
        for req_id in self.input_batch.req_ids:
1013
            num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
1014
1015
            req_state = self.requests[req_id]
            num_computed_tokens = req_state.num_computed_tokens
1016

1017
1018
1019
1020
            # TODO unroll loop and assume/enforce --disable_chunked_mm_input
            # NOTE (NickLucche) here we diverge from logic in other runners, as
            # we assume to only have whole mm items to process. Hence we avoid
            # the intrinsic dynamism that `gather_mm_placeholders` introduces.
1021
1022
            for mm_feature in req_state.mm_features:
                pos_info = mm_feature.mm_position
1023
1024
                start_pos = pos_info.offset
                num_encoder_tokens = pos_info.length
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036

                # The encoder output is needed if the two ranges overlap:
                # [num_computed_tokens,
                #  num_computed_tokens + num_scheduled_tokens) and
                # [start_pos, start_pos + num_encoder_tokens)
                if start_pos >= num_computed_tokens + num_scheduled_tokens:
                    # The encoder output is not needed in this step.
                    break
                if start_pos + num_encoder_tokens <= num_computed_tokens:
                    # The encoder output is already processed and stored
                    # in the decoder's KV cache.
                    continue
1037
1038
1039
1040
1041
1042
1043
1044

                start_idx = max(num_computed_tokens - start_pos, 0)
                end_idx = min(
                    num_computed_tokens - start_pos + num_scheduled_tokens,
                    num_encoder_tokens,
                )
                assert start_idx < end_idx

1045
                mm_hash = mm_feature.identifier
1046
                encoder_output = self.encoder_cache.get(mm_hash, None)
1047
                assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
1048

1049
1050
1051
                assert pos_info.is_embed is None, (
                    "Expected all positions to be contiguous and embeddings."
                )
1052
1053

                req_start_pos = req_start_idx + start_pos - num_computed_tokens
1054
                is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] = True
1055
1056

                # Only whole mm items are processed
1057
                mm_embeds.append(encoder_output)
1058

1059
1060
            req_start_idx += num_scheduled_tokens

1061
        is_mm_embed = is_mm_embed[:padded_total_num_scheduled_tokens].to(self.device)
1062
1063
1064
1065
1066
1067

        return mm_embeds, is_mm_embed

    def _get_model_inputs(
        self,
        input_ids: torch.Tensor,
1068
        mm_embed_inputs: tuple[list[torch.Tensor], torch.Tensor] | None,
1069
    ):
1070
        if self.supports_mm_inputs:
1071
1072
            mm_embeds, is_mm_embed = mm_embed_inputs or (None, None)

1073
1074
1075
            # NOTE(woosuk): To unify token ids and soft tokens (vision
            # embeddings), we always use embeddings (rather than token ids)
            # as input to the multimodal model, even when the input is text.
1076
            inputs_embeds = self.model.embed_input_ids(
1077
                input_ids,
1078
                multimodal_embeddings=mm_embeds,
1079
                is_multimodal=is_mm_embed,
1080
            )
1081

1082
1083
1084
1085
1086
1087
1088
1089
            return None, inputs_embeds
        else:
            # For text-only models, we use token ids as input.
            # While it is possible to use embeddings as input just like the
            # multimodal models, it is not desirable for performance since
            # then the embedding layer is not included in the CUDA graph.
            return input_ids, None

1090
1091
1092
1093
    @torch.no_grad()
    def execute_model(
        self,
        scheduler_output: "SchedulerOutput",
1094
        intermediate_tensors: IntermediateTensors | None = None,
1095
1096
1097
1098
1099
1100
    ) -> ModelRunnerOutput | None:
        if self.scheduler_output is not None:
            raise RuntimeError(
                "State error: sample_tokens() must be called "
                "after execute_model() returns None."
            )
1101
1102
        # Update cached state
        self._update_states(scheduler_output)
1103
        if not scheduler_output.total_num_scheduled_tokens:
1104
1105
1106
1107
            if not has_kv_transfer_group():
                # Return empty ModelRunnerOutput if there's no work to do.
                return EMPTY_MODEL_RUNNER_OUTPUT

1108
            return self.kv_connector_no_forward(scheduler_output, self.vllm_config)
1109

1110
        mm_embed_inputs = None
1111
        if self.supports_mm_inputs:
1112
            # Run the multimodal encoder if any.
1113
            self._execute_mm_encoder(scheduler_output)
1114
1115
            mm_embed_inputs = self._gather_mm_embeddings(scheduler_output)

1116
        torch_xla.sync(wait=False)
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127

        self.scheduler_output = scheduler_output
        self.mm_embed_inputs = mm_embed_inputs
        return None

    @torch.no_grad()
    def sample_tokens(
        self, grammar_output: "GrammarOutput | None"
    ) -> ModelRunnerOutput:
        if self.scheduler_output is None:
            # Nothing to do (PP non-final rank case), output isn't used.
1128
            return None  # type: ignore[return-value]
1129
1130
1131
1132
1133
        scheduler_output = self.scheduler_output
        mm_embed_inputs = self.mm_embed_inputs
        self.scheduler_output = None
        self.mm_embed_inputs = None

1134
        # Prepare inputs, the requests might be split into multiple
1135
1136
1137
1138
        # executions, combine the result of each execution.
        start_index = 0
        combined_selected_tokens: list[torch.Tensor] = []
        combined_logprobs: list[LogprobsLists] = []
1139
1140
1141
1142
1143
1144

        # NOTE: setup current batch's metadata for kv connector.
        # Currently, only verified with NixlConnector
        with set_forward_context(None, self.vllm_config):
            self.maybe_setup_kv_connector(scheduler_output)

1145
        while start_index < self.input_batch.num_reqs:
1146
1147
1148
            attn_metadata, logits_indices, padded_num_reqs, num_reqs, end_index = (
                self._prepare_inputs(scheduler_output, start_index)
            )
1149
            input_ids, inputs_embeds = self._get_model_inputs(
1150
1151
                self.input_ids, mm_embed_inputs
            )
1152
            torch_xla.sync(wait=False)
1153
1154
            # Run the decoder
            with set_forward_context(
1155
1156
1157
1158
                attn_metadata,
                self.vllm_config,
                num_tokens=scheduler_output.total_num_scheduled_tokens,
            ):
1159
1160
1161
1162
1163
                hidden_states = self.model(
                    input_ids=input_ids,
                    positions=self.position_ids,
                    inputs_embeds=inputs_embeds,
                )
1164
            hidden_states = self.select_hidden_states(hidden_states, logits_indices)
1165
            logits = self.compute_logits(hidden_states)
1166
1167
1168
            tpu_sampling_metadata = TPUSupportedSamplingMetadata.from_input_batch(
                self.input_batch, padded_num_reqs, self.device
            )
1169
            if grammar_output is not None:
1170
                require_struct_decoding, grammar_bitmask_padded, arange = (
1171
                    self.prepare_structured_decoding_input(logits, grammar_output)
1172
1173
1174
1175
                )
                logits = self.structured_decode(
                    require_struct_decoding, grammar_bitmask_padded, logits, arange
                )
1176
            selected_token_ids = self.sample_from_logits_func(
1177
1178
                logits, tpu_sampling_metadata
            )
1179
1180
1181
1182
            # NOTE (NickLucche) Use the original logits (before any penalties or
            # temperature scaling) for the top-k logprobs. We can't enforce it
            # due to recompilations outside torch.compiled code, so just make
            # sure `sample_from_logits` does not modify the logits in-place.
1183
1184
1185
1186
1187
            logprobs = (
                self.gather_logprobs(logits, selected_token_ids)
                if tpu_sampling_metadata.logprobs
                else None
            )
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197

            # Remove padding on cpu and keep dynamic op outside of xla graph.
            selected_token_ids = selected_token_ids.cpu()[:num_reqs]

            combined_selected_tokens.append(selected_token_ids)
            if tpu_sampling_metadata.logprobs:
                combined_logprobs.append(logprobs.tolists())

            start_index = end_index

1198
1199
1200
1201
1202
        # NOTE: current kv load and save get h2d/d2h copies involved.
        # Those copies are blocking. Once they become async., kv_save
        # should be called right after each single forward pass,
        # instead of the forwards of the entire input batch.
        self.maybe_wait_for_kv_save()
1203
1204
1205
        finished_sending, finished_recving = self.get_finished_kv_transfers(
            scheduler_output
        )
1206

1207
1208
1209
1210
1211
1212
1213
1214
1215
        selected_token_ids = torch.cat(combined_selected_tokens, dim=0)
        if tpu_sampling_metadata.logprobs:

            def concat_lists(input_lists):
                result = []
                for input_list in input_lists:
                    result.extend(input_list)
                return result

1216
1217
1218
1219
1220
1221
1222
1223
1224
            logprobs_lists = LogprobsLists(
                logprob_token_ids=concat_lists(
                    [lp.logprob_token_ids for lp in combined_logprobs]
                ),
                logprobs=concat_lists([lp.logprobs for lp in combined_logprobs]),
                sampled_token_ranks=concat_lists(
                    [lp.sampled_token_ranks for lp in combined_logprobs]
                ),
            )
1225
1226
        else:
            logprobs_lists = None
1227

1228
1229
        # Update the cache state concurrently. Code above will not block until
        # we use `selected_token_ids`. Add mark_step if post-processing changes
1230
        request_seq_lens: list[tuple[int, CachedRequestState, int]] = []
1231
        discard_sampled_tokens_req_indices = []
1232
        num_reqs = self.input_batch.num_reqs
1233
1234
1235
        for i, req_id in zip(range(num_reqs), self.input_batch.req_ids):
            assert req_id is not None
            req_state = self.requests[req_id]
1236
1237
1238
1239
            seq_len = (
                req_state.num_computed_tokens
                + scheduler_output.num_scheduled_tokens[req_id]
            )
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
            if seq_len >= req_state.num_tokens:
                request_seq_lens.append((i, req_state, seq_len))
            else:
                # Ignore the sampled token from the partial request.
                # Rewind the generator state as if the token was not sampled.
                generator = self.input_batch.generators.get(i)
                if generator is not None:
                    # This relies on cuda-specific torch-internal impl details
                    generator.set_offset(generator.get_offset() - 4)

1250
1251
1252
1253
                # Record the index of the request that should not be sampled,
                # so that we could clear the sampled tokens before returning.
                discard_sampled_tokens_req_indices.append(i)

1254
        assert all(
1255
1256
            req_id is not None for req_id in self.input_batch.req_ids[:num_reqs]
        ), "req_ids contains None"
1257
        req_ids = cast(list[str], self.input_batch.req_ids[:num_reqs])
1258

1259
        prompt_logprobs_dict: dict[str, LogprobsTensors | None] = {}
1260
        for req_id in self.input_batch.req_ids[:num_reqs]:
1261
1262
            prompt_logprobs_dict[req_id] = None

1263
1264
        max_gen_len = selected_token_ids.shape[-1]
        if max_gen_len == 1:
1265
            valid_sampled_token_ids = selected_token_ids.tolist()
1266

1267
1268
1269
1270
            # Mask out the sampled tokens that should not be sampled.
            # TODO: Keep in sync with gpu_model_runner.py, in particular
            #       the "else" case here
            for i in discard_sampled_tokens_req_indices:
1271
                valid_sampled_token_ids[i].clear()
1272
1273

            # Append sampled tokens
1274
1275
1276
1277
1278
            for i, req_state, seq_len in request_seq_lens:
                token_id = valid_sampled_token_ids[i][0]
                self.input_batch.token_ids_cpu[i, seq_len] = token_id
                req_state.output_token_ids.append(token_id)
                self.input_batch.num_tokens[i] += 1
1279

1280
1281
1282
1283
        else:
            valid_mask = selected_token_ids != INVALID_TOKEN_ID
            gen_lens = valid_mask.sum(dim=1).tolist()
            valid_sampled_token_ids = [
1284
                seq.tolist() for seq in selected_token_ids[valid_mask].split(gen_lens)
1285
1286
1287
1288
            ]
            self.input_batch.num_tokens[:num_reqs] += gen_lens
            for i, req_state, seq_len in request_seq_lens:
                target_slice = slice(seq_len - gen_lens[i] + 1, seq_len + 1)
1289
1290
1291
                self.input_batch.token_ids_cpu[i, target_slice] = (
                    valid_sampled_token_ids[i]
                )
1292
1293
                req_state.output_token_ids.extend(valid_sampled_token_ids[i])

1294
1295
1296
1297
        kv_connector_output = (
            None
            if (finished_sending is None and finished_recving is None)
            else KVConnectorOutput(
1298
1299
1300
                finished_sending=finished_sending,
                finished_recving=finished_recving,
            )
1301
        )
1302

1303
        model_runner_output = ModelRunnerOutput(
1304
            req_ids=req_ids,
1305
            req_id_to_index=self.input_batch.req_id_to_index,
1306
            sampled_token_ids=valid_sampled_token_ids,
1307
            logprobs=logprobs_lists,
1308
            prompt_logprobs_dict=prompt_logprobs_dict,
1309
            pooler_output=[],
1310
1311
            kv_connector_output=kv_connector_output,
        )
1312
1313
1314
1315
1316

        # Check there are no new graphs compiled - all the graphs should be
        # captured and compiled during warm up.
        self._verify_num_xla_graphs("execute_model")

1317
1318
        return model_runner_output

1319
1320
1321
1322
1323
    def update_config(self, overrides: dict[str, Any]) -> None:
        # TODO: TPU config may need extra validation
        # https://github.com/vllm-project/vllm/pull/20095#discussion_r2201497754
        allowed_config_names = {"load_config", "model_config"}
        for config_name, config_overrides in overrides.items():
1324
1325
            assert config_name in allowed_config_names, (
                f"Config `{config_name}` not supported. "
1326
                f"Allowed configs: {allowed_config_names}"
1327
            )
1328
1329
1330
1331
            config = getattr(self, config_name)
            new_config = update_config(config, config_overrides)
            setattr(self, config_name, new_config)

1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
    def load_model(self) -> None:
        self.device = self.device_config.device

        # NOTE(woosuk): While the executor assigns the TP ranks to the worker
        # process, the ranks can be different from the ranks internally assigned
        # by the xm runtime. Therefore, there is a mismatch in the rank
        # assignment between the gloo (cpu) runtime and the xm (tpu) runtime.
        # This is not a problem in linear layers because all-reduce is
        # rank-agnostic. However, it matters for all-gather as the ranks
        # determine the order of concatenating the output tensors.
        # As a workaround, we use the xm's rank assignment only when loading
        # the embedding weights.
        xm_tp_rank = xr.global_ordinal()
        with patch(
1346
1347
1348
1349
            "vllm.model_executor.layers.vocab_parallel_embedding."
            "get_tensor_model_parallel_rank",
            return_value=xm_tp_rank,
        ):
1350
1351
1352
            try:
                if self.use_spmd:
                    tpu_loader = TPUModelLoader(
1353
1354
                        load_config=self.vllm_config.load_config
                    )
1355
                    model = tpu_loader.load_model(
1356
                        vllm_config=self.vllm_config,
1357
                        model_config=self.vllm_config.model_config,
1358
1359
                        mesh=self.mesh,
                    )
1360
                else:
1361
                    model_loader = get_model_loader(self.load_config)
1362
1363
                    logger.info("Loading model from scratch...")
                    model = model_loader.load_model(
1364
1365
                        vllm_config=self.vllm_config, model_config=self.model_config
                    )
1366
1367
1368
1369
1370
1371
            except RuntimeError as e:
                raise RuntimeError(
                    f"Unable to load model, a likely reason is the model is "
                    "too large for the current device's HBM memory. "
                    "Consider switching to a smaller model "
                    "or sharding the weights on more chips. "
1372
1373
                    f"See the detailed error: {e}"
                ) from e
1374
        if self.lora_config is not None:
1375
            model = self.load_lora_model(model, self.vllm_config, self.device)
1376
            replace_set_lora(model)
1377

1378
1379
        # Sync all pending XLA execution during model initialization and weight
        # loading.
1380
        torch_xla.sync(wait=False)
1381
        xm.wait_device_ops()
1382
1383
        if not hasattr(self, "model"):
            self.model = model
1384
        self.sampler = TPUSampler()
1385

1386
    def reload_weights(self) -> None:
1387
        assert getattr(self, "model", None) is not None, (
1388
            "Cannot reload weights before model is loaded."
1389
        )
1390
1391
1392
1393
        model_loader = get_model_loader(self.load_config)
        logger.info("Reloading weights inplace...")
        model_loader.load_weights(self.model, model_config=self.model_config)

1394
    @torch.no_grad()
1395
    def _dummy_run(self, num_tokens: int, num_reqs: int, num_blocks: int) -> None:
1396
        if self.supports_mm_inputs:
1397
            input_ids = None
1398
1399
1400
            inputs_embeds = torch.zeros(
                (num_tokens, self.hidden_size), dtype=self.dtype, device=self.device
            )
1401
        else:
1402
            input_ids = torch.zeros((num_tokens), dtype=torch.int32).to(self.device)
1403
            inputs_embeds = None
1404
        actual_num_reqs = min(num_tokens, num_reqs)
1405
        position_ids = torch.zeros(num_tokens, dtype=torch.int32).to(self.device)
1406
        padded_num_slices = _get_padded_num_kv_cache_update_slices(
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
            num_tokens, self.max_num_reqs, self.block_size
        )
        num_kv_update_slices = torch.tensor([padded_num_slices], dtype=torch.int32).to(
            self.device
        )
        slot_mapping = torch.zeros((3, padded_num_slices), dtype=torch.int32).to(
            self.device
        )
        block_tables = torch.zeros((num_reqs, num_blocks), dtype=torch.int32).to(
            self.device
        )
1418
        query_lens = [1] * num_reqs
1419
1420
1421
1422
1423
        query_start_loc = torch.cumsum(
            torch.tensor([0] + query_lens, dtype=torch.int32), dim=0, dtype=torch.int32
        ).to(self.device)
        context_lens = torch.ones((num_reqs,), dtype=torch.int32).to(self.device)
        num_seqs = torch.tensor([actual_num_reqs], dtype=torch.int32).to(self.device)
1424
1425
1426
1427
1428
        attn_metadata = PallasMetadata(
            slot_mapping=slot_mapping,
            block_tables=block_tables,
            context_lens=context_lens,
            query_start_loc=query_start_loc,
1429
            num_seqs=num_seqs,
1430
            num_kv_update_slices=num_kv_update_slices,
1431
            num_slices_per_kv_cache_update_block=self._num_slices_per_kv_cache_update_block,
1432
        )
1433

1434
        if self.supports_mm_inputs:
1435
1436
1437
            torch._dynamo.mark_dynamic(inputs_embeds, 0)
        else:
            torch._dynamo.mark_dynamic(input_ids, 0)
1438
1439
        torch._dynamo.mark_dynamic(position_ids, 0)
        torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
1440
1441
1442
        torch._dynamo.mark_dynamic(attn_metadata.block_tables, (0, 1))
        torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
        torch._dynamo.mark_dynamic(attn_metadata.query_start_loc, 0)
1443

1444
        layer_names = get_layers_from_vllm_config(self.vllm_config, Attention).keys()
1445
        per_layer_attn_metadata = {
1446
            layer_name: attn_metadata for layer_name in layer_names
1447
1448
        }

1449
1450
1451
1452
1453
1454
1455
1456
1457
        with (
            self.maybe_select_dummy_loras(
                self.lora_config, np.array([num_tokens], dtype=np.int32)
            ),
            set_forward_context(per_layer_attn_metadata, self.vllm_config, 0),
        ):
            out = self.model(
                input_ids=input_ids, positions=position_ids, inputs_embeds=inputs_embeds
            )
1458
        self._hidden_states_dtype = out.dtype
1459

1460
1461
1462
    def _set_active_loras(
        self, prompt_lora_mapping, token_lora_mapping, lora_requests
    ) -> None:
1463
        torch_xla.sync(wait=False)  # Captures input updates
1464
1465
1466
        super()._set_active_loras(
            prompt_lora_mapping, token_lora_mapping, lora_requests
        )
1467
        torch_xla.sync(wait=False)  # Captures metadata updates
1468

1469
    def _precompile_mm_encoder(self) -> None:
1470
        if not self.supports_mm_inputs:
1471
1472
            return

1473
1474
        # Pre-compile MM encoder for all supported data modalities.
        hf_config = self.vllm_config.model_config.hf_config
1475
1476
1477
1478
1479
1480
1481

        mm_budget = self.mm_budget
        assert mm_budget is not None

        max_items_per_seq_by_modality = mm_budget.max_items_per_batch_by_modality  # noqa: E501

        for mode, max_items_per_seq in max_items_per_seq_by_modality.items():
1482
            logger.info(
1483
1484
                "Compiling Multimodal %s Encoder with different input shapes.", mode
            )
1485
1486
            start = time.perf_counter()
            # No padding for MM encoder just yet.
1487
            for num_items in range(1, max_items_per_seq + 1):
1488
1489
                logger.info("  -- mode: %s items: %d", mode, num_items)
                batched_dummy_mm_inputs = self._get_mm_dummy_batch(
1490
1491
1492
                    mode,
                    num_items,
                )
1493
                # Run multimodal encoder.
1494
                torch_xla.sync(wait=False)
1495
                mm_embeds = self.model.embed_multimodal(**batched_dummy_mm_inputs)
1496
                torch_xla.sync(wait=False)
1497
1498
1499
                num_patches = mm_embeds[0].shape[0]
                items_size = num_patches * num_items

1500
                # NOTE (NickLucche) pre-compile `embed_input_ids` when mm
1501
1502
1503
1504
1505
1506
1507
1508
                # embeddings are present. We assume `--disable-mm-chunked`,
                # hence only whole items can be scheduled. This implies we just
                # need to compile when `num_items` fit the (padded) `input_ids`
                for num_tokens in self.num_tokens_paddings:
                    if num_tokens >= items_size:
                        # XLA Workaround: if torch.zeros(..device) is used, XLA
                        # compiles a scalar+expansion op, which won't match
                        # the graph generated at runtime. CPU->TPU must be used
1509
1510
1511
                        placeholders_ids = torch.zeros(
                            num_tokens, dtype=torch.int32, device="cpu"
                        )
1512
                        # Align placeholders and actual num mm_embeddings.
1513
                        placeholders_ids[:items_size] = hf_config.image_token_index
1514
1515

                        placeholders_ids = placeholders_ids.to(self.device)
1516
1517
1518
1519

                        mm_mask = torch.tensor([False] * num_tokens)
                        mm_mask[:items_size] = True
                        mm_mask = mm_mask.to(self.device)
1520
                        # Assign outputs or the graph will be cut short.
1521
1522
1523
1524
                        a, b = self._get_model_inputs(
                            placeholders_ids,
                            mm_embed_inputs=([mm_embeds], mm_mask),
                        )
1525
                        assert a is None
1526
                        torch_xla.sync(wait=False)
1527

1528
            # Pre-compile `embed_input_ids` when mm_embeddings are not
1529
1530
            # present. Chunk is only made of text, no mm_placeholders.
            for num_tokens in self.num_tokens_paddings:
1531
1532
1533
                placeholders_ids = torch.zeros(
                    num_tokens, dtype=torch.int32, device="cpu"
                )
1534
                placeholders_ids = placeholders_ids.to(self.device)
1535
1536
1537
1538
                a, b = self._get_model_inputs(
                    placeholders_ids,
                    mm_embed_inputs=None,
                )
1539
                assert a is None
1540
                torch_xla.sync(wait=False)
1541
1542
1543
1544

            xm.wait_device_ops()
            end = time.perf_counter()
            logger.info(
1545
1546
1547
1548
                "Multimodal %s Encoder compilation finished in in %.2f [secs].",
                mode,
                end - start,
            )
1549

1550
    def _precompile_backbone(self) -> None:
1551
1552
        logger.info("Compiling the model with different input shapes.")
        start = time.perf_counter()
1553
        for num_tokens in self.num_tokens_paddings:
1554
            logger.info("  -- num_tokens: %d", num_tokens)
1555
1556
1557
            self._dummy_run(
                num_tokens, self.num_reqs_max_model_len, self.max_num_blocks_per_req
            )
1558
            if self.most_model_len is not None:
1559
1560
1561
1562
1563
                self._dummy_run(
                    num_tokens,
                    self.num_reqs_most_model_len,
                    self.num_blocks_per_most_len_req,
                )
1564
1565
        xm.wait_device_ops()
        end = time.perf_counter()
1566
        logger.info("Compilation finished in %.2f [secs].", end - start)
1567
        self._update_num_xla_graphs("model backbone")
1568

1569
1570
1571
    def _precompile_select_hidden_states(self) -> None:
        # Compile hidden state selection function for bucketed
        # n_tokens x max_num_reqs. Graph is really small so this is fine.
1572
        logger.info("Compiling select_hidden_states with different input shapes.")
1573
1574
        start = time.perf_counter()
        hsize = self.model_config.get_hidden_size()
1575
        for num_tokens in self.num_tokens_paddings:
1576
1577
1578
            dummy_hidden = torch.zeros(
                (num_tokens, hsize), device=self.device, dtype=self._hidden_states_dtype
            )
1579
1580
            torch._dynamo.mark_dynamic(dummy_hidden, 0)
            for num_reqs in self.num_reqs_paddings:
1581
                indices = torch.zeros(num_reqs, dtype=torch.int32, device=self.device)
1582
1583
                torch._dynamo.mark_dynamic(indices, 0)
                self.select_hidden_states(dummy_hidden, indices)
1584
                logger.info("  -- num_tokens: %d, num_seqs: %d", num_tokens, num_reqs)
1585
1586
1587
1588
                # Requests can't be more than tokens. But do compile for the
                # next bigger value in case num_tokens uses bucketed padding.
                if num_reqs >= min(num_tokens, self.max_num_reqs):
                    break
1589
        xm.wait_device_ops()
1590
        end = time.perf_counter()
1591
        logger.info("Compilation finished in %.2f [secs].", end - start)
1592
        self._update_num_xla_graphs("select_hidden_states")
1593

1594
1595
    def _precompile_compute_logits(self) -> None:
        logger.info("Compiling compute_logits with different input shapes.")
1596
1597
1598
        start = time.perf_counter()
        hsize = self.model_config.get_hidden_size()
        for num_reqs in self.num_reqs_paddings:
1599
1600
1601
            dummy_hidden = torch.zeros(
                (num_reqs, hsize), device=self.device, dtype=self._hidden_states_dtype
            )
1602
1603
1604
1605
1606
1607
1608
1609
1610
            torch._dynamo.mark_dynamic(dummy_hidden, 0)
            self.compute_logits(dummy_hidden)
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("compute_logits")

    def _precompile_structured_decoding(self) -> None:
1611
        logger.info("Compiling structured_decoding with different input shapes.")
1612
1613
        start = time.perf_counter()
        for num_reqs in self.num_reqs_paddings:
1614
1615
1616
1617
1618
1619
1620
1621
1622
            dummy_logits = torch.zeros(
                (num_reqs, self.vocab_size),
                device=self.device,
                dtype=self._hidden_states_dtype,
            )
            dummy_require_struct_decoding = self.require_structured_out_cpu[
                :num_reqs
            ].to(self.device)
            dummy_grammar_bitmask = self.grammar_bitmask_cpu[:num_reqs].to(self.device)
1623
1624
1625
1626
            # The first dimension of the above 3 dummy tensors cannot be
            # mark_dynamic because some operations in structured_decode require
            # them to be static.
            arange = self.structured_decode_arange.to(self.device)
1627
1628
1629
1630
1631
1632
            self.structured_decode(
                dummy_require_struct_decoding,
                dummy_grammar_bitmask,
                dummy_logits,
                arange,
            )
1633
1634
1635
1636
1637
1638
1639
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("structured_decoding")

    def _precompile_sample_from_logits(self) -> None:
1640
        logger.info("Compiling sample_from_logits with different input shapes.")
1641
1642
        start = time.perf_counter()
        for num_reqs in self.num_reqs_paddings:
1643
1644
1645
1646
1647
            dummy_logits = torch.zeros(
                (num_reqs, self.vocab_size),
                device=self.device,
                dtype=self._hidden_states_dtype,
            )
1648
1649
            # The first dimension of dummy_logits cannot be mark_dynamic
            # because some operations in the sampler require it to be static.
1650
1651
            for all_greedy in [False, True]:
                generate_params_if_all_greedy = not all_greedy
1652
1653
1654
1655
1656
1657
                sampling_metadata = TPUSupportedSamplingMetadata.from_input_batch(
                    self.input_batch,
                    num_reqs,
                    self.device,
                    generate_params_if_all_greedy,
                )
1658
                sampling_metadata.all_greedy = all_greedy
1659
                with self.maybe_select_dummy_loras(
1660
1661
1662
                    self.lora_config, np.array([num_reqs], dtype=np.int32)
                ):
                    self.sample_from_logits_func(dummy_logits, sampling_metadata)
1663
1664
1665
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
1666
1667
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("sample_from_logits")
1668

1669
1670
1671
1672
    def _precompile_gather_logprobs(self) -> None:
        logger.info("Compiling gather_logprobs with different input shapes.")
        start = time.perf_counter()
        for num_reqs in self.num_reqs_paddings:
1673
1674
1675
1676
1677
1678
            dummy_logits = torch.zeros(
                (num_reqs, self.vocab_size),
                device=self.device,
                dtype=self._hidden_states_dtype,
            )
            dummy_tokens = torch.zeros((num_reqs, 1), dtype=torch.int64).to(self.device)
1679
            with self.maybe_select_dummy_loras(
1680
1681
                self.lora_config, np.array([num_reqs], dtype=np.int32)
            ):
1682
                self.gather_logprobs(dummy_logits, dummy_tokens)
1683
1684
1685
1686
1687
1688
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("gather_logprobs")

1689
1690
1691
1692
    def capture_model(self) -> None:
        """
        Precompile all the subgraphs with possible input shapes.
        """
1693
1694
1695
1696
1697
1698
1699
1700
        with self.maybe_setup_dummy_loras(self.lora_config):
            self._precompile_mm_encoder()
            self._precompile_backbone()
            self._precompile_select_hidden_states()
            self._precompile_compute_logits()
            self._precompile_structured_decoding()
            self._precompile_sample_from_logits()
            self._precompile_gather_logprobs()
1701

1702
1703
1704
1705
1706
    def profile_run(
        self,
        num_tokens: int,
    ) -> None:
        # Profile with multimodal encoder & encoder cache.
1707
        if self.supports_mm_inputs:
1708
1709
            mm_config = self.model_config.multimodal_config
            if mm_config is not None and mm_config.skip_mm_profiling:
1710
                logger.info(
1711
                    "Skipping memory profiling for multimodal encoder and "
1712
1713
                    "encoder cache."
                )
1714
1715
1716
1717
1718
1719
1720
1721
1722
            else:
                mm_budget = self.mm_budget
                assert mm_budget is not None

                # TODO: handle encoder-decoder models once we support them.
                if (encoder_budget := mm_budget.get_encoder_budget()) > 0:
                    # NOTE: Currently model is profiled with a single non-text
                    # modality with the max possible input tokens even when
                    # it supports multiple.
1723
                    dummy_modality = mm_budget.get_modality_with_max_tokens()
1724
1725
1726
                    max_mm_items_per_batch = mm_budget.max_items_per_batch_by_modality[
                        dummy_modality
                    ]
1727
1728
1729
1730
1731
1732
1733
1734
1735

                    logger.info(
                        "Encoder cache will be initialized with a budget of "
                        "%s tokens, and profiled with %s %s items of the "
                        "maximum feature size.",
                        encoder_budget,
                        max_mm_items_per_batch,
                        dummy_modality,
                    )
1736

1737
1738
1739
1740
1741
                    # Create dummy batch of multimodal inputs.
                    batched_dummy_mm_inputs = self._get_mm_dummy_batch(
                        dummy_modality,
                        max_mm_items_per_batch,
                    )
1742

1743
1744
1745
1746
                    # Run multimodal encoder.
                    # Isolate encoder graph from post-processing to minimize
                    # impact of recompilation until it's fixed.
                    start = time.perf_counter()
1747
                    torch_xla.sync(wait=False)
1748
                    dummy_encoder_outputs = self.model.embed_multimodal(
1749
1750
                        **batched_dummy_mm_inputs
                    )
1751
                    torch_xla.sync(wait=False)
1752
1753
1754
1755
                    xm.wait_device_ops()
                    end = time.perf_counter()
                    logger.info(
                        "Multimodal Encoder profiling finished in %.2f [secs].",
1756
1757
                        end - start,
                    )
1758
1759
1760
1761
1762

                    sanity_check_mm_encoder_outputs(
                        dummy_encoder_outputs,
                        expected_num_items=max_mm_items_per_batch,
                    )
1763

1764
                    # Cache the dummy encoder outputs.
1765
                    self.encoder_cache["tmp"] = dict(enumerate(dummy_encoder_outputs))
1766
1767

        # Trigger compilation for general shape.
1768
1769
1770
        self._dummy_run(
            num_tokens, self.num_reqs_max_model_len, self.max_num_blocks_per_req
        )
1771
        if self.most_model_len is not None:
1772
1773
1774
1775
1776
            self._dummy_run(
                num_tokens,
                self.num_reqs_most_model_len,
                self.num_blocks_per_most_len_req,
            )
1777

1778
        torch_xla.sync(wait=False)
1779
1780
1781
1782
        xm.wait_device_ops()
        self.encoder_cache.clear()
        gc.collect()

1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
    def maybe_setup_cross_layer_kv_sharing(
        self,
        kv_caches: dict[str, torch.Tensor],
        kv_cache_config: KVCacheConfig,
    ) -> None:
        """
        Add layers that re-use KV cache to KV cache group of its target layer.
        Mapping of KV cache tensors happens in `initialize_kv_cache_tensors()`
        """
        if not self.shared_kv_cache_layers:
            # No cross-layer KV sharing, return
            return

        add_kv_sharing_layers_to_kv_cache_groups(
            self.shared_kv_cache_layers,
            kv_cache_config.kv_cache_groups,
        )

1801
1802
        for layer_name, target_layer_name in self.shared_kv_cache_layers.items():
            logger.debug("%s reuses KV cache of %s", layer_name, target_layer_name)
1803
1804
            kv_caches[layer_name] = kv_caches[target_layer_name]

1805
1806
1807
1808
    def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
        """
        Initialize KV cache based on `kv_cache_config`.
        Args:
1809
            kv_cache_config: Configuration for the KV cache, including the KV
1810
1811
            cache size of each layer
        """
1812
        if len(kv_cache_config.kv_cache_groups) > 1:
1813
            raise NotImplementedError(
1814
1815
                "Hybrid models with more than one KV cache type are not supported yet."
            )
1816

1817
1818
1819
1820
        if (
            kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
            != self.block_size
        ):
1821
1822
1823
1824
1825
1826
1827
            self.input_batch = InputBatch(
                max_num_reqs=self.max_num_reqs,
                max_model_len=self.max_model_len,
                max_num_batched_tokens=self.max_num_tokens,
                device=self.device,
                pin_memory=self.pin_memory,
                vocab_size=self.model_config.get_vocab_size(),
1828
1829
1830
                block_sizes=[
                    kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
                ],
1831
1832
1833
                kernel_block_sizes=[
                    kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
                ],
1834
1835
            )
        # Verify dtype compatibility between block_table_cpu and input_batch
1836
1837
1838
1839
        assert (
            self.block_table_cpu.dtype
            == self.input_batch.block_table[0].get_cpu_tensor().dtype
        )
1840

1841
1842
1843
        kv_cache_sizes = {}
        for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
            assert len(kv_cache_tensor.shared_by) == 1, (
1844
1845
                "KV cache tensor shared by multiple layers is not supported in TPU."
            )
1846
            kv_cache_sizes[kv_cache_tensor.shared_by[0]] = kv_cache_tensor.size
1847

1848
        kv_caches: dict[str, torch.Tensor] = {}
1849
1850
1851
        for kv_cache_group in kv_cache_config.kv_cache_groups:
            kv_cache_spec = kv_cache_group.kv_cache_spec
            for layer_name in kv_cache_group.layer_names:
1852
1853
1854
                tensor_size = kv_cache_sizes[layer_name]
                assert tensor_size % kv_cache_spec.page_size_bytes == 0
                num_blocks = tensor_size // kv_cache_spec.page_size_bytes  # noqa
1855
                if isinstance(kv_cache_spec, AttentionSpec):
1856
1857
1858
                    if self.use_spmd:
                        num_kv_heads = kv_cache_spec.num_kv_heads
                        assert self.original_parallel_config is not None
1859
                        tp_size = self.original_parallel_config.tensor_parallel_size
1860
1861
1862
                        # TODO: Handle kv cache duplication under SPMD mode.
                        assert num_kv_heads % tp_size == 0, (
                            f"num_kv_heads {num_kv_heads} must be divisible by "
1863
1864
                            f"tp_size {tp_size} under SPMD mode"
                        )
1865
                    kv_cache_shape = PallasAttentionBackend.get_kv_cache_shape(
1866
1867
1868
1869
1870
                        num_blocks,
                        kv_cache_spec.block_size,
                        kv_cache_spec.num_kv_heads,
                        kv_cache_spec.head_size,
                    )
1871
1872
                    dtype = kv_cache_spec.dtype

1873
1874
1875
                    tpu_kv_cache = torch.zeros(kv_cache_shape, dtype=dtype).to(
                        self.device
                    )
1876

1877
                    kv_caches[layer_name] = tpu_kv_cache
1878
1879
                else:
                    raise NotImplementedError
1880

1881
1882
        # Set up cross-layer KV cache sharing if needed
        self.maybe_setup_cross_layer_kv_sharing(kv_caches, kv_cache_config)
1883

1884
1885
1886
        bind_kv_cache(
            kv_caches,
            self.vllm_config.compilation_config.static_forward_context,
1887
1888
            self.kv_caches,
        )
1889

1890
1891
1892
        if self.use_spmd:
            # Shard KV Cache
            for cache in self.kv_caches:
1893
                xs.mark_sharding(cache, self.mesh, (None, "x", None, None))
1894

1895
1896
1897
1898
        if has_kv_transfer_group():
            get_kv_transfer_group().register_kv_caches(kv_caches)
            get_kv_transfer_group().set_host_xfer_buffer_ops(copy_kv_blocks)

1899
    def reset_dynamo_cache(self):
1900
1901
1902
1903
        # NOTE: We check `is_multimodal_model` instead of `supports_mm_inputs`
        # since the compiled model object of the language backbone of a
        # multimodal model needs to be extracted via `get_language_model`.
        if self.model_config.is_multimodal_model:
1904
            compiled_model = self.model.get_language_model().model
1905
1906
        else:
            compiled_model = self.model.model
1907
        if isinstance(compiled_model, TorchCompileWithNoGuardsWrapper):
1908
1909
            logger.info("Clear dynamo cache and cached dynamo bytecode.")
            torch._dynamo.eval_frame.remove_from_cache(
1910
                compiled_model.original_code_object()
1911
            )
1912
1913
1914
            # Reset the wrapper to re-initialize.
            compiled_model.compiled = False
            TorchCompileWithNoGuardsWrapper.__init__(compiled_model)
1915

1916
1917
1918
1919
1920
    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
    def select_hidden_states(self, hidden_states, indices_do_sample):
        return hidden_states[indices_do_sample]

    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1921
    def compute_logits(self, sample_hidden_states: torch.Tensor) -> torch.Tensor:
1922
        return self.model.compute_logits(sample_hidden_states)
1923

1924
1925
1926
    # TODO: Under SPMD mode, sample_from_logits has correctness issue.
    #       Re-enable the torch.compile once the issue is fixed in torchxla.
    # @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1927
    def sample_from_logits(
1928
1929
        self, logits: torch.Tensor, sampling_metadata: TPUSupportedSamplingMetadata
    ) -> torch.Tensor:
1930
        """
1931
        Sample with xla-friendly function. This function is to be traced
1932
1933
        separately from `forward` for lighter compilation overhead.
        """
1934
1935
1936
        if sampling_metadata.all_greedy:
            out_tokens = torch.argmax(logits, dim=-1, keepdim=True)
        else:
1937
            out_tokens = self.sampler(logits, sampling_metadata).sampled_token_ids
1938
1939
        return out_tokens

1940
    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1941
1942
1943
    def gather_logprobs(
        self, logits: torch.Tensor, sampled_tokens: torch.Tensor
    ) -> LogprobsTensors:
1944
1945
1946
1947
1948
1949
1950
1951
1952
        """
        Gather the top_logprobs with corresponding tokens. Use a fixed number
        of logprobs as an alternative to having multiple pre-compiled graphs.
        Select the number of logprobs actually demanded by each request on CPU.
        """
        logprobs = self.sampler.compute_logprobs(logits)
        return self.sampler.gather_logprobs(
            logprobs,
            self.model_config.max_logprobs,
1953
1954
            token_ids=sampled_tokens.squeeze(-1),
        )
1955

1956
    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1957
1958
1959
1960
1961
1962
1963
    def structured_decode(
        self,
        require_struct_decoding: torch.Tensor,
        grammar_bitmask: torch.Tensor,
        logits: torch.Tensor,
        arange: torch.Tensor,
    ) -> torch.Tensor:
1964
1965
1966
        return torch.where(
            require_struct_decoding,
            self.apply_grammar_bitmask(logits, grammar_bitmask, arange),
1967
1968
            logits,
        )
1969

1970
1971
1972
1973
    def apply_grammar_bitmask(
        self, logits: torch.Tensor, grammar_bitmask: torch.Tensor, arange: torch.Tensor
    ):
        assert logits.shape[0] == grammar_bitmask.shape[0]
1974
1975
        logits_cloned = logits.clone()
        for i in range(logits.shape[0]):
1976
1977
1978
1979
1980
            unpacked_bitmask = (
                torch.bitwise_right_shift(grammar_bitmask[i][:, None], arange[None, :])
                & 1
            ) == 0
            unpacked_bitmask = unpacked_bitmask.reshape(-1)[: self.vocab_size]
1981
            logits_cloned[i] = logits_cloned[i].masked_fill(
1982
1983
                unpacked_bitmask, -float("inf")
            )
1984
1985
        return logits_cloned

1986
1987
    def embed_multimodal(self, *args, **kwargs):
        return self.model.embed_multimodal(*args, **kwargs)
1988

1989
1990
    def embed_input_ids(self, *args, **kwargs):
        return self.model.embed_input_ids(*args, **kwargs)
1991

1992
    def prepare_structured_decoding_input(
1993
        self, logits: torch.Tensor, grammar_output: "GrammarOutput"
1994
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
1995
        grammar_bitmask = grammar_output.grammar_bitmask
1996
1997
1998
1999
2000
2001
        num_reqs, _ = logits.shape

        # Reset pre-allocated tensors
        self.grammar_bitmask_cpu.zero_()
        self.require_structured_out_cpu.zero_()

2002
        cumulative_mask_idx = 0
2003
        for req_id in grammar_output.structured_output_request_ids:
2004
            if req_id not in self.input_batch.req_id_to_index:
2005
2006
                continue
            batch_index = self.input_batch.req_id_to_index[req_id]
2007
            self.grammar_bitmask_cpu[batch_index] = torch.from_numpy(
2008
2009
                grammar_bitmask[cumulative_mask_idx]
            )
2010
2011
2012
2013
2014
2015
            # It's not guaranteed that all requests in this batch require
            # structured output, so create a bool tensor to represent
            # the requests that need structured output.
            self.require_structured_out_cpu[batch_index] = True
            cumulative_mask_idx += 1

2016
2017
2018
2019
2020
        return (
            self.require_structured_out_cpu[:num_reqs].to(logits.device),
            self.grammar_bitmask_cpu[:num_reqs].to(logits.device),
            self.structured_decode_arange.to(logits.device),
        )
2021

2022
2023
2024
2025
2026
2027
    def _get_mm_dummy_batch(
        self,
        modality: str,
        max_items_per_batch: int,
    ) -> BatchedTensorInputs:
        """Dummy data for profiling and precompiling multimodal models."""
2028
2029
        assert self.mm_budget is not None

2030
        dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
2031
            model_config=self.model_config,
2032
            seq_len=self.max_model_len,
2033
            mm_counts={modality: 1},
2034
            cache=self.mm_budget.cache,
2035
        )
2036
2037
2038
        dummy_mm_data = dummy_decoder_data.multi_modal_data

        # Result in the maximum GPU consumption of the model
2039
2040
        dummy_mm_item = dummy_mm_data[modality][0]
        dummy_mm_items = [dummy_mm_item] * max_items_per_batch
2041

2042
        model = cast(SupportsMultiModal, self.model)
2043
2044
2045
2046
2047
2048
2049
        return next(
            grouped_mm_kwargs
            for _, _, grouped_mm_kwargs in group_mm_kwargs_by_modality(
                dummy_mm_items,
                device=self.device,
                pin_memory=self.pin_memory,
                merge_by_field_config=model.merge_by_field_config,
2050
                multimodal_cpu_fields=model.multimodal_cpu_fields,
2051
2052
            )
        )
2053

2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065

def _get_req_paddings(min_req_size: int, max_req_size: int) -> list[int]:
    logger.info("Preparing request paddings:")
    # assert min_req_size is power of 2
    assert (min_req_size & (min_req_size - 1) == 0) and min_req_size > 0
    paddings: list = []
    num = max(MIN_NUM_SEQS, min_req_size)
    while num <= max_req_size and (len(paddings) == 0 or paddings[-1] != num):
        paddings.append(num)
        logger.info("    %d", num)
        num = _get_padded_num_reqs_with_upper_limit(num + 1, max_req_size)
    return paddings
2066
2067


2068
def _get_padded_num_reqs_with_upper_limit(x: int, upper_limit: int) -> int:
2069
    res = MIN_NUM_SEQS if x <= MIN_NUM_SEQS else 1 << (x - 1).bit_length()
2070
    return min(res, upper_limit)
2071
2072


2073
2074
2075
2076
def _get_token_paddings(
    min_token_size: int, max_token_size: int, padding_gap: int
) -> list[int]:
    """Generate a list of padding size, starting from min_token_size,
2077
    ending with a number that can cover max_token_size
2078

2079
2080
2081
    If padding_gap == 0 then:
        increase 2X each time (exponential)
    else:
2082
        first increase the size to twice,
2083
        then increase the padding size by padding_gap.
2084
    """
2085
2086
    # assert min_token_size is power of 2
    assert (min_token_size & (min_token_size - 1) == 0) and min_token_size > 0
2087
2088
    paddings = []
    num = min_token_size
2089
2090

    if padding_gap == 0:
2091
        logger.info("Using exponential token paddings:")
2092
        while True:
2093
2094
            logger.info("    %d", num)
            paddings.append(num)
2095
2096
            if num >= max_token_size:
                break
2097
2098
            num *= 2
    else:
2099
        logger.info("Using incremental token paddings:")
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
        while num <= padding_gap:
            logger.info("    %d", num)
            paddings.append(num)
            num *= 2
        num //= 2
        while num < max_token_size:
            num += padding_gap
            logger.info("    %d", num)
            paddings.append(num)

2110
2111
2112
2113
    return paddings


def _get_padded_token_len(paddings: list[int], x: int) -> int:
2114
    """Return the first element in paddings list greater or equal to x."""
2115
2116
2117
    index = bisect.bisect_left(paddings, x)
    assert index < len(paddings)
    return paddings[index]
2118
2119


2120
2121
2122
def _get_padded_num_kv_cache_update_slices(
    num_tokens: int, max_num_reqs: int, page_size: int
) -> int:
2123
2124
    """Calculates the padded number of KV cache update slices to avoid
    recompilation."""
2125
2126
2127
2128
2129
    # NOTE(chengjiyao): let's say R_i is the token num for i-th request,
    # so it occupies most 2 + R_i // page_size pages. The total maximum
    # possible number of pages needed is sum(2 + R_i // page_size), which
    # is <= 2 * max_num_reqs + sum(R_i) // page_size
    # = 2 * max_num_reqs + num_tokens // page_size
2130
2131
2132
2133
2134
    padded_num_slices = 2 * max_num_reqs + num_tokens // page_size
    padded_num_slices = min(padded_num_slices, num_tokens)
    return padded_num_slices


2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
def _get_num_slices_per_kv_cache_update_block(page_size_bytes: int) -> int:
    """Find the optimum number of slices to copy per Pallas program instance.

    Increasing the number of slices copied in one instance of the kernel program
    will increase HBM bandwidth utilization via more in-flight DMAs.

    However, it will also use more VMEM, and experimentally, we observed
    performance regression at 128 slices on v6e, likely due to running
    out of scalar registers. Thus this function will limit the number of
    slices to 64.
    """
2146
2147
2148
    # The default vmem_limit_bytes of a pallas kernel is 32MB. Here we
    # calculate num_slices_per_block based on 16MB in case any register spills.
    vmem_limit = 16 * 1024 * 1024
2149
2150
2151
2152
2153
2154
2155
2156
    num_slices_per_block = vmem_limit // page_size_bytes
    assert num_slices_per_block > 0, "Number of slices should be positive"
    num_slices_per_block = prev_power_of_2(num_slices_per_block)
    if num_slices_per_block > 64:
        num_slices_per_block = 64
    return num_slices_per_block


2157
2158
2159
2160
2161
2162
def replace_set_lora(model):
    def _tpu_set_lora(
        self,
        index: int,
        lora_a: torch.Tensor,
        lora_b: torch.Tensor,
2163
        embeddings_tensor: torch.Tensor | None,
2164
2165
2166
2167
    ):
        # TODO: The integer index leads to a recompilation, but converting it
        # to a tensor doesn't seem to work anymore. This might be fixed with a
        # later release of torch_xla.
2168
        self._original_set_lora(index, lora_a, lora_b, embeddings_tensor)
2169
        torch_xla.sync(wait=False)
2170
2171
2172

    def _tpu_reset_lora(self, index: int):
        self._original_reset_lora(index)
2173
        torch_xla.sync(wait=False)
2174
2175
2176
2177
2178

    for _, module in model.named_modules():
        if isinstance(module, BaseLayerWithLoRA):
            module._original_set_lora = module.set_lora
            module._original_reset_lora = module.reset_lora
2179
2180
2181
2182
2183
2184
            module.set_lora = _tpu_set_lora.__get__(  # type: ignore[method-assign]
                module, module.__class__
            )
            module.reset_lora = _tpu_reset_lora.__get__(  # type: ignore[method-assign]
                module, module.__class__
            )