tpu_model_runner.py 89.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import bisect
4
import gc
5
import time
6
from typing import TYPE_CHECKING, Any, Optional, cast
7
8
9
10
11
from unittest.mock import patch

import numpy as np
import torch
import torch.nn as nn
12

13
# TPU XLA related
14
import torch_xla
15
import torch_xla.core.xla_model as xm
16
import torch_xla.distributed.spmd as xs
17
18
import torch_xla.runtime as xr

19
import vllm.envs as envs
20
from vllm.attention import Attention
21
from vllm.attention.backends.abstract import AttentionType
22
from vllm.attention.layers.chunked_local_attention import ChunkedLocalAttention
23
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
24
25
26
27
28
29
30
from vllm.config import (
    ParallelConfig,
    VllmConfig,
    get_layers_from_vllm_config,
    update_config,
)
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group
31
from vllm.distributed.kv_transfer.kv_connector.utils import copy_kv_blocks
32
from vllm.forward_context import set_forward_context
33
from vllm.logger import init_logger
34
from vllm.lora.layers import BaseLayerWithLoRA
35
from vllm.model_executor.model_loader import get_model_loader
36
from vllm.model_executor.model_loader.tpu import TPUModelLoader
37
38
39
40
from vllm.model_executor.models.interfaces import (
    SupportsMultiModal,
    supports_transcription,
)
41
from vllm.model_executor.models.interfaces_base import (
42
43
44
    is_pooling_model,
    is_text_generation_model,
)
45
from vllm.multimodal import MULTIMODAL_REGISTRY
46
47
48
49
50
from vllm.multimodal.inputs import (
    BatchedTensorInputs,
    MultiModalKwargsItem,
    PlaceholderRange,
)
51
from vllm.multimodal.utils import group_mm_kwargs_by_modality
52
from vllm.sequence import IntermediateTensors
53
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
from vllm.utils import LayerBlockType, cdiv, is_pin_memory_available, prev_power_of_2
from vllm.v1.attention.backends.pallas import (
    TPU_STR_DTYPE_TO_TORCH_DTYPE,
    PallasAttentionBackend,
    PallasMetadata,
    get_page_size_bytes,
)
from vllm.v1.kv_cache_interface import (
    AttentionSpec,
    FullAttentionSpec,
    KVCacheConfig,
    KVCacheSpec,
    SlidingWindowSpec,
)
from vllm.v1.outputs import (
    EMPTY_MODEL_RUNNER_OUTPUT,
    LogprobsLists,
    LogprobsTensors,
    ModelRunnerOutput,
)
74
75
from vllm.v1.sample.tpu.metadata import TPUSupportedSamplingMetadata
from vllm.v1.sample.tpu.sampler import Sampler as TPUSampler
76
from vllm.v1.worker.kv_connector_model_runner_mixin import (
77
78
79
    KVConnectorModelRunnerMixin,
    KVConnectorOutput,
)
80
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
81
from vllm.v1.worker.tpu_input_batch import CachedRequestState, InputBatch
82

83
84
85
86
87
88
from .utils import (
    MultiModalBudget,
    add_kv_sharing_layers_to_kv_cache_groups,
    bind_kv_cache,
    sanity_check_mm_encoder_outputs,
)
89

90
if TYPE_CHECKING:
91
    from vllm.v1.core.sched.output import SchedulerOutput
92
93
94

logger = init_logger(__name__)

95
INVALID_TOKEN_ID = -1
96
97
# Smallest output size
MIN_NUM_SEQS = 8
98
99


100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
#########################################################
# Ways to avoid recompilation
#########################################################
#
# The model executor has two primary components:
# 1. preparing the model and sampler inputs
# 2. executing the model and sampler.
# The core idea is to avoid any TPU computation during input preparation. For
# better compilation tracking and increased flexibility, the model execution and
# sampler are divided into several distinct components.
#
# Below are the detailed steps:
#
# Step 1
# It is recommended to avoid TPU operations when preparing the model and sampler
# inputs. CPU tensors can be prepared and transferred to the XLA device using
# cpu_tensor.to(xla_device), which only triggers CPU to TPU transfers and avoids
# compilation.
#
# Step 2
# The TPU execution should be decomposed into subgraphs (4 at the moment):
# 1. the main model
# 2. selecting hidden states for each request
# 3. sampler
# 4. encoder.
# Each subgraph should be decorated in a torch.compile. This is used to make
# sure that we have the same subgraph topology in both dummy_run and
# xecute_model. The results from these subgraphs should either be passed to
# other subgraphs, or transferred from TPU to CPU using xla_tensor.cpu() for
# subsequent processing on the CPU.
#
# Step 3
# The dummy_run should be comprehensive, ensuring all potential input shapes and
# branch predictions are included as subgraph inputs to facilitate
# pre-compilation.
135
class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
136
137
138
139
    def __init__(
        self,
        vllm_config: VllmConfig,
        device: torch.device,
140
        original_parallel_config: Optional[ParallelConfig] = None,
141
142
143
144
145
146
147
    ):
        self.vllm_config = vllm_config
        self.model_config = vllm_config.model_config
        self.cache_config = vllm_config.cache_config
        self.lora_config = vllm_config.lora_config
        self.load_config = vllm_config.load_config
        self.parallel_config = vllm_config.parallel_config
148
        self.original_parallel_config = original_parallel_config
149
150
151
152
153
154
155
156
157
158
        self.scheduler_config = vllm_config.scheduler_config
        self.speculative_config = vllm_config.speculative_config
        self.observability_config = vllm_config.observability_config
        self.device_config = vllm_config.device_config

        model_config = self.model_config
        cache_config = self.cache_config
        scheduler_config = self.scheduler_config
        parallel_config = self.parallel_config
        self.device = device
159
        self.check_recompilation = envs.VLLM_XLA_CHECK_RECOMPILATION
160

161
162
163
164
165
166
        # SPMD Related
        self.use_spmd = envs.VLLM_XLA_USE_SPMD
        if self.use_spmd:
            num_devices = xr.global_runtime_device_count()
            mesh_shape = (num_devices, 1)
            device_ids = np.array(range(num_devices))
167
            self.mesh = xs.Mesh(device_ids, mesh_shape, ("x", "y"))
168

169
        self.enforce_eager = model_config.enforce_eager
170
171
172
173

        self.num_xla_graphs = 0
        self._update_num_xla_graphs("init")

174
175
        self.pin_memory = is_pin_memory_available()
        self.dtype = self.model_config.dtype
176
        if cache_config.cache_dtype == "auto":
177
178
            model_dtype = self.dtype
            if isinstance(model_dtype, str):
179
                self.kv_cache_dtype = TPU_STR_DTYPE_TO_TORCH_DTYPE[model_dtype]
180
181
            else:
                self.kv_cache_dtype = model_dtype
182
        else:
183
            self.kv_cache_dtype = TPU_STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
184
        self._hidden_states_dtype = self.dtype
185
186
187
188

        self.sliding_window = model_config.get_sliding_window()
        self.block_size = cache_config.block_size
        self.max_model_len = model_config.max_model_len
189
        self.most_model_len = envs.VLLM_TPU_MOST_MODEL_LEN
190
        self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
191
192
193
194
195
        self.num_blocks_per_most_len_req = (
            cdiv(self.most_model_len, self.block_size)
            if self.most_model_len is not None
            else None
        )
196
197
        # InputBatch needs to work with sampling tensors greater than padding
        # to avoid dynamic shapes. Also, avoid suboptimal alignment.
198
        self.max_num_reqs = max(scheduler_config.max_num_seqs, MIN_NUM_SEQS)
199
200
201
        self.num_tokens_paddings = _get_token_paddings(
            min_token_size=16,
            max_token_size=scheduler_config.max_num_batched_tokens,
202
203
            padding_gap=envs.VLLM_TPU_BUCKET_PADDING_GAP,
        )
204
205
206
        # In case `max_num_tokens < max(num_tokens_paddings)` use the actual
        # padded max value to pre-allocate data structures and pre-compile.
        self.max_num_tokens = self.num_tokens_paddings[-1]
207
208
209

        # Model-related.
        self.num_attn_layers = model_config.get_num_layers_by_block_type(
210
211
212
            parallel_config, LayerBlockType.attention
        )
        self.num_query_heads = model_config.get_num_attention_heads(parallel_config)
213
214
215
        self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
        self.head_size = model_config.get_head_size()
        self.hidden_size = model_config.get_hidden_size()
216
        self.vocab_size = model_config.get_vocab_size()
217

218
219
220
        if self.lora_config is not None:
            self.vocab_size += self.lora_config.lora_extra_vocab_size

221
222
223
        # Multi-modal data support
        self.mm_registry = MULTIMODAL_REGISTRY
        self.uses_mrope = model_config.uses_mrope
224
        self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs(
225
226
            model_config
        )
227
228
229
        # TODO: Support M-RoPE (e.g, Qwen2-VL)
        assert not self.uses_mrope, "TPU does not support M-RoPE yet."

230
231
232
233
234
235
236
237
238
239
        self._num_slices_per_kv_cache_update_block = (
            _get_num_slices_per_kv_cache_update_block(
                get_page_size_bytes(
                    block_size=self.block_size,
                    num_kv_heads=self.num_kv_heads,
                    head_size=self.head_size,
                    kv_cache_dtype=self.kv_cache_dtype,
                )
            )
        )
240

241
        # Lazy initialization
242
        self.model: nn.Module  # Set after load_model
243
        self.kv_caches: list[torch.Tensor] = []
244
245
        # mm_hash -> encoder_output
        self.encoder_cache: dict[str, torch.Tensor] = {}
246
247
248

        # Request states.
        self.requests: dict[str, CachedRequestState] = {}
249

250
251
252
253
254
255
256
257
        # Initialize input batch early to avoid AttributeError in _update_states
        self.input_batch = InputBatch(
            max_num_reqs=self.max_num_reqs,
            max_model_len=self.max_model_len,
            max_num_batched_tokens=self.max_num_tokens,
            device=self.device,
            pin_memory=self.pin_memory,
            vocab_size=self.model_config.get_vocab_size(),
258
            block_sizes=[self.block_size],
259
260
        )

261
262
263
        # Cached torch/numpy tensor
        # The pytorch tensor and numpy array share the same buffer.
        # Sometimes the numpy op is faster so we create both.
264
265
266
        self.input_ids_cpu = torch.zeros(
            self.max_num_tokens, dtype=torch.int32, device="cpu"
        )
267

268
269
270
        self.positions_cpu = torch.zeros(
            self.max_num_tokens, dtype=torch.int32, device="cpu"
        )
271
272
        self.positions_np = self.positions_cpu.numpy()
        self.block_table_cpu = torch.zeros(
273
            (self.max_num_reqs, self.max_num_blocks_per_req),
274
            dtype=torch.int32,
275
276
            device="cpu",
        )
277
        # adjust num_reqs to avoid SMEM OOM.
278
279
280
281
282
283
284
285
286
287
        self.num_reqs_most_model_len = (
            min(
                PallasAttentionBackend.get_max_num_seqs(
                    self.most_model_len, self.block_size
                ),
                self.max_num_reqs,
            )
            if self.most_model_len is not None
            else None
        )
288
        self.num_reqs_max_model_len = min(
289
290
291
292
293
294
295
296
297
298
299
            PallasAttentionBackend.get_max_num_seqs(
                self.max_model_len, self.block_size
            ),
            self.max_num_reqs,
        )
        self.query_start_loc_cpu = torch.zeros(
            self.max_num_tokens + 1,
            dtype=torch.int32,
            device="cpu",
            pin_memory=self.pin_memory,
        )
300
301
        self.query_start_loc_np = self.query_start_loc_cpu.numpy()

302
303
304
305
306
307
        self.seq_lens_cpu = torch.zeros(
            self.max_num_tokens,
            dtype=torch.int32,
            device="cpu",
            pin_memory=self.pin_memory,
        )
308
        self.seq_lens_np = self.seq_lens_cpu.numpy()
309

310
311
        # Only relevant for multimodal models
        if self.supports_mm_inputs:
312
313
314
315
316
317
            self.is_mm_embed_cpu = torch.zeros(
                self.max_num_tokens,
                dtype=torch.bool,
                device="cpu",
                pin_memory=self.pin_memory,
            )
318

319
320
        # Range tensor with values [0 .. self.max_num_tokens - 1].
        # Used to initialize positions / context_lens / seq_lens
321
322
        # Keep in int64 to avoid overflow with long context
        self.arange_np = np.arange(self.max_num_tokens, dtype=np.int64)
323
        self.num_reqs_paddings = _get_req_paddings(
324
325
            min_req_size=MIN_NUM_SEQS, max_req_size=self.max_num_reqs
        )
326

327
328
329
330
331
332
        # Layer pairings for cross-layer KV sharing.
        # If an Attention layer `layer_name` is in the keys of this dict, it
        # means this layer will perform attention using the keys and values
        # from the KV cache of `shared_kv_cache_layers[layer_name]`.
        self.shared_kv_cache_layers: dict[str, str] = {}

333
334
335
336
337
        # tensors for structured decoding
        self.grammar_bitmask_cpu = torch.zeros(
            (self.max_num_reqs, cdiv(self.vocab_size, 32)),
            dtype=torch.int32,
            device="cpu",
338
339
            pin_memory=self.pin_memory,
        )
340
341
342
343
        self.require_structured_out_cpu = torch.zeros(
            (self.max_num_reqs, 1),
            dtype=torch.bool,
            device="cpu",
344
345
            pin_memory=self.pin_memory,
        )
346
        self.structured_decode_arange = torch.arange(
347
348
            0, 32, device="cpu", pin_memory=self.pin_memory
        )
349

350
351
352
353
354
355
356
357
358
        self.mm_budget = (
            MultiModalBudget(
                self.model_config,
                self.scheduler_config,
                self.mm_registry,
            )
            if self.supports_mm_inputs
            else None
        )
359

360
361
362
363
364
        if not self.use_spmd:
            self.sample_from_logits_func = torch.compile(
                self.sample_from_logits,
                backend="openxla",
                fullgraph=True,
365
366
                dynamic=False,
            )
367
368
369
        else:
            self.sample_from_logits_func = self.sample_from_logits

370
371
372
373
374
375
376
377
378
379
    def _update_num_xla_graphs(self, case_str):
        check_comp = self.check_recompilation and not self.enforce_eager
        if not check_comp:
            return

        total_cached_graphs = xr.get_num_cached_compilation_graph()
        new_compiled_graphs = total_cached_graphs - self.num_xla_graphs
        if new_compiled_graphs == 0:
            return

380
381
382
        logger.info(
            "Add new %d compiled XLA graphs due to %s", new_compiled_graphs, case_str
        )
383
384
385
386
387
388
389
390
391
392
393
        self.num_xla_graphs += new_compiled_graphs

    def _verify_num_xla_graphs(self, case_str):
        check_comp = self.check_recompilation and not self.enforce_eager
        if not check_comp:
            return

        curr_cached_graph = xr.get_num_cached_compilation_graph()
        assert self.num_xla_graphs == curr_cached_graph, (
            "Recompilation after warm up is detected during {}."
            " num_xla_graphs = {} curr_cached_graph = {}".format(
394
395
396
                case_str, self.num_xla_graphs, curr_cached_graph
            )
        )
397

398
399
400
401
402
403
404
405
    def _update_states(self, scheduler_output: "SchedulerOutput") -> bool:
        """Update the cached states and the persistent batch with the scheduler
        output.

        The updated states are used by the `_prepare_inputs` function to create
        the input GPU tensors for the model.

        Returns:
406
            True if there is a new/resumed/paused/finished request.
407
408
409
410
411
412
413
414
415
416
417
418
            If False, we can skip copying SamplingMetadata to the GPU.
        """
        # Remove finished requests from the cached states.
        for req_id in scheduler_output.finished_req_ids:
            self.requests.pop(req_id, None)

        # Remove the finished requests from the persistent batch.
        # NOTE(woosuk): There could be an edge case where finished_req_ids and
        # scheduled_req_ids overlap. This happens when a request is aborted and
        # then resubmitted with the same ID. In this case, we treat them as two
        # distinct requests - clearing the cached states for the first request
        # and handling the second as a new request.
419
        removed_req_indices: list[int] = []
420
421
422
423
424
        for req_id in scheduler_output.finished_req_ids:
            req_index = self.input_batch.remove_request(req_id)
            if req_index is not None:
                removed_req_indices.append(req_index)

425
        # Free the cached encoder outputs.
426
427
        for mm_hash in scheduler_output.free_encoder_mm_hashes:
            self.encoder_cache.pop(mm_hash, None)
428

429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
        # Remove the unscheduled requests from the persistent batch.
        # NOTE(woosuk): The unscheduled requests are either preempted requests
        # or running requests that are not scheduled in this step. We remove
        # them from the persistent batch but keep their cached states since
        # they will be scheduled again sometime in the future.
        scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys()
        cached_req_ids = self.input_batch.req_id_to_index.keys()
        unscheduled_req_ids = cached_req_ids - scheduled_req_ids
        # NOTE(woosuk): The persistent batch optimization assumes that
        # consecutive batches contain mostly the same requests. If batches
        # have low request overlap (e.g., alternating between two distinct
        # sets of requests), this optimization becomes very inefficient.
        for req_id in unscheduled_req_ids:
            req_index = self.input_batch.remove_request(req_id)
            assert req_index is not None
            removed_req_indices.append(req_index)

446
        req_ids_to_add: list[str] = []
447
448
        # Add new requests to the cached states.
        for new_req_data in scheduler_output.scheduled_new_reqs:
449
            assert new_req_data.sampling_params is not None, (
450
                "Pooling is not supported in TPU yet"
451
            )
452
453
454
455
456
457
            req_id = new_req_data.req_id
            sampling_params = new_req_data.sampling_params

            self.requests[req_id] = CachedRequestState(
                req_id=req_id,
                prompt_token_ids=new_req_data.prompt_token_ids,
458
                prompt_embeds=new_req_data.prompt_embeds,
459
                mm_features=new_req_data.mm_features,
460
                sampling_params=sampling_params,
461
                pooling_params=None,
462
                generator=None,
463
464
465
466
467
468
469
470
471
                block_ids=new_req_data.block_ids,
                num_computed_tokens=new_req_data.num_computed_tokens,
                output_token_ids=[],
                lora_request=new_req_data.lora_request,
            )

            req_ids_to_add.append(req_id)

        # Update the states of the running/resumed requests.
472
473
        req_data = scheduler_output.scheduled_cached_reqs
        for i, req_id in enumerate(req_data.req_ids):
474
            req_state = self.requests[req_id]
475
476
477
            num_computed_tokens = req_data.num_computed_tokens[i]
            new_block_ids = req_data.new_block_ids[i]
            resumed_from_preemption = req_data.resumed_from_preemption[i]
478
479

            # Update the cached states.
480
481
            req_state.num_computed_tokens = num_computed_tokens
            if not resumed_from_preemption:
482
483
                if new_block_ids is not None:
                    # Append the new blocks to the existing block IDs.
484
                    for block_ids, new_ids in zip(req_state.block_ids, new_block_ids):
485
                        block_ids.extend(new_ids)
486
            else:
487
                assert new_block_ids is not None
488
489
                # The request is resumed from preemption.
                # Replace the existing block IDs with the new ones.
490
                req_state.block_ids = new_block_ids
491
492
493
494
495
496
497
498
499
500

            req_index = self.input_batch.req_id_to_index.get(req_id)
            if req_index is None:
                # The request is not in the persistent batch.
                # The request was either preempted and resumed later, or was not
                # scheduled in the previous step and needs to be added again.
                req_ids_to_add.append(req_id)
                continue

            # Update the persistent batch.
501
            self.input_batch.num_computed_tokens_cpu[req_index] = num_computed_tokens
502
            if new_block_ids is not None:
503
                self.input_batch.block_table.append_row(new_block_ids, req_index)
504
505
506
507
508
509

        # Add the new or resumed requests to the persistent batch.
        # The smaller empty indices are filled first.
        removed_req_indices = sorted(removed_req_indices, reverse=True)
        for req_id in req_ids_to_add:
            req_state = self.requests[req_id]
510
511
            # Fill the empty index or append to the end
            req_index = removed_req_indices.pop() if removed_req_indices else None
512
513
514
515
516
            self.input_batch.add_request(req_state, req_index)

        # Condense the batched states if there are empty indices.
        if removed_req_indices:
            self.input_batch.condense(removed_req_indices)
517

518
519
520
521
522
        return len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0

    def get_model(self) -> nn.Module:
        return self.model

523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
    def get_supported_generation_tasks(self) -> list[GenerationTask]:
        model = self.get_model()
        supported_tasks = list[GenerationTask]()

        if is_text_generation_model(model):
            supported_tasks.append("generate")

        if supports_transcription(model):
            if model.supports_transcription_only:
                return ["transcription"]

            supported_tasks.append("transcription")

        return supported_tasks

538
539
540
541
542
    def get_supported_pooling_tasks(self) -> list[PoolingTask]:
        model = self.get_model()
        if not is_pooling_model(model):
            return []

543
        return list(model.pooler.get_supported_tasks())
544

545
546
547
548
549
550
551
552
553
554
    def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
        tasks = list[SupportedTask]()

        if self.model_config.runner_type == "generate":
            tasks.extend(self.get_supported_generation_tasks())
        if self.model_config.runner_type == "pooling":
            tasks.extend(self.get_supported_pooling_tasks())

        return tuple(tasks)

555
    def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
556
        """
557
        Generates the KVCacheSpec by parsing the kv cache format from each
558
559
        Attention module in the static forward context.
        Returns:
560
            KVCacheSpec: A dictionary mapping layer names to their KV cache
561
562
563
            format. Layers that do not need KV cache are not included.
        """

564
        layers = get_layers_from_vllm_config(self.vllm_config, Attention)
565
        block_size = self.vllm_config.cache_config.block_size
566
        kv_cache_spec: dict[str, KVCacheSpec] = {}
567
        for layer_name, attn_module in layers.items():
568
            if (kv_tgt_layer := attn_module.kv_sharing_target_layer_name) is not None:
569
570
571
572
573
574
575
576
577
578
                # The layer doesn't need its own KV cache and will use that of
                # the target layer. We skip creating a KVCacheSpec for it, so
                # that KV cache management logic will act as this layer does
                # not exist, and doesn't allocate KV cache for the layer. This
                # enables the memory saving of cross-layer kv sharing, allowing
                # a given amount of memory to accommodate longer context lengths
                # or enable more requests to be processed simultaneously.
                self.shared_kv_cache_layers[layer_name] = kv_tgt_layer
                continue

579
            if attn_module.attn_type == AttentionType.DECODER:
580
                if isinstance(attn_module, ChunkedLocalAttention):
581
582
                    logger.warning_once(
                        "Using irope in Pallas is not supported yet, it "
583
584
                        "will fall back to global attention for long context."
                    )
585
586
587
588
589
                if attn_module.sliding_window is not None:
                    kv_cache_spec[layer_name] = SlidingWindowSpec(
                        block_size=block_size,
                        num_kv_heads=attn_module.num_kv_heads,
                        head_size=attn_module.head_size,
590
                        dtype=self.kv_cache_dtype,
591
592
593
594
595
596
597
                        sliding_window=attn_module.sliding_window,
                    )
                else:
                    kv_cache_spec[layer_name] = FullAttentionSpec(
                        block_size=block_size,
                        num_kv_heads=attn_module.num_kv_heads,
                        head_size=attn_module.head_size,
598
                        dtype=self.kv_cache_dtype,
599
                    )
600
601
602
603
            elif attn_module.attn_type in (
                AttentionType.ENCODER,
                AttentionType.ENCODER_ONLY,
            ):
604
605
606
607
608
                # encoder-only attention does not need KV cache.
                continue
            elif attn_module.attn_type == AttentionType.ENCODER_DECODER:
                raise NotImplementedError
            else:
609
                raise ValueError(f"Unknown attention type: {attn_module.attn_type}")
610
611
612

        return kv_cache_spec

613
614
615
    def _get_slot_mapping_metadata(
        self, num_reqs, num_scheduled_tokens_per_req
    ) -> np.ndarray:
616
617
618
619
620
621
622
623
624
625
626
627
        """
        Computes metadata for mapping slots to blocks in the key-value (KV)
        cache for a batch of requests.

        This function determines, for each request in the batch, how the
        scheduled tokens are distributed across memory blocks, and generates
        metadata needed to map slices of tokens to their corresponding positions
        in the KV cache.

        Args:
            num_reqs (int): Number of requests in the current batch.
            num_scheduled_tokens_per_req (int or np.ndarray): Number of tokens
628
                to be scheduled for each request.
629
630
631

        Returns:
            np.ndarray: A 2D array of shape (total_block_len, 3), where each row
632
                contains:
633
                - kv_cache_start_index (int): The starting index in the KV cache
634
                  for the corresponding slice.
635
                - new_kv_start_index (int): The starting index in the new KV
636
                  cache for the corresponding slice.
637
638
639
                - slice_len (int): The length of the slice.
        """
        slices_start = self.input_batch.num_computed_tokens_cpu[:num_reqs]
640
641
642
643
        slices_end = (
            self.input_batch.num_computed_tokens_cpu[:num_reqs]
            + num_scheduled_tokens_per_req
        )
644
645
646
647
        local_block_start_idx = slices_start // self.block_size
        local_block_end_idx = (slices_end - 1) // self.block_size
        no_repeat_req_indices = self.arange_np[:num_reqs]
        global_block_start_idx = (
648
649
            no_repeat_req_indices * self.max_num_blocks_per_req + local_block_start_idx
        )
650
651
652
653
654
655
656
        block_lens = local_block_end_idx - local_block_start_idx + 1
        global_block_start_idx = np.repeat(global_block_start_idx, block_lens)
        slice_arange = np.concatenate([self.arange_np[:n] for n in block_lens])
        global_block_indices = global_block_start_idx + slice_arange
        block_table_cpu = self.input_batch.block_table[0].get_cpu_tensor()
        block_numbers = block_table_cpu.flatten()[global_block_indices].numpy()
        total_block_len = np.sum(block_lens)
657
658
659
        slot_mapping_slices = np.repeat(
            np.array([[0, self.block_size]], dtype=np.int32), total_block_len, axis=0
        )
660
661
662
        cu_block_lens = np.zeros(len(block_lens) + 1, dtype=np.int32)
        np.cumsum(block_lens, out=cu_block_lens[1:])
        for req_idx in range(num_reqs):
663
664
665
666
667
668
            slot_mapping_slices[cu_block_lens[req_idx]][0] = (
                slices_start[req_idx] % self.block_size
            )
            slot_mapping_slices[cu_block_lens[req_idx + 1] - 1][1] = (
                slices_end[req_idx] - 1
            ) % self.block_size + 1
669
670
671
        slice_lens = slot_mapping_slices[:, 1] - slot_mapping_slices[:, 0]
        cu_slices_lens = np.zeros(len(slice_lens) + 1, dtype=np.int32)
        np.cumsum(slice_lens, out=cu_slices_lens[1:])
672
673
674
        kv_cache_start_indices = slot_mapping_slices[:, 0] + (
            block_numbers * self.block_size
        )
675
676
        new_kv_start_indices = cu_slices_lens[:-1]
        slot_mapping_metadata = np.stack(
677
678
            [kv_cache_start_indices, new_kv_start_indices, slice_lens], axis=1
        )
679
680
        return slot_mapping_metadata

681
    def _prepare_inputs(self, scheduler_output: "SchedulerOutput", start_index: int):
682
        assert scheduler_output.total_num_scheduled_tokens > 0
683
684
        num_reqs = self.input_batch.num_reqs
        assert num_reqs > 0
685
        assert start_index < num_reqs
686

687
        # Get the number of scheduled tokens for each request.
688
        use_max_model_len = self.most_model_len is None
689
690
        num_scheduled_tokens_per_req = []
        max_num_scheduled_tokens_all_reqs = 0
691
692
693
694
695
        end_index = start_index

        # Use either most_model_len or max_model_len depending on request size.
        for i in range(start_index, num_reqs):
            req_id = self.input_batch.req_ids[i]
696
            assert req_id is not None
697
            num_tokens = scheduler_output.num_scheduled_tokens[req_id]
698
699
            if not use_max_model_len and num_tokens > self.most_model_len:
                use_max_model_len = True
700
            num_scheduled_tokens_per_req.append(num_tokens)
701
702
        if use_max_model_len:
            if len(num_scheduled_tokens_per_req) > self.num_reqs_max_model_len:
703
704
705
                num_scheduled_tokens_per_req = num_scheduled_tokens_per_req[
                    : self.num_reqs_max_model_len
                ]
706
707
708
709
                end_index = start_index + self.num_reqs_max_model_len
            else:
                end_index = num_reqs
        else:
710
711
712
713
            if len(num_scheduled_tokens_per_req) > self.num_reqs_most_model_len:
                num_scheduled_tokens_per_req = num_scheduled_tokens_per_req[
                    : self.num_reqs_most_model_len
                ]
714
715
716
717
                end_index = start_index + self.num_reqs_most_model_len
            else:
                end_index = num_reqs
        max_num_scheduled_tokens_all_reqs = max(num_scheduled_tokens_per_req)
718
719
720
        num_scheduled_tokens_per_req = np.array(
            num_scheduled_tokens_per_req, dtype=np.int32
        )
721
        total_num_scheduled_tokens = sum(num_scheduled_tokens_per_req)
722
723
        assert max_num_scheduled_tokens_all_reqs > 0

724
725
        num_reqs = len(num_scheduled_tokens_per_req)

726
727
728
        # Get request indices.
        # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2]
        # For each scheduled token, what are the corresponding req index.
729
        req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens_per_req)
730
731
732
733
734

        # Get batched arange.
        # E.g., [2, 5, 3] -> [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
        # For each scheduled token, what is its position in corresponding req.
        arange = np.concatenate(
735
736
            [self.arange_np[:n] for n in num_scheduled_tokens_per_req]
        )
737
738
739

        # Get positions.
        positions_np = self.positions_np[:total_num_scheduled_tokens]
740
741
742
743
744
        np.add(
            self.input_batch.num_computed_tokens_cpu[req_indices],
            arange,
            out=positions_np,
        )
745
746
747
748
749

        # Get token indices.
        # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]
        # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2]
        # where M is the max_model_len.
750
751
752
        token_indices = (
            positions_np + req_indices * self.input_batch.token_ids_cpu.shape[1]
        )
753
754
755
756

        # NOTE(woosuk): We use torch.index_select instead of np.take here
        # because torch.index_select is much faster than np.take for large
        # tensors.
757
758
759
760
761
762
        torch.index_select(
            self.input_batch.token_ids_cpu_tensor.flatten(),
            0,
            torch.from_numpy(token_indices),
            out=self.input_ids_cpu[:total_num_scheduled_tokens],
        )
763
764
765

        # Prepare the attention metadata.
        self.query_start_loc_np[0] = 0
766
767
768
769
        np.cumsum(
            num_scheduled_tokens_per_req, out=self.query_start_loc_np[1 : num_reqs + 1]
        )
        self.query_start_loc_np[num_reqs + 1 :] = 1
770
771

        self.seq_lens_np[:num_reqs] = (
772
773
774
            self.input_batch.num_computed_tokens_cpu[:num_reqs]
            + num_scheduled_tokens_per_req
        )
775
776

        # Do the padding and copy the tensors to the TPU.
777
        padded_total_num_scheduled_tokens = _get_padded_token_len(
778
779
            self.num_tokens_paddings, total_num_scheduled_tokens
        )
780
781
        # Zero out to avoid spurious values from prev iteration (last cp chunk)
        self.input_ids_cpu[
782
783
784
785
786
787
788
789
            total_num_scheduled_tokens:padded_total_num_scheduled_tokens
        ] = 0
        self.input_ids = self.input_ids_cpu[:padded_total_num_scheduled_tokens].to(
            self.device
        )
        self.position_ids = self.positions_cpu[:padded_total_num_scheduled_tokens].to(
            self.device
        )
790
        if use_max_model_len:
791
792
793
794
795
796
797
798
799
800
            block_tables = self.block_table_cpu[
                : self.num_reqs_max_model_len, : self.max_num_blocks_per_req
            ]
            block_tables[:num_reqs, : self.max_num_blocks_per_req] = (
                self.input_batch.block_table[0].get_cpu_tensor()[:num_reqs]
            )
            query_start_loc = self.query_start_loc_cpu[
                : self.num_reqs_max_model_len + 1
            ].to(self.device)
            seq_lens = self.seq_lens_cpu[: self.num_reqs_max_model_len].to(self.device)
801
        else:
802
803
804
805
806
807
808
809
810
811
812
813
            block_tables = self.block_table_cpu[
                : self.num_reqs_most_model_len, : self.num_blocks_per_most_len_req
            ]
            block_tables[:num_reqs, : self.num_blocks_per_most_len_req] = (
                self.input_batch.block_table[0].get_cpu_tensor()[
                    :num_reqs, : self.num_blocks_per_most_len_req
                ]
            )
            query_start_loc = self.query_start_loc_cpu[
                : self.num_reqs_most_model_len + 1
            ].to(self.device)
            seq_lens = self.seq_lens_cpu[: self.num_reqs_most_model_len].to(self.device)
814
        block_tables = block_tables.to(self.device)
815

816
        # Calculate the slot mapping
817
        slot_mapping_metadata = self._get_slot_mapping_metadata(
818
819
            num_reqs, num_scheduled_tokens_per_req
        )
820
        num_kv_update_slices = slot_mapping_metadata.shape[0]
821
        padded_num_slices = _get_padded_num_kv_cache_update_slices(
822
823
            padded_total_num_scheduled_tokens, self.max_num_reqs, self.block_size
        )
824
825
826
        slot_mapping_metadata = np.pad(
            slot_mapping_metadata,
            [[0, padded_num_slices - len(slot_mapping_metadata)], [0, 0]],
827
828
            constant_values=0,
        )
829
        slot_mapping_metadata = np.transpose(slot_mapping_metadata)
830
        slot_mapping_metadata = torch.tensor(slot_mapping_metadata, device=self.device)
831

832
833
834
835
836
        if self.lora_config is not None:
            # We need to respect padding when activating LoRA adapters
            padded_num_scheduled_tokens_per_req = np.copy(
                num_scheduled_tokens_per_req
            )  # Copying to avoid accidental state corruption bugs
837
            padded_num_scheduled_tokens_per_req[-1] += (
838
                padded_total_num_scheduled_tokens - total_num_scheduled_tokens
839
            )
840

841
            self.set_active_loras(self.input_batch, padded_num_scheduled_tokens_per_req)
842

843
        attn_metadata = PallasMetadata(
844
            slot_mapping=slot_mapping_metadata,
845
            block_tables=block_tables,
846
847
            context_lens=seq_lens,
            query_start_loc=query_start_loc,
848
849
850
851
852
            num_seqs=torch.tensor([num_reqs], dtype=torch.int32, device=self.device),
            num_kv_update_slices=torch.tensor(
                [num_kv_update_slices], dtype=torch.int32, device=self.device
            ),
            num_slices_per_kv_cache_update_block=self._num_slices_per_kv_cache_update_block,
853
        )
854
855
856
857
858
        # NOTE(woosuk): Due to chunked prefills, there can be at most 1 partial
        # request in the batch. While we should not sample any token from this
        # partial request, we do so for simplicity. We will ignore the sampled
        # token from the partial request.
        # TODO: Support prompt logprobs.
859
        padded_num_reqs = _get_padded_num_reqs_with_upper_limit(
860
861
            num_reqs, self.max_num_reqs
        )
862
863
        # Indices at which we sample (positions of last token in the sequence).
        # Padded to avoid recompiling when `num_reqs` varies.
864
        logits_indices = self.query_start_loc_cpu[1 : padded_num_reqs + 1] - 1
865
        logits_indices = logits_indices.to(self.device)
866

867
868
869
870
871
        if self.lora_config is not None:
            # We need to respect padding when activating LoRA adapters
            padded_num_scheduled_tokens_per_req = np.copy(
                num_scheduled_tokens_per_req
            )  # Copying to avoid accidental state corruption bugs
872
            padded_num_scheduled_tokens_per_req[-1] += (
873
                padded_total_num_scheduled_tokens - total_num_scheduled_tokens
874
            )
875

876
            self.set_active_loras(self.input_batch, padded_num_scheduled_tokens_per_req)
877

878
        layer_names = get_layers_from_vllm_config(self.vllm_config, Attention).keys()
879
        per_layer_attn_metadata = {
880
            layer_name: attn_metadata for layer_name in layer_names
881
        }
882
883
884
885
886
887
888
        return (
            per_layer_attn_metadata,
            logits_indices,
            padded_num_reqs,
            num_reqs,
            end_index,
        )
889

890
    def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):
891
892
893
894
895
        scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
        if not scheduled_encoder_inputs:
            return

        # Batch the multi-modal inputs.
896
        mm_kwargs = list[MultiModalKwargsItem]()
897
898
        # List of tuple (mm_hash, pos_info)
        mm_hashes_pos = list[tuple[str, PlaceholderRange]]()
899
900
        for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
            req_state = self.requests[req_id]
901
902

            for mm_input_id in encoder_input_ids:
903
904
905
906
                mm_feature = req_state.mm_features[mm_input_id]
                mm_hash = mm_feature.identifier
                mm_kwargs.append(mm_feature.data)
                mm_hashes_pos.append((mm_hash, mm_feature.mm_position))
907
908
909
910
911
912
913
914

        # Batch mm inputs as much as we can: if a request in the batch has
        # multiple modalities or a different modality than the previous one,
        # we process it separately to preserve item order.
        # FIXME(ywang96): This is a hacky way to deal with multiple modalities
        # in the same batch while still being able to benefit from batching
        # multimodal inputs. The proper solution should be reordering the
        # encoder outputs.
915
        model = cast(SupportsMultiModal, self.model)
916
        encoder_outputs = []
917
        for _, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
918
919
920
921
            mm_kwargs,
            device=self.device,
            pin_memory=self.pin_memory,
            merge_by_field_config=model.merge_by_field_config,
922
        ):
923
924
925
926
927
928
929
            # Run the encoder.
            # `curr_group_outputs` is either of the following:
            # 1. A tensor of shape (num_items, feature_size, hidden_size)
            # in case feature_size is fixed across all multimodal items.
            # 2. A list or tuple (length: num_items) of tensors, each of shape
            # (feature_size, hidden_size) in case the feature size is dynamic
            # depending on the input multimodal items.
930
            torch_xla.sync(wait=False)
931
            curr_group_outputs = model.get_multimodal_embeddings(**mm_kwargs_group)
932
            torch_xla.sync(wait=False)
933

934
935
            sanity_check_mm_encoder_outputs(
                curr_group_outputs,
936
                expected_num_items=num_items,
937
938
            )

939
940
941
942
943
944
            if isinstance(curr_group_outputs, torch.Tensor):
                encoder_outputs.append(curr_group_outputs)
            else:
                assert isinstance(curr_group_outputs, (list, tuple))
                for output in curr_group_outputs:
                    encoder_outputs.append(output)
945
946

        # Cache the encoder outputs.
947
948
949
        # NOTE (NickLucche) here we diverge from logic in other runners, as we
        # assume to only have whole mm items to process. Hence we avoid the
        # intrinsic dynamism that `scatter_mm_placeholders` introduces.
950
        for (mm_hash, pos_info), output in zip(mm_hashes_pos, encoder_outputs):
951
952
953
            assert pos_info.is_embed is None, (
                "Expected all positions to be contiguous and embeddings."
            )
954
            self.encoder_cache[mm_hash] = output
955
956

    def _gather_mm_embeddings(
957
958
        self,
        scheduler_output: "SchedulerOutput",
959
960
961
    ) -> tuple[list[torch.Tensor], torch.Tensor]:
        total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
        padded_total_num_scheduled_tokens = _get_padded_token_len(
962
963
            self.num_tokens_paddings, total_num_scheduled_tokens
        )
964
965
966
967
968
969

        is_mm_embed = self.is_mm_embed_cpu
        is_mm_embed[:padded_total_num_scheduled_tokens] = False
        mm_embeds = list[torch.Tensor]()
        req_start_idx = 0

970
        for req_id in self.input_batch.req_ids:
971
            num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
972
973
            req_state = self.requests[req_id]
            num_computed_tokens = req_state.num_computed_tokens
974

975
976
977
978
            # TODO unroll loop and assume/enforce --disable_chunked_mm_input
            # NOTE (NickLucche) here we diverge from logic in other runners, as
            # we assume to only have whole mm items to process. Hence we avoid
            # the intrinsic dynamism that `gather_mm_placeholders` introduces.
979
980
            for mm_feature in req_state.mm_features:
                pos_info = mm_feature.mm_position
981
982
                start_pos = pos_info.offset
                num_encoder_tokens = pos_info.length
983
984
985
986
987
988
989
990
991
992
993
994

                # The encoder output is needed if the two ranges overlap:
                # [num_computed_tokens,
                #  num_computed_tokens + num_scheduled_tokens) and
                # [start_pos, start_pos + num_encoder_tokens)
                if start_pos >= num_computed_tokens + num_scheduled_tokens:
                    # The encoder output is not needed in this step.
                    break
                if start_pos + num_encoder_tokens <= num_computed_tokens:
                    # The encoder output is already processed and stored
                    # in the decoder's KV cache.
                    continue
995
996
997
998
999
1000
1001
1002

                start_idx = max(num_computed_tokens - start_pos, 0)
                end_idx = min(
                    num_computed_tokens - start_pos + num_scheduled_tokens,
                    num_encoder_tokens,
                )
                assert start_idx < end_idx

1003
                mm_hash = mm_feature.identifier
1004
                encoder_output = self.encoder_cache.get(mm_hash, None)
1005
                assert encoder_output is not None, f"Encoder cache miss for {mm_hash}."
1006

1007
1008
1009
                assert pos_info.is_embed is None, (
                    "Expected all positions to be contiguous and embeddings."
                )
1010
1011

                req_start_pos = req_start_idx + start_pos - num_computed_tokens
1012
                is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] = True
1013
1014

                # Only whole mm items are processed
1015
                mm_embeds.append(encoder_output)
1016

1017
1018
            req_start_idx += num_scheduled_tokens

1019
        is_mm_embed = is_mm_embed[:padded_total_num_scheduled_tokens].to(self.device)
1020
1021
1022
1023
1024
1025
1026
1027

        return mm_embeds, is_mm_embed

    def _get_model_inputs(
        self,
        input_ids: torch.Tensor,
        mm_embed_inputs: Optional[tuple[list[torch.Tensor], torch.Tensor]],
    ):
1028
        if self.supports_mm_inputs:
1029
1030
            mm_embeds, is_mm_embed = mm_embed_inputs or (None, None)

1031
1032
1033
            # NOTE(woosuk): To unify token ids and soft tokens (vision
            # embeddings), we always use embeddings (rather than token ids)
            # as input to the multimodal model, even when the input is text.
1034
            inputs_embeds = self.model.get_input_embeddings(
1035
                input_ids,
1036
                multimodal_embeddings=mm_embeds,
1037
                is_multimodal=is_mm_embed,
1038
            )
1039

1040
1041
1042
1043
1044
1045
1046
1047
            return None, inputs_embeds
        else:
            # For text-only models, we use token ids as input.
            # While it is possible to use embeddings as input just like the
            # multimodal models, it is not desirable for performance since
            # then the embedding layer is not included in the CUDA graph.
            return input_ids, None

1048
1049
1050
1051
    @torch.no_grad()
    def execute_model(
        self,
        scheduler_output: "SchedulerOutput",
1052
        intermediate_tensors: Optional[IntermediateTensors] = None,
1053
1054
1055
    ) -> ModelRunnerOutput:
        # Update cached state
        self._update_states(scheduler_output)
1056
        if not scheduler_output.total_num_scheduled_tokens:
1057
1058
1059
1060
            if not has_kv_transfer_group():
                # Return empty ModelRunnerOutput if there's no work to do.
                return EMPTY_MODEL_RUNNER_OUTPUT

1061
            return self.kv_connector_no_forward(scheduler_output, self.vllm_config)
1062

1063
        if self.supports_mm_inputs:
1064
            # Run the multimodal encoder if any.
1065
            self._execute_mm_encoder(scheduler_output)
1066
            mm_embed_inputs = self._gather_mm_embeddings(scheduler_output)
1067
        else:
1068
1069
            mm_embed_inputs = None

1070
        torch_xla.sync(wait=False)
1071
        # Prepare inputs, the requests might be split into multiple
1072
1073
1074
1075
        # executions, combine the result of each execution.
        start_index = 0
        combined_selected_tokens: list[torch.Tensor] = []
        combined_logprobs: list[LogprobsLists] = []
1076
1077
1078
1079
1080
1081

        # NOTE: setup current batch's metadata for kv connector.
        # Currently, only verified with NixlConnector
        with set_forward_context(None, self.vllm_config):
            self.maybe_setup_kv_connector(scheduler_output)

1082
        while start_index < self.input_batch.num_reqs:
1083
1084
1085
            attn_metadata, logits_indices, padded_num_reqs, num_reqs, end_index = (
                self._prepare_inputs(scheduler_output, start_index)
            )
1086
            input_ids, inputs_embeds = self._get_model_inputs(
1087
1088
                self.input_ids, mm_embed_inputs
            )
1089
            torch_xla.sync(wait=False)
1090
1091
            # Run the decoder
            with set_forward_context(
1092
1093
1094
1095
                attn_metadata,
                self.vllm_config,
                num_tokens=scheduler_output.total_num_scheduled_tokens,
            ):
1096
1097
1098
1099
1100
                hidden_states = self.model(
                    input_ids=input_ids,
                    positions=self.position_ids,
                    inputs_embeds=inputs_embeds,
                )
1101
            hidden_states = self.select_hidden_states(hidden_states, logits_indices)
1102
            logits = self.compute_logits(hidden_states)
1103
1104
1105
            tpu_sampling_metadata = TPUSupportedSamplingMetadata.from_input_batch(
                self.input_batch, padded_num_reqs, self.device
            )
1106
            if scheduler_output.grammar_bitmask is not None:
1107
1108
1109
1110
1111
1112
                require_struct_decoding, grammar_bitmask_padded, arange = (
                    self.prepare_structured_decoding_input(logits, scheduler_output)
                )
                logits = self.structured_decode(
                    require_struct_decoding, grammar_bitmask_padded, logits, arange
                )
1113
            selected_token_ids = self.sample_from_logits_func(
1114
1115
                logits, tpu_sampling_metadata
            )
1116
1117
1118
1119
            # NOTE (NickLucche) Use the original logits (before any penalties or
            # temperature scaling) for the top-k logprobs. We can't enforce it
            # due to recompilations outside torch.compiled code, so just make
            # sure `sample_from_logits` does not modify the logits in-place.
1120
1121
1122
1123
1124
            logprobs = (
                self.gather_logprobs(logits, selected_token_ids)
                if tpu_sampling_metadata.logprobs
                else None
            )
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134

            # Remove padding on cpu and keep dynamic op outside of xla graph.
            selected_token_ids = selected_token_ids.cpu()[:num_reqs]

            combined_selected_tokens.append(selected_token_ids)
            if tpu_sampling_metadata.logprobs:
                combined_logprobs.append(logprobs.tolists())

            start_index = end_index

1135
1136
1137
1138
1139
        # NOTE: current kv load and save get h2d/d2h copies involved.
        # Those copies are blocking. Once they become async., kv_save
        # should be called right after each single forward pass,
        # instead of the forwards of the entire input batch.
        self.maybe_wait_for_kv_save()
1140
1141
1142
        finished_sending, finished_recving = self.get_finished_kv_transfers(
            scheduler_output
        )
1143

1144
1145
1146
1147
1148
1149
1150
1151
1152
        selected_token_ids = torch.cat(combined_selected_tokens, dim=0)
        if tpu_sampling_metadata.logprobs:

            def concat_lists(input_lists):
                result = []
                for input_list in input_lists:
                    result.extend(input_list)
                return result

1153
1154
1155
1156
1157
1158
1159
1160
1161
            logprobs_lists = LogprobsLists(
                logprob_token_ids=concat_lists(
                    [lp.logprob_token_ids for lp in combined_logprobs]
                ),
                logprobs=concat_lists([lp.logprobs for lp in combined_logprobs]),
                sampled_token_ranks=concat_lists(
                    [lp.sampled_token_ranks for lp in combined_logprobs]
                ),
            )
1162
1163
        else:
            logprobs_lists = None
1164

1165
1166
        # Update the cache state concurrently. Code above will not block until
        # we use `selected_token_ids`. Add mark_step if post-processing changes
1167
        request_seq_lens: list[tuple[int, CachedRequestState, int]] = []
1168
        discard_sampled_tokens_req_indices = []
1169
        num_reqs = self.input_batch.num_reqs
1170
1171
1172
        for i, req_id in zip(range(num_reqs), self.input_batch.req_ids):
            assert req_id is not None
            req_state = self.requests[req_id]
1173
1174
1175
1176
            seq_len = (
                req_state.num_computed_tokens
                + scheduler_output.num_scheduled_tokens[req_id]
            )
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
            if seq_len >= req_state.num_tokens:
                request_seq_lens.append((i, req_state, seq_len))
            else:
                # Ignore the sampled token from the partial request.
                # Rewind the generator state as if the token was not sampled.
                generator = self.input_batch.generators.get(i)
                if generator is not None:
                    # This relies on cuda-specific torch-internal impl details
                    generator.set_offset(generator.get_offset() - 4)

1187
1188
1189
1190
                # Record the index of the request that should not be sampled,
                # so that we could clear the sampled tokens before returning.
                discard_sampled_tokens_req_indices.append(i)

1191
        assert all(
1192
1193
            req_id is not None for req_id in self.input_batch.req_ids[:num_reqs]
        ), "req_ids contains None"
1194
        req_ids = cast(list[str], self.input_batch.req_ids[:num_reqs])
1195

1196
        prompt_logprobs_dict: dict[str, Optional[LogprobsTensors]] = {}
1197
        for req_id in self.input_batch.req_ids[:num_reqs]:
1198
1199
            prompt_logprobs_dict[req_id] = None

1200
1201
1202
        max_gen_len = selected_token_ids.shape[-1]
        if max_gen_len == 1:
            valid_sampled_token_ids = selected_token_ids.tolist()
1203

1204
1205
1206
1207
1208
1209
1210
            # Mask out the sampled tokens that should not be sampled.
            # TODO: Keep in sync with gpu_model_runner.py, in particular
            #       the "else" case here
            for i in discard_sampled_tokens_req_indices:
                valid_sampled_token_ids[i].clear()

            # Append sampled tokens
1211
1212
1213
1214
1215
            for i, req_state, seq_len in request_seq_lens:
                token_id = valid_sampled_token_ids[i][0]
                self.input_batch.token_ids_cpu[i, seq_len] = token_id
                req_state.output_token_ids.append(token_id)
                self.input_batch.num_tokens[i] += 1
1216

1217
1218
1219
1220
        else:
            valid_mask = selected_token_ids != INVALID_TOKEN_ID
            gen_lens = valid_mask.sum(dim=1).tolist()
            valid_sampled_token_ids = [
1221
                seq.tolist() for seq in selected_token_ids[valid_mask].split(gen_lens)
1222
1223
1224
1225
            ]
            self.input_batch.num_tokens[:num_reqs] += gen_lens
            for i, req_state, seq_len in request_seq_lens:
                target_slice = slice(seq_len - gen_lens[i] + 1, seq_len + 1)
1226
1227
1228
                self.input_batch.token_ids_cpu[i, target_slice] = (
                    valid_sampled_token_ids[i]
                )
1229
1230
                req_state.output_token_ids.extend(valid_sampled_token_ids[i])

1231
1232
1233
1234
        kv_connector_output = (
            None
            if (finished_sending is None and finished_recving is None)
            else KVConnectorOutput(
1235
1236
1237
                finished_sending=finished_sending,
                finished_recving=finished_recving,
            )
1238
        )
1239

1240
        model_runner_output = ModelRunnerOutput(
1241
            req_ids=req_ids,
1242
            req_id_to_index=self.input_batch.req_id_to_index,
1243
            sampled_token_ids=valid_sampled_token_ids,
1244
            logprobs=logprobs_lists,
1245
            prompt_logprobs_dict=prompt_logprobs_dict,
1246
            pooler_output=[],
1247
1248
            kv_connector_output=kv_connector_output,
        )
1249
1250
1251
1252
1253

        # Check there are no new graphs compiled - all the graphs should be
        # captured and compiled during warm up.
        self._verify_num_xla_graphs("execute_model")

1254
1255
        return model_runner_output

1256
1257
1258
1259
1260
    def update_config(self, overrides: dict[str, Any]) -> None:
        # TODO: TPU config may need extra validation
        # https://github.com/vllm-project/vllm/pull/20095#discussion_r2201497754
        allowed_config_names = {"load_config", "model_config"}
        for config_name, config_overrides in overrides.items():
1261
1262
            assert config_name in allowed_config_names, (
                f"Config `{config_name}` not supported. "
1263
                f"Allowed configs: {allowed_config_names}"
1264
            )
1265
1266
1267
1268
            config = getattr(self, config_name)
            new_config = update_config(config, config_overrides)
            setattr(self, config_name, new_config)

1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
    def load_model(self) -> None:
        self.device = self.device_config.device

        # NOTE(woosuk): While the executor assigns the TP ranks to the worker
        # process, the ranks can be different from the ranks internally assigned
        # by the xm runtime. Therefore, there is a mismatch in the rank
        # assignment between the gloo (cpu) runtime and the xm (tpu) runtime.
        # This is not a problem in linear layers because all-reduce is
        # rank-agnostic. However, it matters for all-gather as the ranks
        # determine the order of concatenating the output tensors.
        # As a workaround, we use the xm's rank assignment only when loading
        # the embedding weights.
        xm_tp_rank = xr.global_ordinal()
        with patch(
1283
1284
1285
1286
            "vllm.model_executor.layers.vocab_parallel_embedding."
            "get_tensor_model_parallel_rank",
            return_value=xm_tp_rank,
        ):
1287
1288
1289
            try:
                if self.use_spmd:
                    tpu_loader = TPUModelLoader(
1290
1291
                        load_config=self.vllm_config.load_config
                    )
1292
                    model = tpu_loader.load_model(
1293
                        vllm_config=self.vllm_config,
1294
                        model_config=self.vllm_config.model_config,
1295
1296
                        mesh=self.mesh,
                    )
1297
                else:
1298
                    model_loader = get_model_loader(self.load_config)
1299
1300
                    logger.info("Loading model from scratch...")
                    model = model_loader.load_model(
1301
1302
                        vllm_config=self.vllm_config, model_config=self.model_config
                    )
1303
1304
1305
1306
1307
1308
            except RuntimeError as e:
                raise RuntimeError(
                    f"Unable to load model, a likely reason is the model is "
                    "too large for the current device's HBM memory. "
                    "Consider switching to a smaller model "
                    "or sharding the weights on more chips. "
1309
1310
                    f"See the detailed error: {e}"
                ) from e
1311
        if self.lora_config is not None:
1312
            model = self.load_lora_model(model, self.vllm_config, self.device)
1313
            replace_set_lora(model)
1314

1315
1316
        # Sync all pending XLA execution during model initialization and weight
        # loading.
1317
        torch_xla.sync(wait=False)
1318
        xm.wait_device_ops()
1319
1320
        if not hasattr(self, "model"):
            self.model = model
1321
        self.sampler = TPUSampler()
1322

1323
    def reload_weights(self) -> None:
1324
        assert getattr(self, "model", None) is not None, (
1325
            "Cannot reload weights before model is loaded."
1326
        )
1327
1328
1329
1330
        model_loader = get_model_loader(self.load_config)
        logger.info("Reloading weights inplace...")
        model_loader.load_weights(self.model, model_config=self.model_config)

1331
    @torch.no_grad()
1332
    def _dummy_run(self, num_tokens: int, num_reqs: int, num_blocks: int) -> None:
1333
        if self.supports_mm_inputs:
1334
            input_ids = None
1335
1336
1337
            inputs_embeds = torch.zeros(
                (num_tokens, self.hidden_size), dtype=self.dtype, device=self.device
            )
1338
        else:
1339
            input_ids = torch.zeros((num_tokens), dtype=torch.int32).to(self.device)
1340
            inputs_embeds = None
1341
        actual_num_reqs = min(num_tokens, num_reqs)
1342
        position_ids = torch.zeros(num_tokens, dtype=torch.int32).to(self.device)
1343
        padded_num_slices = _get_padded_num_kv_cache_update_slices(
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
            num_tokens, self.max_num_reqs, self.block_size
        )
        num_kv_update_slices = torch.tensor([padded_num_slices], dtype=torch.int32).to(
            self.device
        )
        slot_mapping = torch.zeros((3, padded_num_slices), dtype=torch.int32).to(
            self.device
        )
        block_tables = torch.zeros((num_reqs, num_blocks), dtype=torch.int32).to(
            self.device
        )
1355
        query_lens = [1] * num_reqs
1356
1357
1358
1359
1360
        query_start_loc = torch.cumsum(
            torch.tensor([0] + query_lens, dtype=torch.int32), dim=0, dtype=torch.int32
        ).to(self.device)
        context_lens = torch.ones((num_reqs,), dtype=torch.int32).to(self.device)
        num_seqs = torch.tensor([actual_num_reqs], dtype=torch.int32).to(self.device)
1361
1362
1363
1364
1365
        attn_metadata = PallasMetadata(
            slot_mapping=slot_mapping,
            block_tables=block_tables,
            context_lens=context_lens,
            query_start_loc=query_start_loc,
1366
            num_seqs=num_seqs,
1367
            num_kv_update_slices=num_kv_update_slices,
1368
            num_slices_per_kv_cache_update_block=self._num_slices_per_kv_cache_update_block,
1369
        )
1370

1371
        if self.supports_mm_inputs:
1372
1373
1374
            torch._dynamo.mark_dynamic(inputs_embeds, 0)
        else:
            torch._dynamo.mark_dynamic(input_ids, 0)
1375
1376
        torch._dynamo.mark_dynamic(position_ids, 0)
        torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
1377
1378
1379
        torch._dynamo.mark_dynamic(attn_metadata.block_tables, (0, 1))
        torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
        torch._dynamo.mark_dynamic(attn_metadata.query_start_loc, 0)
1380

1381
        layer_names = get_layers_from_vllm_config(self.vllm_config, Attention).keys()
1382
        per_layer_attn_metadata = {
1383
            layer_name: attn_metadata for layer_name in layer_names
1384
1385
        }

1386
1387
1388
1389
1390
1391
1392
1393
1394
        with (
            self.maybe_select_dummy_loras(
                self.lora_config, np.array([num_tokens], dtype=np.int32)
            ),
            set_forward_context(per_layer_attn_metadata, self.vllm_config, 0),
        ):
            out = self.model(
                input_ids=input_ids, positions=position_ids, inputs_embeds=inputs_embeds
            )
1395
        self._hidden_states_dtype = out.dtype
1396

1397
1398
1399
    def _set_active_loras(
        self, prompt_lora_mapping, token_lora_mapping, lora_requests
    ) -> None:
1400
        torch_xla.sync(wait=False)  # Captures input updates
1401
1402
1403
        super()._set_active_loras(
            prompt_lora_mapping, token_lora_mapping, lora_requests
        )
1404
        torch_xla.sync(wait=False)  # Captures metadata updates
1405

1406
    def _precompile_mm_encoder(self) -> None:
1407
        if not self.supports_mm_inputs:
1408
1409
            return

1410
1411
        # Pre-compile MM encoder for all supported data modalities.
        hf_config = self.vllm_config.model_config.hf_config
1412
1413
1414
1415
1416
1417
1418

        mm_budget = self.mm_budget
        assert mm_budget is not None

        max_items_per_seq_by_modality = mm_budget.max_items_per_batch_by_modality  # noqa: E501

        for mode, max_items_per_seq in max_items_per_seq_by_modality.items():
1419
            logger.info(
1420
1421
                "Compiling Multimodal %s Encoder with different input shapes.", mode
            )
1422
1423
            start = time.perf_counter()
            # No padding for MM encoder just yet.
1424
            for num_items in range(1, max_items_per_seq + 1):
1425
1426
                logger.info("  -- mode: %s items: %d", mode, num_items)
                batched_dummy_mm_inputs = self._get_mm_dummy_batch(
1427
1428
1429
                    mode,
                    num_items,
                )
1430
                # Run multimodal encoder.
1431
                torch_xla.sync(wait=False)
1432
                mm_embeds = self.model.get_multimodal_embeddings(
1433
1434
                    **batched_dummy_mm_inputs
                )
1435
                torch_xla.sync(wait=False)
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
                num_patches = mm_embeds[0].shape[0]
                items_size = num_patches * num_items

                # NOTE (NickLucche) pre-compile `get_input_embeddings` when mm
                # embeddings are present. We assume `--disable-mm-chunked`,
                # hence only whole items can be scheduled. This implies we just
                # need to compile when `num_items` fit the (padded) `input_ids`
                for num_tokens in self.num_tokens_paddings:
                    if num_tokens >= items_size:
                        # XLA Workaround: if torch.zeros(..device) is used, XLA
                        # compiles a scalar+expansion op, which won't match
                        # the graph generated at runtime. CPU->TPU must be used
1448
1449
1450
                        placeholders_ids = torch.zeros(
                            num_tokens, dtype=torch.int32, device="cpu"
                        )
1451
                        # Align placeholders and actual num mm_embeddings.
1452
                        placeholders_ids[:items_size] = hf_config.image_token_index
1453
1454

                        placeholders_ids = placeholders_ids.to(self.device)
1455
1456
1457
1458

                        mm_mask = torch.tensor([False] * num_tokens)
                        mm_mask[:items_size] = True
                        mm_mask = mm_mask.to(self.device)
1459
                        # Assign outputs or the graph will be cut short.
1460
1461
1462
1463
                        a, b = self._get_model_inputs(
                            placeholders_ids,
                            mm_embed_inputs=([mm_embeds], mm_mask),
                        )
1464
                        assert a is None
1465
                        torch_xla.sync(wait=False)
1466
1467
1468
1469

            # Pre-compile `get_input_embeddings` when mm_embeddings are not
            # present. Chunk is only made of text, no mm_placeholders.
            for num_tokens in self.num_tokens_paddings:
1470
1471
1472
                placeholders_ids = torch.zeros(
                    num_tokens, dtype=torch.int32, device="cpu"
                )
1473
                placeholders_ids = placeholders_ids.to(self.device)
1474
1475
1476
1477
                a, b = self._get_model_inputs(
                    placeholders_ids,
                    mm_embed_inputs=None,
                )
1478
                assert a is None
1479
                torch_xla.sync(wait=False)
1480
1481
1482
1483

            xm.wait_device_ops()
            end = time.perf_counter()
            logger.info(
1484
1485
1486
1487
                "Multimodal %s Encoder compilation finished in in %.2f [secs].",
                mode,
                end - start,
            )
1488

1489
    def _precompile_backbone(self) -> None:
1490
1491
        logger.info("Compiling the model with different input shapes.")
        start = time.perf_counter()
1492
        for num_tokens in self.num_tokens_paddings:
1493
            logger.info("  -- num_tokens: %d", num_tokens)
1494
1495
1496
            self._dummy_run(
                num_tokens, self.num_reqs_max_model_len, self.max_num_blocks_per_req
            )
1497
            if self.most_model_len is not None:
1498
1499
1500
1501
1502
                self._dummy_run(
                    num_tokens,
                    self.num_reqs_most_model_len,
                    self.num_blocks_per_most_len_req,
                )
1503
1504
        xm.wait_device_ops()
        end = time.perf_counter()
1505
        logger.info("Compilation finished in %.2f [secs].", end - start)
1506
        self._update_num_xla_graphs("model backbone")
1507

1508
1509
1510
    def _precompile_select_hidden_states(self) -> None:
        # Compile hidden state selection function for bucketed
        # n_tokens x max_num_reqs. Graph is really small so this is fine.
1511
        logger.info("Compiling select_hidden_states with different input shapes.")
1512
1513
        start = time.perf_counter()
        hsize = self.model_config.get_hidden_size()
1514
        for num_tokens in self.num_tokens_paddings:
1515
1516
1517
            dummy_hidden = torch.zeros(
                (num_tokens, hsize), device=self.device, dtype=self._hidden_states_dtype
            )
1518
1519
            torch._dynamo.mark_dynamic(dummy_hidden, 0)
            for num_reqs in self.num_reqs_paddings:
1520
                indices = torch.zeros(num_reqs, dtype=torch.int32, device=self.device)
1521
1522
                torch._dynamo.mark_dynamic(indices, 0)
                self.select_hidden_states(dummy_hidden, indices)
1523
                logger.info("  -- num_tokens: %d, num_seqs: %d", num_tokens, num_reqs)
1524
1525
1526
1527
                # Requests can't be more than tokens. But do compile for the
                # next bigger value in case num_tokens uses bucketed padding.
                if num_reqs >= min(num_tokens, self.max_num_reqs):
                    break
1528
        xm.wait_device_ops()
1529
        end = time.perf_counter()
1530
        logger.info("Compilation finished in %.2f [secs].", end - start)
1531
        self._update_num_xla_graphs("select_hidden_states")
1532

1533
1534
    def _precompile_compute_logits(self) -> None:
        logger.info("Compiling compute_logits with different input shapes.")
1535
1536
1537
        start = time.perf_counter()
        hsize = self.model_config.get_hidden_size()
        for num_reqs in self.num_reqs_paddings:
1538
1539
1540
            dummy_hidden = torch.zeros(
                (num_reqs, hsize), device=self.device, dtype=self._hidden_states_dtype
            )
1541
1542
1543
1544
1545
1546
1547
1548
1549
            torch._dynamo.mark_dynamic(dummy_hidden, 0)
            self.compute_logits(dummy_hidden)
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("compute_logits")

    def _precompile_structured_decoding(self) -> None:
1550
        logger.info("Compiling structured_decoding with different input shapes.")
1551
1552
        start = time.perf_counter()
        for num_reqs in self.num_reqs_paddings:
1553
1554
1555
1556
1557
1558
1559
1560
1561
            dummy_logits = torch.zeros(
                (num_reqs, self.vocab_size),
                device=self.device,
                dtype=self._hidden_states_dtype,
            )
            dummy_require_struct_decoding = self.require_structured_out_cpu[
                :num_reqs
            ].to(self.device)
            dummy_grammar_bitmask = self.grammar_bitmask_cpu[:num_reqs].to(self.device)
1562
1563
1564
1565
            # The first dimension of the above 3 dummy tensors cannot be
            # mark_dynamic because some operations in structured_decode require
            # them to be static.
            arange = self.structured_decode_arange.to(self.device)
1566
1567
1568
1569
1570
1571
            self.structured_decode(
                dummy_require_struct_decoding,
                dummy_grammar_bitmask,
                dummy_logits,
                arange,
            )
1572
1573
1574
1575
1576
1577
1578
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("structured_decoding")

    def _precompile_sample_from_logits(self) -> None:
1579
        logger.info("Compiling sample_from_logits with different input shapes.")
1580
1581
        start = time.perf_counter()
        for num_reqs in self.num_reqs_paddings:
1582
1583
1584
1585
1586
            dummy_logits = torch.zeros(
                (num_reqs, self.vocab_size),
                device=self.device,
                dtype=self._hidden_states_dtype,
            )
1587
1588
            # The first dimension of dummy_logits cannot be mark_dynamic
            # because some operations in the sampler require it to be static.
1589
1590
            for all_greedy in [False, True]:
                generate_params_if_all_greedy = not all_greedy
1591
1592
1593
1594
1595
1596
                sampling_metadata = TPUSupportedSamplingMetadata.from_input_batch(
                    self.input_batch,
                    num_reqs,
                    self.device,
                    generate_params_if_all_greedy,
                )
1597
                sampling_metadata.all_greedy = all_greedy
1598
                with self.maybe_select_dummy_loras(
1599
1600
1601
                    self.lora_config, np.array([num_reqs], dtype=np.int32)
                ):
                    self.sample_from_logits_func(dummy_logits, sampling_metadata)
1602
1603
1604
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
1605
1606
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("sample_from_logits")
1607

1608
1609
1610
1611
    def _precompile_gather_logprobs(self) -> None:
        logger.info("Compiling gather_logprobs with different input shapes.")
        start = time.perf_counter()
        for num_reqs in self.num_reqs_paddings:
1612
1613
1614
1615
1616
1617
            dummy_logits = torch.zeros(
                (num_reqs, self.vocab_size),
                device=self.device,
                dtype=self._hidden_states_dtype,
            )
            dummy_tokens = torch.zeros((num_reqs, 1), dtype=torch.int64).to(self.device)
1618
            with self.maybe_select_dummy_loras(
1619
1620
                self.lora_config, np.array([num_reqs], dtype=np.int32)
            ):
1621
                self.gather_logprobs(dummy_logits, dummy_tokens)
1622
1623
1624
1625
1626
1627
            logger.info("  -- num_seqs: %d", num_reqs)
        xm.wait_device_ops()
        end = time.perf_counter()
        logger.info("Compilation finished in %.2f [secs].", end - start)
        self._update_num_xla_graphs("gather_logprobs")

1628
1629
1630
1631
    def capture_model(self) -> None:
        """
        Precompile all the subgraphs with possible input shapes.
        """
1632
1633
1634
1635
1636
1637
1638
1639
        with self.maybe_setup_dummy_loras(self.lora_config):
            self._precompile_mm_encoder()
            self._precompile_backbone()
            self._precompile_select_hidden_states()
            self._precompile_compute_logits()
            self._precompile_structured_decoding()
            self._precompile_sample_from_logits()
            self._precompile_gather_logprobs()
1640

1641
1642
1643
1644
1645
    def profile_run(
        self,
        num_tokens: int,
    ) -> None:
        # Profile with multimodal encoder & encoder cache.
1646
        if self.supports_mm_inputs:
1647
            if self.model_config.multimodal_config.skip_mm_profiling:
1648
                logger.info(
1649
                    "Skipping memory profiling for multimodal encoder and "
1650
1651
                    "encoder cache."
                )
1652
1653
1654
1655
1656
1657
1658
1659
1660
            else:
                mm_budget = self.mm_budget
                assert mm_budget is not None

                # TODO: handle encoder-decoder models once we support them.
                if (encoder_budget := mm_budget.get_encoder_budget()) > 0:
                    # NOTE: Currently model is profiled with a single non-text
                    # modality with the max possible input tokens even when
                    # it supports multiple.
1661
                    dummy_modality = mm_budget.get_modality_with_max_tokens()
1662
1663
1664
                    max_mm_items_per_batch = mm_budget.max_items_per_batch_by_modality[
                        dummy_modality
                    ]
1665
1666
1667
1668
1669
1670
1671
1672
1673

                    logger.info(
                        "Encoder cache will be initialized with a budget of "
                        "%s tokens, and profiled with %s %s items of the "
                        "maximum feature size.",
                        encoder_budget,
                        max_mm_items_per_batch,
                        dummy_modality,
                    )
1674

1675
1676
1677
1678
1679
                    # Create dummy batch of multimodal inputs.
                    batched_dummy_mm_inputs = self._get_mm_dummy_batch(
                        dummy_modality,
                        max_mm_items_per_batch,
                    )
1680

1681
1682
1683
1684
                    # Run multimodal encoder.
                    # Isolate encoder graph from post-processing to minimize
                    # impact of recompilation until it's fixed.
                    start = time.perf_counter()
1685
                    torch_xla.sync(wait=False)
1686
1687
1688
                    dummy_encoder_outputs = self.model.get_multimodal_embeddings(
                        **batched_dummy_mm_inputs
                    )
1689
                    torch_xla.sync(wait=False)
1690
1691
1692
1693
                    xm.wait_device_ops()
                    end = time.perf_counter()
                    logger.info(
                        "Multimodal Encoder profiling finished in %.2f [secs].",
1694
1695
                        end - start,
                    )
1696
1697
1698
1699
1700

                    sanity_check_mm_encoder_outputs(
                        dummy_encoder_outputs,
                        expected_num_items=max_mm_items_per_batch,
                    )
1701

1702
                    # Cache the dummy encoder outputs.
1703
                    self.encoder_cache["tmp"] = dict(enumerate(dummy_encoder_outputs))
1704
1705

        # Trigger compilation for general shape.
1706
1707
1708
        self._dummy_run(
            num_tokens, self.num_reqs_max_model_len, self.max_num_blocks_per_req
        )
1709
        if self.most_model_len is not None:
1710
1711
1712
1713
1714
            self._dummy_run(
                num_tokens,
                self.num_reqs_most_model_len,
                self.num_blocks_per_most_len_req,
            )
1715

1716
        torch_xla.sync(wait=False)
1717
1718
1719
1720
        xm.wait_device_ops()
        self.encoder_cache.clear()
        gc.collect()

1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
    def maybe_setup_cross_layer_kv_sharing(
        self,
        kv_caches: dict[str, torch.Tensor],
        kv_cache_config: KVCacheConfig,
    ) -> None:
        """
        Add layers that re-use KV cache to KV cache group of its target layer.
        Mapping of KV cache tensors happens in `initialize_kv_cache_tensors()`
        """
        if not self.shared_kv_cache_layers:
            # No cross-layer KV sharing, return
            return

        add_kv_sharing_layers_to_kv_cache_groups(
            self.shared_kv_cache_layers,
            kv_cache_config.kv_cache_groups,
        )

1739
1740
        for layer_name, target_layer_name in self.shared_kv_cache_layers.items():
            logger.debug("%s reuses KV cache of %s", layer_name, target_layer_name)
1741
1742
            kv_caches[layer_name] = kv_caches[target_layer_name]

1743
1744
1745
1746
    def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
        """
        Initialize KV cache based on `kv_cache_config`.
        Args:
1747
            kv_cache_config: Configuration for the KV cache, including the KV
1748
1749
            cache size of each layer
        """
1750
        if len(kv_cache_config.kv_cache_groups) > 1:
1751
            raise NotImplementedError(
1752
1753
                "Hybrid models with more than one KV cache type are not supported yet."
            )
1754

1755
1756
1757
1758
        if (
            kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
            != self.block_size
        ):
1759
1760
1761
1762
1763
1764
1765
            self.input_batch = InputBatch(
                max_num_reqs=self.max_num_reqs,
                max_model_len=self.max_model_len,
                max_num_batched_tokens=self.max_num_tokens,
                device=self.device,
                pin_memory=self.pin_memory,
                vocab_size=self.model_config.get_vocab_size(),
1766
1767
1768
                block_sizes=[
                    kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
                ],
1769
1770
            )
        # Verify dtype compatibility between block_table_cpu and input_batch
1771
1772
1773
1774
        assert (
            self.block_table_cpu.dtype
            == self.input_batch.block_table[0].get_cpu_tensor().dtype
        )
1775

1776
1777
1778
        kv_cache_sizes = {}
        for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
            assert len(kv_cache_tensor.shared_by) == 1, (
1779
1780
                "KV cache tensor shared by multiple layers is not supported in TPU."
            )
1781
            kv_cache_sizes[kv_cache_tensor.shared_by[0]] = kv_cache_tensor.size
1782

1783
        kv_caches: dict[str, torch.Tensor] = {}
1784
1785
1786
        for kv_cache_group in kv_cache_config.kv_cache_groups:
            kv_cache_spec = kv_cache_group.kv_cache_spec
            for layer_name in kv_cache_group.layer_names:
1787
1788
1789
                tensor_size = kv_cache_sizes[layer_name]
                assert tensor_size % kv_cache_spec.page_size_bytes == 0
                num_blocks = tensor_size // kv_cache_spec.page_size_bytes  # noqa
1790
                if isinstance(kv_cache_spec, AttentionSpec):
1791
1792
1793
                    if self.use_spmd:
                        num_kv_heads = kv_cache_spec.num_kv_heads
                        assert self.original_parallel_config is not None
1794
                        tp_size = self.original_parallel_config.tensor_parallel_size
1795
1796
1797
                        # TODO: Handle kv cache duplication under SPMD mode.
                        assert num_kv_heads % tp_size == 0, (
                            f"num_kv_heads {num_kv_heads} must be divisible by "
1798
1799
                            f"tp_size {tp_size} under SPMD mode"
                        )
1800
                    kv_cache_shape = PallasAttentionBackend.get_kv_cache_shape(
1801
1802
1803
1804
1805
                        num_blocks,
                        kv_cache_spec.block_size,
                        kv_cache_spec.num_kv_heads,
                        kv_cache_spec.head_size,
                    )
1806
1807
                    dtype = kv_cache_spec.dtype

1808
1809
1810
                    tpu_kv_cache = torch.zeros(kv_cache_shape, dtype=dtype).to(
                        self.device
                    )
1811

1812
                    kv_caches[layer_name] = tpu_kv_cache
1813
1814
                else:
                    raise NotImplementedError
1815

1816
1817
        # Set up cross-layer KV cache sharing if needed
        self.maybe_setup_cross_layer_kv_sharing(kv_caches, kv_cache_config)
1818

1819
1820
1821
        bind_kv_cache(
            kv_caches,
            self.vllm_config.compilation_config.static_forward_context,
1822
1823
            self.kv_caches,
        )
1824

1825
1826
1827
        if self.use_spmd:
            # Shard KV Cache
            for cache in self.kv_caches:
1828
                xs.mark_sharding(cache, self.mesh, (None, "x", None, None))
1829

1830
1831
1832
1833
        if has_kv_transfer_group():
            get_kv_transfer_group().register_kv_caches(kv_caches)
            get_kv_transfer_group().set_host_xfer_buffer_ops(copy_kv_blocks)

1834
    def reset_dynamo_cache(self):
1835
1836
1837
1838
        # NOTE: We check `is_multimodal_model` instead of `supports_mm_inputs`
        # since the compiled model object of the language backbone of a
        # multimodal model needs to be extracted via `get_language_model`.
        if self.model_config.is_multimodal_model:
1839
            compiled_model = self.model.get_language_model().model
1840
1841
1842
1843
1844
        else:
            compiled_model = self.model.model
        if isinstance(compiled_model, TorchCompileWrapperWithCustomDispatcher):
            logger.info("Clear dynamo cache and cached dynamo bytecode.")
            torch._dynamo.eval_frame.remove_from_cache(
1845
1846
                compiled_model.original_code_object
            )
1847
            compiled_model.compiled_codes.clear()
1848

1849
1850
1851
1852
1853
    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
    def select_hidden_states(self, hidden_states, indices_do_sample):
        return hidden_states[indices_do_sample]

    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1854
    def compute_logits(self, sample_hidden_states: torch.Tensor) -> torch.Tensor:
1855
        return self.model.compute_logits(sample_hidden_states)
1856

1857
1858
1859
    # TODO: Under SPMD mode, sample_from_logits has correctness issue.
    #       Re-enable the torch.compile once the issue is fixed in torchxla.
    # @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1860
    def sample_from_logits(
1861
1862
        self, logits: torch.Tensor, sampling_metadata: TPUSupportedSamplingMetadata
    ) -> torch.Tensor:
1863
        """
1864
        Sample with xla-friendly function. This function is to be traced
1865
1866
        separately from `forward` for lighter compilation overhead.
        """
1867
1868
1869
        if sampling_metadata.all_greedy:
            out_tokens = torch.argmax(logits, dim=-1, keepdim=True)
        else:
1870
            out_tokens = self.sampler(logits, sampling_metadata).sampled_token_ids
1871
1872
        return out_tokens

1873
    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1874
1875
1876
    def gather_logprobs(
        self, logits: torch.Tensor, sampled_tokens: torch.Tensor
    ) -> LogprobsTensors:
1877
1878
1879
1880
1881
1882
1883
1884
1885
        """
        Gather the top_logprobs with corresponding tokens. Use a fixed number
        of logprobs as an alternative to having multiple pre-compiled graphs.
        Select the number of logprobs actually demanded by each request on CPU.
        """
        logprobs = self.sampler.compute_logprobs(logits)
        return self.sampler.gather_logprobs(
            logprobs,
            self.model_config.max_logprobs,
1886
1887
            token_ids=sampled_tokens.squeeze(-1),
        )
1888

1889
    @torch.compile(backend="openxla", fullgraph=True, dynamic=False)
1890
1891
1892
1893
1894
1895
1896
    def structured_decode(
        self,
        require_struct_decoding: torch.Tensor,
        grammar_bitmask: torch.Tensor,
        logits: torch.Tensor,
        arange: torch.Tensor,
    ) -> torch.Tensor:
1897
1898
1899
        return torch.where(
            require_struct_decoding,
            self.apply_grammar_bitmask(logits, grammar_bitmask, arange),
1900
1901
            logits,
        )
1902

1903
1904
1905
1906
    def apply_grammar_bitmask(
        self, logits: torch.Tensor, grammar_bitmask: torch.Tensor, arange: torch.Tensor
    ):
        assert logits.shape[0] == grammar_bitmask.shape[0]
1907
1908
        logits_cloned = logits.clone()
        for i in range(logits.shape[0]):
1909
1910
1911
1912
1913
            unpacked_bitmask = (
                torch.bitwise_right_shift(grammar_bitmask[i][:, None], arange[None, :])
                & 1
            ) == 0
            unpacked_bitmask = unpacked_bitmask.reshape(-1)[: self.vocab_size]
1914
            logits_cloned[i] = logits_cloned[i].masked_fill(
1915
1916
                unpacked_bitmask, -float("inf")
            )
1917
1918
        return logits_cloned

1919
1920
    def get_multimodal_embeddings(self, *args, **kwargs):
        return self.model.get_multimodal_embeddings(*args, **kwargs)
1921

1922
1923
1924
    def get_input_embeddings(self, *args, **kwargs):
        return self.model.get_input_embeddings(*args, **kwargs)

1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
    def prepare_structured_decoding_input(
        self, logits: torch.Tensor, scheduler_output: "SchedulerOutput"
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        grammar_bitmask = scheduler_output.grammar_bitmask
        assert grammar_bitmask is not None
        num_reqs, _ = logits.shape

        # Reset pre-allocated tensors
        self.grammar_bitmask_cpu.zero_()
        self.require_structured_out_cpu.zero_()

1936
1937
        sorted_struct_requests = sorted(
            scheduler_output.structured_output_request_ids.items(),
1938
1939
            key=lambda item: item[1],
        )
1940
1941
1942
        cumulative_mask_idx = 0
        for req_id, _ in sorted_struct_requests:
            if req_id not in self.input_batch.req_id_to_index:
1943
1944
                continue
            batch_index = self.input_batch.req_id_to_index[req_id]
1945
            self.grammar_bitmask_cpu[batch_index] = torch.from_numpy(
1946
1947
                grammar_bitmask[cumulative_mask_idx]
            )
1948
1949
1950
1951
1952
1953
            # It's not guaranteed that all requests in this batch require
            # structured output, so create a bool tensor to represent
            # the requests that need structured output.
            self.require_structured_out_cpu[batch_index] = True
            cumulative_mask_idx += 1

1954
1955
1956
1957
1958
        return (
            self.require_structured_out_cpu[:num_reqs].to(logits.device),
            self.grammar_bitmask_cpu[:num_reqs].to(logits.device),
            self.structured_decode_arange.to(logits.device),
        )
1959

1960
1961
1962
1963
1964
1965
    def _get_mm_dummy_batch(
        self,
        modality: str,
        max_items_per_batch: int,
    ) -> BatchedTensorInputs:
        """Dummy data for profiling and precompiling multimodal models."""
1966
1967
        assert self.mm_budget is not None

1968
        dummy_decoder_data = self.mm_registry.get_decoder_dummy_data(
1969
            model_config=self.model_config,
1970
            seq_len=self.max_model_len,
1971
            mm_counts={modality: 1},
1972
            cache=self.mm_budget.cache,
1973
        )
1974
1975
1976
        dummy_mm_data = dummy_decoder_data.multi_modal_data

        # Result in the maximum GPU consumption of the model
1977
1978
        dummy_mm_item = dummy_mm_data[modality][0]
        dummy_mm_items = [dummy_mm_item] * max_items_per_batch
1979

1980
        model = cast(SupportsMultiModal, self.model)
1981
1982
1983
1984
1985
1986
1987
1988
1989
        return next(
            grouped_mm_kwargs
            for _, _, grouped_mm_kwargs in group_mm_kwargs_by_modality(
                dummy_mm_items,
                device=self.device,
                pin_memory=self.pin_memory,
                merge_by_field_config=model.merge_by_field_config,
            )
        )
1990

1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002

def _get_req_paddings(min_req_size: int, max_req_size: int) -> list[int]:
    logger.info("Preparing request paddings:")
    # assert min_req_size is power of 2
    assert (min_req_size & (min_req_size - 1) == 0) and min_req_size > 0
    paddings: list = []
    num = max(MIN_NUM_SEQS, min_req_size)
    while num <= max_req_size and (len(paddings) == 0 or paddings[-1] != num):
        paddings.append(num)
        logger.info("    %d", num)
        num = _get_padded_num_reqs_with_upper_limit(num + 1, max_req_size)
    return paddings
2003
2004


2005
def _get_padded_num_reqs_with_upper_limit(x: int, upper_limit: int) -> int:
2006
    res = MIN_NUM_SEQS if x <= MIN_NUM_SEQS else 1 << (x - 1).bit_length()
2007
    return min(res, upper_limit)
2008
2009


2010
2011
2012
2013
def _get_token_paddings(
    min_token_size: int, max_token_size: int, padding_gap: int
) -> list[int]:
    """Generate a list of padding size, starting from min_token_size,
2014
    ending with a number that can cover max_token_size
2015

2016
2017
2018
    If padding_gap == 0 then:
        increase 2X each time (exponential)
    else:
2019
        first increase the size to twice,
2020
        then increase the padding size by padding_gap.
2021
    """
2022
2023
    # assert min_token_size is power of 2
    assert (min_token_size & (min_token_size - 1) == 0) and min_token_size > 0
2024
2025
    paddings = []
    num = min_token_size
2026
2027

    if padding_gap == 0:
2028
        logger.info("Using exponential token paddings:")
2029
        while True:
2030
2031
            logger.info("    %d", num)
            paddings.append(num)
2032
2033
            if num >= max_token_size:
                break
2034
2035
            num *= 2
    else:
2036
        logger.info("Using incremental token paddings:")
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
        while num <= padding_gap:
            logger.info("    %d", num)
            paddings.append(num)
            num *= 2
        num //= 2
        while num < max_token_size:
            num += padding_gap
            logger.info("    %d", num)
            paddings.append(num)

2047
2048
2049
2050
    return paddings


def _get_padded_token_len(paddings: list[int], x: int) -> int:
2051
    """Return the first element in paddings list greater or equal to x."""
2052
2053
2054
    index = bisect.bisect_left(paddings, x)
    assert index < len(paddings)
    return paddings[index]
2055
2056


2057
2058
2059
def _get_padded_num_kv_cache_update_slices(
    num_tokens: int, max_num_reqs: int, page_size: int
) -> int:
2060
2061
    """Calculates the padded number of KV cache update slices to avoid
    recompilation."""
2062
2063
2064
2065
2066
    # NOTE(chengjiyao): let's say R_i is the token num for i-th request,
    # so it occupies most 2 + R_i // page_size pages. The total maximum
    # possible number of pages needed is sum(2 + R_i // page_size), which
    # is <= 2 * max_num_reqs + sum(R_i) // page_size
    # = 2 * max_num_reqs + num_tokens // page_size
2067
2068
2069
2070
2071
    padded_num_slices = 2 * max_num_reqs + num_tokens // page_size
    padded_num_slices = min(padded_num_slices, num_tokens)
    return padded_num_slices


2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
def _get_num_slices_per_kv_cache_update_block(page_size_bytes: int) -> int:
    """Find the optimum number of slices to copy per Pallas program instance.

    Increasing the number of slices copied in one instance of the kernel program
    will increase HBM bandwidth utilization via more in-flight DMAs.

    However, it will also use more VMEM, and experimentally, we observed
    performance regression at 128 slices on v6e, likely due to running
    out of scalar registers. Thus this function will limit the number of
    slices to 64.
    """
2083
2084
2085
    # The default vmem_limit_bytes of a pallas kernel is 32MB. Here we
    # calculate num_slices_per_block based on 16MB in case any register spills.
    vmem_limit = 16 * 1024 * 1024
2086
2087
2088
2089
2090
2091
2092
2093
    num_slices_per_block = vmem_limit // page_size_bytes
    assert num_slices_per_block > 0, "Number of slices should be positive"
    num_slices_per_block = prev_power_of_2(num_slices_per_block)
    if num_slices_per_block > 64:
        num_slices_per_block = 64
    return num_slices_per_block


2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
def replace_set_lora(model):
    def _tpu_set_lora(
        self,
        index: int,
        lora_a: torch.Tensor,
        lora_b: torch.Tensor,
        embeddings_tensor: Optional[torch.Tensor],
        bias: Optional[torch.Tensor] = None,
    ):
        # TODO: The integer index leads to a recompilation, but converting it
        # to a tensor doesn't seem to work anymore. This might be fixed with a
        # later release of torch_xla.
        self._original_set_lora(index, lora_a, lora_b, embeddings_tensor, bias)
2107
        torch_xla.sync(wait=False)
2108
2109
2110

    def _tpu_reset_lora(self, index: int):
        self._original_reset_lora(index)
2111
        torch_xla.sync(wait=False)
2112
2113
2114
2115
2116
2117

    for _, module in model.named_modules():
        if isinstance(module, BaseLayerWithLoRA):
            module._original_set_lora = module.set_lora
            module._original_reset_lora = module.reset_lora
            module.set_lora = _tpu_set_lora.__get__(module, module.__class__)
2118
            module.reset_lora = _tpu_reset_lora.__get__(module, module.__class__)