cpu_model_runner.py 13.6 KB
Newer Older
1
from typing import List, Optional, Tuple
2
3

import torch
4
from torch import nn
5
6

from vllm.attention import AttentionMetadata, get_attn_backend
7
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
                         ModelConfig, ParallelConfig, SchedulerConfig,
                         VisionLanguageConfig)
10
11
12
13
from vllm.distributed import broadcast_tensor_dict
from vllm.logger import init_logger
from vllm.model_executor import SamplingMetadata
from vllm.model_executor.model_loader import get_model
14
15
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
from vllm.utils import make_tensor_with_pad
16
17
18
19
20
21
22
23
24
25
26
27
28
29

logger = init_logger(__name__)

_PAD_SLOT_ID = -1


class CPUModelRunner:

    def __init__(
        self,
        model_config: ModelConfig,
        parallel_config: ParallelConfig,
        scheduler_config: SchedulerConfig,
        device_config: DeviceConfig,
30
        cache_config: CacheConfig,
31
        load_config: LoadConfig,
32
        lora_config: Optional[LoRAConfig],
33
        vision_language_config: Optional[VisionLanguageConfig],
34
35
36
37
38
39
40
41
        kv_cache_dtype: Optional[str] = "auto",
        is_driver_worker: bool = False,
        *args,
        **kwargs,
    ):
        self.model_config = model_config
        self.parallel_config = parallel_config
        self.scheduler_config = scheduler_config
42
43
        # Currently, CPU worker doesn't support chunked prefill.
        assert self.scheduler_config.chunked_prefill_enabled is False
44
45
        self.device_config = device_config
        self.cache_config = cache_config
46
        self.lora_config = lora_config
47
        self.vision_language_config = vision_language_config
48
        self.load_config = load_config
49
50
51
52
53
        self.is_driver_worker = is_driver_worker

        self.device = self.device_config.device

        self.kv_cache_dtype = kv_cache_dtype
54
55
        self.sliding_window = model_config.get_sliding_window()
        self.block_size = cache_config.block_size
56
57
58
59
60
61
62
63
64
        self.attn_backend = get_attn_backend(
            self.model_config.get_num_attention_heads(self.parallel_config),
            self.model_config.get_head_size(),
            self.model_config.get_num_kv_heads(self.parallel_config),
            self.model_config.get_sliding_window(),
            self.model_config.dtype,
            self.kv_cache_dtype,
            self.block_size,
        )
65

66
67
68
        # Lazy initialization.
        self.model: nn.Module  # Set after init_Model

69
    def load_model(self) -> None:
70
71
72
73
74
75
76
        self.model = get_model(
            model_config=self.model_config,
            load_config=self.load_config,
            device_config=self.device_config,
            vision_language_config=self.vision_language_config,
            lora_config=self.lora_config,
            parallel_config=self.parallel_config,
77
78
            scheduler_config=self.scheduler_config,
            cache_config=self.cache_config)
79
80
81
82

    def _prepare_prompt(
        self,
        seq_group_metadata_list: List[SequenceGroupMetadata],
83
84
    ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
               Optional[torch.Tensor]]:
85
86
87
88
        assert len(seq_group_metadata_list) > 0
        input_tokens: List[int] = []
        input_positions: List[int] = []
        slot_mapping: List[int] = []
89
        seq_lens: List[int] = []
90
        multi_modal_input_list: List[torch.Tensor] = []
91
92
93
94
95
96
97
98
99
100

        for seq_group_metadata in seq_group_metadata_list:
            assert seq_group_metadata.is_prompt
            seq_ids = list(seq_group_metadata.seq_data.keys())
            assert len(seq_ids) == 1
            seq_id = seq_ids[0]

            seq_data = seq_group_metadata.seq_data[seq_id]
            prompt_tokens = seq_data.get_token_ids()
            computed_len = seq_data.get_num_computed_tokens()
101
            seq_len = len(prompt_tokens)
102

103
            seq_lens.append(seq_len)  # Prompt token num
104
105
106
107
108
            input_tokens.extend(prompt_tokens)  # Token ids

            # Token position ids
            # NOTE(woosuk): Here we assume that the first token in the prompt
            # is always the first token in the sequence.
109
            input_positions.extend(list(range(computed_len, seq_len)))
110

111
112
113
114
            if seq_group_metadata.multi_modal_data:
                multi_modal_input_list.append(
                    seq_group_metadata.multi_modal_data.data)

115
116
117
            # Compute the slot mapping.
            block_table = seq_group_metadata.block_tables[seq_id]
            # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
118
            # where start_idx is max(0, seq_len - sliding_window).
119
120
121
122
123
            # For example, if the prompt len is 10, sliding window is 8, and
            # block size is 4, the first two tokens are masked and the slot
            # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
            start_idx = 0
            if self.sliding_window is not None:
124
                start_idx = max(0, seq_len - self.sliding_window)
125

126
            for i in range(computed_len, seq_len):
127
128
129
130
131
132
133
134
135
136
                if i < start_idx:
                    slot_mapping.append(_PAD_SLOT_ID)
                    continue

                block_number = block_table[i //
                                           self.block_size]  # type: ignore
                block_offset = i % self.block_size  # type: ignore
                slot = block_number * self.block_size + block_offset
                slot_mapping.append(slot)

137
138
139
140
141
142
143
144
145
        if multi_modal_input_list:
            assert self.vision_language_config, (
                "Multi-modal inputs are only supported by "
                "vision language models.")
            multi_modal_input = torch.cat(multi_modal_input_list,
                                          dim=0).to(self.device)
        else:
            multi_modal_input = None

146
147
148
149
150
151
152
153
154
155
156
157
158
159
        num_prompt_tokens = len(input_tokens)

        input_tokens = torch.tensor(input_tokens,
                                    dtype=torch.long,
                                    device=self.device)  # type: ignore
        input_positions = torch.tensor(input_positions,
                                       dtype=torch.long,
                                       device=self.device)  # type: ignore
        slot_mapping = torch.tensor(slot_mapping,
                                    dtype=torch.long,
                                    device=self.device)  # type: ignore

        attn_metadata = self.attn_backend.make_metadata(
            is_prompt=True,
160
161
162
163
            seq_lens=seq_lens,
            seq_lens_tensor=None,
            max_seq_len=None,
            num_prefills=len(seq_lens),
164
165
166
167
168
169
170
            num_prefill_tokens=num_prompt_tokens,
            num_decode_tokens=0,
            prefill_metadata=None,
            decode_metadata=None,
            block_tables=torch.tensor([]),
            slot_mapping=slot_mapping,
        )
171
        return (input_tokens, input_positions, attn_metadata, seq_lens,
172
                multi_modal_input)
173
174
175
176
177
178
179
180
181

    def _prepare_decode(
        self,
        seq_group_metadata_list: List[SequenceGroupMetadata],
    ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
        assert len(seq_group_metadata_list) > 0
        input_tokens: List[int] = []
        input_positions: List[int] = []
        slot_mapping: List[int] = []
182
        seq_lens: List[int] = []
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
        block_tables: List[List[int]] = []

        for seq_group_metadata in seq_group_metadata_list:
            assert not seq_group_metadata.is_prompt
            assert seq_group_metadata.token_chunk_size == 1

            seq_ids = list(seq_group_metadata.seq_data.keys())

            for seq_id in seq_ids:
                seq_data = seq_group_metadata.seq_data[seq_id]
                generation_token = seq_data.get_last_token_id()
                input_tokens.append(generation_token)

                seq_len = seq_data.get_len()
                position = seq_len - 1
                input_positions.append(position)

200
                seq_len = seq_len if self.sliding_window is None else min(
201
                    seq_len, self.sliding_window)
202
                seq_lens.append(seq_len)
203
204
205
206
207
208
209
210
211
212
213
214
215

                block_table = seq_group_metadata.block_tables[seq_id]
                block_number = block_table[position // self.block_size]
                block_offset = position % self.block_size
                slot = block_number * self.block_size + block_offset
                slot_mapping.append(slot)

                if self.sliding_window is not None:
                    sliding_window_blocks = (self.sliding_window //
                                             self.block_size)
                    block_table = block_table[-sliding_window_blocks:]
                block_tables.append(block_table)

216
        max_seq_len = max(seq_lens)
217
218
219
220
221
222
223
224
225
226

        input_tokens = torch.tensor(input_tokens,
                                    dtype=torch.long,
                                    device=self.device)
        input_positions = torch.tensor(input_positions,
                                       dtype=torch.long,
                                       device=self.device)
        slot_mapping = torch.tensor(slot_mapping,
                                    dtype=torch.long,
                                    device=self.device)
227
228
229
        seq_lens_tensor = torch.tensor(seq_lens,
                                       dtype=torch.int,
                                       device=self.device)
230
231
232
233
234
235
236
237
238
239
240
241
242
243

        max_block_table_len = max(
            len(block_table) for block_table in block_tables)
        block_tables = make_tensor_with_pad(
            block_tables,
            max_len=max_block_table_len,
            pad=0,
            dtype=torch.int,
            device=self.device,
        )

        attn_metadata = self.attn_backend.make_metadata(
            is_prompt=False,
            slot_mapping=slot_mapping,
244
245
246
            seq_lens=seq_lens,
            seq_lens_tensor=seq_lens_tensor,
            max_seq_len=max_seq_len,
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
            num_prefill_tokens=0,
            num_decode_tokens=len(input_tokens),
            num_prefills=0,
            prefill_metadata=None,
            decode_metadata=None,
            block_tables=block_tables,
        )
        return (
            input_tokens,
            input_positions,
            attn_metadata,
        )

    def prepare_input_tensors(
        self,
262
        seq_group_metadata_list: List[SequenceGroupMetadata],
263
264
    ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
               Optional[torch.Tensor]]:
265
        multi_modal_input = None
266
267
268
269
270
271
        if self.is_driver_worker:
            # NOTE: We assume that all sequences in the group are all prompts or
            # all decodes.
            is_prompt = seq_group_metadata_list[0].is_prompt
            # Prepare input tensors.
            if is_prompt:
272
                (input_tokens, input_positions, attn_metadata, seq_lens,
273
274
                 multi_modal_input
                 ) = self._prepare_prompt(seq_group_metadata_list)
275
276
277
            else:
                (input_tokens, input_positions,
                 attn_metadata) = self._prepare_decode(seq_group_metadata_list)
278
                seq_lens = []
279
280
            sampling_metadata = SamplingMetadata.prepare(
                seq_group_metadata_list,
281
282
                seq_lens,
                # query_lens is not needed if chunked prefill is not
283
                # supported. Since CPU worker doesn't support chunked prefill
284
285
                # just use seq_lens instead.
                seq_lens,
286
287
                self.device,
                pin_memory=False)
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
            # Broadcast the metadata.
            metadata_dict = {
                "input_tokens": input_tokens,
                "input_positions": input_positions,
                "selected_token_indices":
                sampling_metadata.selected_token_indices,
            }
            metadata_dict.update(attn_metadata.asdict_zerocopy())
            broadcast_tensor_dict(metadata_dict, src=0)
        else:
            metadata_dict = broadcast_tensor_dict(src=0)
            input_tokens = metadata_dict.pop("input_tokens")
            input_positions = metadata_dict.pop("input_positions")
            selected_token_indices = metadata_dict.pop(
                "selected_token_indices")
            attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
            sampling_metadata = SamplingMetadata(
                seq_groups=None,
                seq_data=None,
307
                seq_lens=None,
308
309
310
311
312
                selected_token_indices=selected_token_indices,
                categorized_sample_indices=None,
                generators=None,
            )

313
314
        return (input_tokens, input_positions, attn_metadata,
                sampling_metadata, multi_modal_input)
315
316
317
318

    @torch.inference_mode()
    def execute_model(
        self,
319
        seq_group_metadata_list: List[SequenceGroupMetadata],
320
321
        kv_caches: List[torch.Tensor],
    ) -> Optional[SamplerOutput]:
322
323
        (input_tokens, input_positions, attn_metadata, sampling_metadata,
         multi_modal_input
324
325
326
327
328
329
330
331
332
         ) = self.prepare_input_tensors(seq_group_metadata_list)

        model_executable = self.model
        execute_model_kwargs = {
            "input_ids": input_tokens,
            "positions": input_positions,
            "kv_caches": kv_caches,
            "attn_metadata": attn_metadata,
        }
333
334
        if self.vision_language_config:
            execute_model_kwargs.update({"image_input": multi_modal_input})
335
336
337
338
339
340
341

        hidden_states = model_executable(**execute_model_kwargs)

        # Compute the logits.
        logits = self.model.compute_logits(hidden_states, sampling_metadata)

        # Only perform sampling in the driver worker.
342
        if not self.is_driver_worker:
343
344
345
346
347
348
349
350
            return None

        # Sample the next token.
        output = self.model.sample(
            logits=logits,
            sampling_metadata=sampling_metadata,
        )
        return output