processor.py 23.5 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import time
5
from collections.abc import Mapping
6
from typing import Any, Literal, Optional, Union
7

8
from vllm.config import VllmConfig
9
from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs
10
from vllm.inputs.parse import split_enc_dec_inputs
11
from vllm.inputs.preprocess import InputPreprocessor
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
15
from vllm.multimodal.cache import processor_cache_from_config
16
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict
17
from vllm.multimodal.processing import EncDecMultiModalProcessor
18
from vllm.multimodal.utils import argsort_mm_positions
19
20
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
21
from vllm.transformers_utils.tokenizer import AnyTokenizer
22
from vllm.utils import length_from_prompt_token_ids_or_embeds
23
from vllm.v1.engine import EngineCoreRequest
24
from vllm.v1.structured_output.backend_guidance import validate_guidance_grammar
25
from vllm.v1.structured_output.backend_lm_format_enforcer import (
26
27
    validate_structured_output_request_lm_format_enforcer,
)
28
from vllm.v1.structured_output.backend_outlines import (
29
30
31
    validate_structured_output_request_outlines,
)
from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
32

33
34
logger = init_logger(__name__)

35
36
37
38

class Processor:
    def __init__(
        self,
39
        vllm_config: VllmConfig,
40
        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
41
    ) -> None:
42
43
44
45
        self.vllm_config = vllm_config
        self.model_config = vllm_config.model_config
        self.cache_config = vllm_config.cache_config
        self.lora_config = vllm_config.lora_config
46
        self.structured_outputs_config = vllm_config.structured_outputs_config
47

48
        self.generation_config_fields = self.model_config.try_get_generation_config()
49

50
        self.mm_registry = mm_registry
51
        self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry)
52

53
54
55
56
57
        self.input_preprocessor = InputPreprocessor(
            self.model_config,
            mm_registry,
            mm_processor_cache=self.mm_processor_cache,
        )
58

59
60
61
62
63
64
65
66
    @property
    def tokenizer(self) -> Optional[AnyTokenizer]:
        return self.input_preprocessor.tokenizer

    @tokenizer.setter
    def tokenizer(self, tokenizer: Optional[AnyTokenizer]) -> None:
        self.input_preprocessor.tokenizer = tokenizer

67
68
    def _validate_logprobs(
        self,
69
        params: SamplingParams,
70
71
    ) -> None:
        max_logprobs = self.model_config.max_logprobs
72
        if max_logprobs == -1:
73
74
            max_logprobs = self.model_config.get_vocab_size()

75
        # Validate sample logprobs.
76
77
78
79
80
81
82
        if params.logprobs:
            num_logprobs = params.logprobs
            if num_logprobs == -1:
                num_logprobs = self.model_config.get_vocab_size()
            if num_logprobs > max_logprobs:
                raise ValueError(
                    f"Requested sample logprobs of {num_logprobs}, "
83
84
                    f"which is greater than max allowed: {max_logprobs}"
                )
85
86

        # Validate prompt logprobs.
87
88
89
90
91
92
93
        if params.prompt_logprobs:
            num_prompt_logprobs = params.prompt_logprobs
            if num_prompt_logprobs == -1:
                num_prompt_logprobs = self.model_config.get_vocab_size()
            if num_prompt_logprobs > max_logprobs:
                raise ValueError(
                    f"Requested prompt logprobs of {num_prompt_logprobs}, "
94
95
                    f"which is greater than max allowed: {max_logprobs}"
                )
96

97
    def _validate_sampling_params(
98
        self,
99
        params: SamplingParams,
100
    ) -> None:
101
        self._validate_structured_output(params)
102
        self._validate_logit_bias(params)
103

104
105
        if params.allowed_token_ids is None:
            return
106
107
        if not params.allowed_token_ids:
            raise ValueError("allowed_token_ids is not None and empty!")
108
109
110
111
        if self.tokenizer is None:
            # When skip_tokenizer_init=True, we can't validate token IDs
            # Skip validation and let the model handle invalid tokens
            return
112
        vocab_size = len(self.tokenizer)
113
        if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids):
114
            raise ValueError("allowed_token_ids contains out-of-vocab token id!")
115

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    def _validate_logit_bias(
        self,
        params: SamplingParams,
    ) -> None:
        """Validate logit_bias token IDs are within vocabulary range."""
        if not params.logit_bias:
            return

        vocab_size = self.model_config.get_vocab_size()
        invalid_token_ids = []

        for token_id in params.logit_bias:
            if token_id < 0 or token_id >= vocab_size:
                invalid_token_ids.append(token_id)

        if invalid_token_ids:
            raise ValueError(
                f"token_id(s) {invalid_token_ids} in logit_bias contain "
134
135
                f"out-of-vocab token ids. Vocabulary size: {vocab_size}"
            )
136

137
138
139
140
    def _validate_supported_sampling_params(
        self,
        params: SamplingParams,
    ) -> None:
141
142
        # Best of not yet supported.
        if params.best_of is not None and params.best_of > 1:
143
            raise ValueError("vLLM V1 does not yet support best_of.")
144
145
        # Logits processors not supported.
        if params.logits_processors:
146
147
148
            raise ValueError(
                "vLLM V1 does not support per request user provided logits processors."
            )
149
150
151
152
153
154
155
156
157
158

    def _validate_params(
        self,
        params: Union[SamplingParams, PoolingParams],
    ):
        """
        Validate supported SamplingParam.
        Should raise ValueError if unsupported for API Server.
        """

159
160
        if isinstance(params, PoolingParams):
            return
161
162

        self._validate_logprobs(params)
163
        self._validate_sampling_params(params)
164
165
        self._validate_supported_sampling_params(params)

166
167
168
169
    def _validate_multi_modal_uuids(self, prompt: PromptType) -> None:
        """
        Validate that user-provided multi_modal_uuids align with
        multi_modal_data in the incoming request prompt(s).
170
        Only checks lengths; `None` entries are allowed and will be
171
172
173
174
175
176
177
178
179
180
181
182
183
184
        auto-hashed downstream.
        """

        def _validate_single_prompt(single_prompt: Union[dict, str]) -> None:
            if not isinstance(single_prompt, dict):
                return
            mm_data = single_prompt.get("multi_modal_data")
            mm_uuids = single_prompt.get("multi_modal_uuids")
            if not mm_data or not mm_uuids:
                return

            for modality, items in mm_data.items():
                if modality in mm_uuids:
                    data_len = len(items) if isinstance(items, list) else 1
185
186
187
188
189
                    uuid_len = (
                        len(mm_uuids[modality])
                        if isinstance(mm_uuids[modality], list)
                        else 1
                    )
190
191
192
193
194
                    if uuid_len != data_len:
                        raise ValueError(
                            f"multi_modal_uuids for modality '{modality}' "
                            "must have same length as data: got "
                            f"{uuid_len} uuids vs "
195
196
                            f"{data_len} items."
                        )
197
198
199
                else:
                    raise ValueError(
                        f"multi_modal_uuids for modality '{modality}' must "
200
201
                        "be provided if multi_modal_data is provided."
                    )
202
203
204
205
206
207
208
209
210
211
212
213

        # Handle explicit encoder/decoder prompts or singleton prompt
        if isinstance(prompt, dict) and "encoder_prompt" in prompt:
            enc = prompt.get("encoder_prompt")
            dec = prompt.get("decoder_prompt")
            if enc is not None:
                _validate_single_prompt(enc)
            if dec is not None:
                _validate_single_prompt(dec)
        else:
            _validate_single_prompt(prompt)  # type: ignore[arg-type]

214
    def _validate_lora(self, lora_request: Optional[LoRARequest]) -> None:
215
216
217
218
219
        if lora_request is None:
            return

        # LoRA request passed in while LoRA is not enabled
        if not self.lora_config:
220
221
222
            raise ValueError(
                f"Got lora_request {lora_request} but LoRA is not enabled!"
            )
223

224
225
226
227
228
229
        if self.tokenizer is not None:
            logger.warning_once(
                "vLLM has deprecated support for supporting different "
                "tokenizers for different LoRAs. By default, vLLM uses base "
                "model's tokenizer. If you are using a LoRA "
                "with its own tokenizer, consider specifying `--tokenizer "
230
231
                "[lora_path]` to use the LoRA tokenizer."
            )
232

233
    def _validate_structured_output(self, params: SamplingParams) -> None:
234
        if not params.structured_outputs or not self.structured_outputs_config:
235
            return
236

237
        if self.model_config.skip_tokenizer_init and params.structured_outputs:
238
239
240
241
            raise ValueError(
                "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'"  # noqa: E501
            )

242
243
244
        backend = self.structured_outputs_config.backend
        if _backend := params.structured_outputs._backend:
            # Request-level backend selection is not supported.
245
246
247
            # The values may differ if `params` is reused and was set
            # to a specific backend based on `auto` behavior in a previous
            # request. We remember that it was set as a result of `auto`
248
            # using the `_backend_was_auto` field set in the params.
249
250
251
            if backend != _backend and not (
                backend == "auto" and params.structured_outputs._backend_was_auto
            ):
252
                raise ValueError(
253
254
255
                    "Request-level structured output backend selection is not "
                    f"supported. The request specified '{_backend}', but vLLM "
                    f"was initialised with '{backend}'. This error can be "
256
257
                    "resolved by removing '_backend' from the request."
                )
258
        else:
259
            params.structured_outputs._backend = backend
260

261
        # Request content validation
262
263
264
265
        if (
            isinstance(params.structured_outputs.choice, list)
            and not params.structured_outputs.choice
        ):
266
            # It is invalid for choice to be an empty list
267
268
269
            raise ValueError(
                f"Choice '{params.structured_outputs.choice}' cannot be an empty list"  # noqa: E501
            )
270

271
        if backend.startswith("xgrammar"):
272
            # xgrammar with no fallback
273
            validate_xgrammar_grammar(params)
274
        elif backend.startswith("guidance"):
275
276
277
278
279
            # TODO: ideally we would have the LLTokenizer here as Lark syntax
            # allows <|special_token|> and similar, see
            # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
            # Without tokenizer these are disallowed in grammars.
            validate_guidance_grammar(params, tokenizer=None)
280
        elif backend == "outlines":
281
282
            # outlines backend
            validate_structured_output_request_outlines(params)
283
        elif backend == "lm-format-enforcer":
284
285
            # lm format enforcer backend
            validate_structured_output_request_lm_format_enforcer(params)
286
        else:
287
            # NOTE: backend must be "auto" here, because we have
288
            # checked supported_backends above.
289
290
291
292
            # In this mode, we set opinionated defaults based on what we think
            # will satisfy the most use cases without having to worry about
            # this setting. We include fallback behavior here, but not with any
            # other setting where a specific backend was specified.
293
            try:
294
                validate_xgrammar_grammar(params)
295
                params.structured_outputs._backend = "xgrammar"
296
            except ValueError:
297
298
                # The request either failed validation
                # or includes some jsonschema feature(s) that
299
                # are not supported in xgrammar. Fall back to guidance.
300
                validate_guidance_grammar(params, tokenizer=None)
301
                params.structured_outputs._backend = "guidance"
302
            # Remember that this backend was set automatically
303
            params.structured_outputs._backend_was_auto = True
304

305
    def _maybe_build_mm_uuids(
306
307
308
        self,
        request_id: str,
        prompt: PromptType,
309
    ) -> Optional[MultiModalUUIDDict]:
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
        """Build per-item multimodal hash overrides when enabled. In this case,
        multimodal data items are identified by their request id, modality and
        index rather than their content.

        Returns a dictionary of modality -> list[str] of overrides, or None if
        disabled or no multimodal data is present.
        """

        def _extract_mm_data(p: PromptType):
            if isinstance(p, dict) and "encoder_prompt" in p:
                enc = p.get("encoder_prompt")
                if isinstance(enc, dict):
                    return enc.get("multi_modal_data")
                return None
            if isinstance(p, dict):
                return p.get("multi_modal_data")
            return None

        mm_data = _extract_mm_data(prompt)
        if not mm_data:
            return None

332
        mm_uuids: MultiModalUUIDDict = {}
333
334
        for modality, data in mm_data.items():
            n = len(data) if isinstance(data, list) else 1
335
            mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)]
336
        return mm_uuids
337

338
339
340
341
342
    def process_inputs(
        self,
        request_id: str,
        prompt: PromptType,
        params: Union[SamplingParams, PoolingParams],
343
        arrival_time: Optional[float] = None,
344
        lora_request: Optional[LoRARequest] = None,
345
        tokenization_kwargs: Optional[dict[str, Any]] = None,
346
347
        trace_headers: Optional[Mapping[str, str]] = None,
        priority: int = 0,
348
        data_parallel_rank: Optional[int] = None,
349
    ) -> EngineCoreRequest:
350
        self._validate_lora(lora_request)
351
        self._validate_params(params)
352

353
        data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
354
355
356
357
358
359
360
        if data_parallel_rank is not None and not (
            0 <= data_parallel_rank < data_parallel_size
        ):
            raise ValueError(
                f"data_parallel_rank {data_parallel_rank} "
                f"is out of range [0, {data_parallel_size})."
            )
361

362
363
364
        if arrival_time is None:
            arrival_time = time.time()

365
366
367
        # Optionally generate multimodal hash overrides to avoid hashing
        # multimodal data items by their content as their identifiers.

368
369
        # NOTE: when users explicitly turn off BOTH prefix caching and input
        # processing caching, no multimodal features or embeddings will be
370
371
372
        # reused across requests, therefore identifying multimodal data items
        # by their content is no longer necessary, and we create uuids with
        # request id-modality-index as multimodal hash overrides.
373
374
375
376
377
        if (
            self.model_config.multimodal_config
            and self.model_config.multimodal_config.mm_processor_cache_gb == 0
            and not self.cache_config.enable_prefix_caching
        ):
378
            mm_uuids = self._maybe_build_mm_uuids(request_id, prompt)
379
        else:
380
381
382
383
            # Otherwise, use user-provided uuids as multimodal hash overrides
            # if provided.
            self._validate_multi_modal_uuids(prompt)
            if isinstance(prompt, dict):
384
                mm_uuids = prompt.get("multi_modal_uuids")
385
            else:
386
                mm_uuids = None
387

388
389
390
391
        # Process inputs, which includes:
        # 1. Tokenize text prompt, with LoRA request if one exists.
        # 2. For multimodal models with a merged preprocessor, preprocess
        #   multimodal data and expand prompt token ids accordingly.
392
        processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(
393
            prompt,
394
            tokenization_kwargs=tokenization_kwargs,
395
            mm_uuids=mm_uuids,
396
        )
397
        from vllm.platforms import current_platform
398

399
400
401
402
403
        current_platform.validate_request(
            prompt=prompt,
            params=params,
            processed_inputs=processed_inputs,
        )
404

405
        eos_token_id = self.input_preprocessor.get_eos_token_id()
406

407
        encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
408
409
        self._validate_model_inputs(encoder_inputs, decoder_inputs)

410
411
412
413
        # Mypy does not always properly infer the types of some elements of
        # discriminated unions of TypedDicts, because of how it handles
        # inheritance of TypedDict. If we explicitly extract the items we want
        # we can avoid type errors from using `dict.get` later in the method.
414
415
416
417
418
419
420
421
422
423
        prompt_token_ids = (
            decoder_inputs["prompt_token_ids"]
            if decoder_inputs["type"] != "embeds"
            else None
        )
        prompt_embeds = (
            decoder_inputs["prompt_embeds"]
            if decoder_inputs["type"] == "embeds"
            else None
        )
424

425
426
427
428
429
430
431
        sampling_params = None
        pooling_params = None
        if isinstance(params, SamplingParams):
            # TODO: can we avoid cloning here in multiproc case?
            sampling_params = params.clone()
            # If unset max tokens, then generate up to the max_model_len.
            if sampling_params.max_tokens is None:
432
                seq_len = length_from_prompt_token_ids_or_embeds(
433
434
435
                    prompt_token_ids, prompt_embeds
                )
                sampling_params.max_tokens = self.model_config.max_model_len - seq_len
436
            sampling_params.update_from_generation_config(
437
438
                self.generation_config_fields, eos_token_id
            )
439
            if self.tokenizer is not None:
440
                sampling_params.update_from_tokenizer(self.tokenizer)
441
442
        else:
            pooling_params = params.clone()
443

444
        # Multimodal related.
445
446
        mm_features: Optional[list[MultiModalFeatureSpec]] = None

447
448
        if decoder_inputs["type"] == "multimodal":
            decoder_mm_inputs = decoder_inputs["mm_kwargs"]
449
            decoder_mm_positions = decoder_inputs["mm_placeholders"]
450
            decoder_mm_hashes = decoder_inputs["mm_hashes"]
451
452
453
454

            # Merge and flatten multimodal placeholders, hashes and inputs
            # from dictionaries to lists, and sort them by each item's position
            # in the input sequence.
455
456
            sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions)

457
458
459
460
461
462
463
            mm_features = []
            for modality, idx in sorted_mm_idxs:
                mm_features.append(
                    MultiModalFeatureSpec(
                        data=decoder_mm_inputs[modality][idx],
                        modality=modality,
                        identifier=decoder_mm_hashes[modality][idx],
464
465
466
                        mm_position=decoder_mm_positions[modality][idx],
                    )
                )
467

468
        return EngineCoreRequest(
469
            request_id=request_id,
470
471
            prompt_token_ids=prompt_token_ids,
            prompt_embeds=prompt_embeds,
472
            mm_features=mm_features,
473
            sampling_params=sampling_params,
474
            pooling_params=pooling_params,
475
476
477
            eos_token_id=eos_token_id,
            arrival_time=arrival_time,
            lora_request=lora_request,
478
            cache_salt=decoder_inputs.get("cache_salt"),
479
            priority=priority,
480
            data_parallel_rank=data_parallel_rank,
481
            trace_headers=trace_headers,
482
        )
483

484
485
486
    def _validate_model_inputs(
        self, encoder_inputs: Optional[SingletonInputs], decoder_inputs: SingletonInputs
    ):
487
        if encoder_inputs is not None:
488
            self._validate_model_input(encoder_inputs, prompt_type="encoder")
489

490
        self._validate_model_input(decoder_inputs, prompt_type="decoder")
491

492
493
494
495
496
497
    def _validate_model_input(
        self,
        prompt_inputs: SingletonInputs,
        *,
        prompt_type: Literal["encoder", "decoder"],
    ):
498
        model_config = self.model_config
499

500
501
502
503
504
505
506
507
508
509
510
        prompt_ids = (
            None
            if prompt_inputs["type"] == "embeds"
            else prompt_inputs["prompt_token_ids"]
        )
        prompt_embeds = (
            prompt_inputs["prompt_embeds"]
            if prompt_inputs["type"] == "embeds"
            else None
        )
        prompt_len = length_from_prompt_token_ids_or_embeds(prompt_ids, prompt_embeds)
511
        if not prompt_ids:
512
513
            if prompt_type == "encoder" and model_config.is_multimodal_model:
                pass  # Mllama may have empty encoder inputs for text-only data
514
515
            elif prompt_inputs["type"] == "embeds":
                pass  # Prompt embeds should not have prompt_ids.
516
517
            else:
                raise ValueError(f"The {prompt_type} prompt cannot be empty")
518

519
520
        tokenizer = self.tokenizer
        if tokenizer is not None:
521
            max_input_id = max(prompt_ids or [], default=0)
522
523
524
525
526
527
528
529
530
531
532

            # NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while
            # self.model_config.get_vocab_size() is the model’s vocab size.
            # For Qwen3 models, the language model has extra tokens that do
            # not exist in the tokenizer, and vice versa for multimodal
            # placeholder tokens in some multimodal models.
            # See https://github.com/QwenLM/Qwen3/issues/29#issuecomment-1933720399 # noqa: E501
            # and https://github.com/vllm-project/vllm/pull/22471#discussion_r2312251421 # noqa: E501

            # Here we take the max of the two to determine if a token id is
            # truly out-of-vocabulary.
533
534
535
536
            if max_input_id > max(
                tokenizer.max_token_id, self.model_config.get_vocab_size() - 1
            ):
                raise ValueError(f"Token id {max_input_id} is out of vocabulary")
537
538

        max_prompt_len = self.model_config.max_model_len
539
        if prompt_len > max_prompt_len:
540
541
542
543
544
545
546
547
548
            if prompt_type == "encoder" and model_config.is_multimodal_model:
                mm_registry = self.input_preprocessor.mm_registry
                mm_processor = mm_registry.create_processor(
                    model_config,
                    tokenizer=tokenizer,
                )
                assert isinstance(mm_processor, EncDecMultiModalProcessor)

                if mm_processor.pad_dummy_encoder_prompt:
549
                    return  # Skip encoder length check for Whisper
550
551

            if model_config.is_multimodal_model:
552
                suggestion = (
553
554
555
                    "Make sure that `max_model_len` is no smaller than the "
                    "number of text tokens plus multimodal tokens. For image "
                    "inputs, the number of image tokens depends on the number "
556
557
                    "of images, and possibly their aspect ratios as well."
                )
558
559
560
            else:
                suggestion = (
                    "Make sure that `max_model_len` is no smaller than the "
561
562
                    "number of text tokens."
                )
563
564

            raise ValueError(
565
                f"The {prompt_type} prompt (length {prompt_len}) is "
566
                f"longer than the maximum model length of {max_prompt_len}. "
567
568
                f"{suggestion}"
            )
569

570
571
572
            # TODO: Find out how many placeholder tokens are there so we can
            # check that chunked prefill does not truncate them
            # max_batch_len = self.scheduler_config.max_num_batched_tokens
573
574
575

    def clear_cache(self) -> None:
        self.input_preprocessor.clear_cache()