processor.py 18.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
import importlib
import inspect
6
from functools import lru_cache
7
from typing import TYPE_CHECKING, Any, cast, get_args, get_type_hints
8

9
10
11
12
13
from transformers import (
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoVideoProcessor,
14
    processing_utils,
15
)
16
17
from transformers.feature_extraction_utils import FeatureExtractionMixin
from transformers.image_processing_utils import BaseImageProcessor
18
from transformers.processing_utils import ProcessorMixin
19
from transformers.video_processing_utils import BaseVideoProcessor
20
21
from typing_extensions import TypeVar

22
from vllm.logger import init_logger
23
from vllm.transformers_utils import processors
24
from vllm.transformers_utils.gguf_utils import is_gguf
25
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
26
from vllm.transformers_utils.utils import convert_model_repo_to_path
27
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
28

29
30
logger = init_logger(__name__)

31
if TYPE_CHECKING:
32
    from vllm.config import ModelConfig
33

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

def _transformers_v4_compatibility_import():
    """Some remote code processors still import `ChatTemplateLoadKwargs` which was a
    subset of `ProcessorChatTemplateKwargs` as defined in Transformers v4.
    In Transformers v5 these were merged into `ProcessorChatTemplateKwargs` and
    `ChatTemplateLoadKwargs` was removed. For backward compatibility, we add an alias
    for `ChatTemplateLoadKwargs` if it doesn't exist.

    This can be removed if `HCXVisionForCausalLM` is upstreamed to Transformers."""
    old_import = getattr(processing_utils, "ChatTemplateLoadKwargs", None)
    new_import = getattr(processing_utils, "ProcessorChatTemplateKwargs", None)
    if old_import is None and new_import is not None:
        processing_utils.ChatTemplateLoadKwargs = new_import


49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def _transformers_v4_compatibility_init() -> Any:
    """Some remote code processors may define `optional_attributes` in their
    `ProcessorMixin` subclass, and then pass these arbitrary attributes directly to
    `ProcessorMixin.__init__`, which is no longer allowed in Transformers v5. For
    backward compatibility, we intercept these optional attributes and set them on the
    processor instance before calling the original `ProcessorMixin.__init__`.

    This can be removed if `Molmo2ForConditionalGeneration` is upstreamed to
    Transformers."""
    # Transformers v4
    if hasattr(ProcessorMixin, "optional_attributes"):
        return
    # Transformers v5
    if hasattr(ProcessorMixin.__init__, "_vllm_patched"):
        return

    original_init = ProcessorMixin.__init__

    def __init__(self, *args, **kwargs):
        for optional_attribute in getattr(self, "optional_attributes", []):
            if optional_attribute in kwargs:
                setattr(self, optional_attribute, kwargs.pop(optional_attribute))

        original_init(self, *args, **kwargs)

    # Only patch if ProcessorMixin is not mocked (for docs builds)
    if not hasattr(ProcessorMixin, "_mock_name"):
        __init__._vllm_patched = True  # type: ignore[attr-defined]
        ProcessorMixin.__init__ = __init__


80
_transformers_v4_compatibility_import()
81
_transformers_v4_compatibility_init()
82

83
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
84
_V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor)
85
86
87
88
89
90
91
92
93
94
95
96
97


class HashableDict(dict):
    """
    A dictionary that can be hashed by lru_cache.
    """

    # NOTE: pythonic dict is not hashable,
    # we override on it directly for simplicity
    def __hash__(self) -> int:  # type: ignore[override]
        return hash(frozenset(self.items()))


98
99
100
101
102
103
104
105
106
class HashableList(list):
    """
    A list that can be hashed by lru_cache.
    """

    def __hash__(self) -> int:  # type: ignore[override]
        return hash(tuple(self))


107
def _get_processor_factory_fn(processor_cls: type | tuple[type, ...]):
108
109
110
111
112
113
114
    if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin:
        return AutoProcessor.from_pretrained
    if hasattr(processor_cls, "from_pretrained"):
        return processor_cls.from_pretrained

    return processor_cls

115

116
117
def _merge_mm_kwargs(
    model_config: "ModelConfig",
118
    processor_cls: type | tuple[type, ...],
119
120
121
122
123
124
125
126
127
128
129
130
131
    /,
    **kwargs,
):
    mm_config = model_config.get_multimodal_config()
    merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)

    factory = _get_processor_factory_fn(processor_cls)
    allowed_kwargs = get_allowed_kwarg_only_overrides(
        factory,
        merged_kwargs,
        requires_kw_only=False,
        allow_var_kwargs=True,
    )
132
133
134
    # NOTE: Pythonic dict is not hashable and will raise unhashable type
    # error when calling `cached_get_processor`, therefore we need to
    # wrap it to a hashable dict.
135
    for key, value in allowed_kwargs.items():
136
        if isinstance(value, dict):
137
            allowed_kwargs[key] = HashableDict(value)
138
        if isinstance(value, list):
139
140
141
            allowed_kwargs[key] = HashableList(value)

    return allowed_kwargs
142

143

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def get_processor_cls_name_from_config(
    processor_name: str,
    revision: str | None = "main",
) -> str | None:
    config_file = [
        "processor_config.json",
        "preprocessor_config.json",
        "tokenizer_config.json",
    ]
    for file in config_file:
        config = get_hf_file_to_dict(file, processor_name, revision=revision)
        if config and "processor_class" in config:
            return config["processor_class"]
    return None


160
161
def get_processor(
    processor_name: str,
162
    *args: Any,
163
    revision: str | None = None,
164
    trust_remote_code: bool = False,
165
    processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
166
    **kwargs: Any,
167
) -> _P:
168
    """Load a processor for the given model name via HuggingFace."""
169
170
    if revision is None:
        revision = "main"
171
    try:
172
        processor_name = convert_model_repo_to_path(processor_name)
173
174
175
176
177
178
179
180
181
182
183
        registered_cls_name = get_processor_cls_name_from_config(
            processor_name, revision=revision
        )
        registered_processor_cls = (
            getattr(processors, registered_cls_name, None)
            if registered_cls_name
            else None
        )
        registered_processor_cls = cast(type[_P] | None, registered_processor_cls)
        # Use registered processor class when it's available
        # and explicit processor_cls is not set.
184
        if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin:
185
186
            _processor_cls = registered_processor_cls or AutoProcessor
            processor = _processor_cls.from_pretrained(
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
                processor_name,
                *args,
                revision=revision,
                trust_remote_code=trust_remote_code,
                **kwargs,
            )
        elif issubclass(processor_cls, ProcessorMixin):
            processor = processor_cls.from_pretrained(
                processor_name,
                *args,
                revision=revision,
                trust_remote_code=trust_remote_code,
                **kwargs,
            )
        else:
            # Processors that are standalone classes unrelated to HF
            processor = processor_cls(*args, **kwargs)
204
205
206
207
208
209
210
211
212
213
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the processor. If the processor is "
                "a custom processor not yet available in the HuggingFace "
                "transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
214
215
                "`--trust-remote-code` flag in the CLI."
            )
216
217
218
219
            raise RuntimeError(err_msg) from e
        else:
            raise e

220
    if not isinstance(processor, processor_cls):
221
222
223
224
225
        raise TypeError(
            "Invalid type of HuggingFace processor. "
            f"Expected type: {processor_cls}, but "
            f"found type: {type(processor)}"
        )
226
227

    return processor
228
229


230
231
232
cached_get_processor = lru_cache(get_processor)


233
@lru_cache
234
235
236
def get_processor_kwargs_type(
    processor: ProcessorMixin,
) -> type[processing_utils.ProcessingKwargs]:
237
238
    try:
        # get kwargs annotations in processor
239
240
        call_params = inspect.signature(type(processor).__call__).parameters
        call_kwargs = call_params.get("kwargs")
241
        call_kwargs_annotations = call_kwargs.annotation if call_kwargs else None
242

243
        # if the processor has explicit kwargs annotation, use it
244
        if call_kwargs_annotations not in (None, inspect._empty):  # noqa: SIM102
245
246
247
248
            # get_type_hints will parse all type annotations at runtime,
            # and if an annotation refers to a type or
            # name that hasn’t been imported or defined, it will raise an error.
            # So we use __annotations__ to get the raw annotations directly.
249
250
            if anno_args := get_args(call_kwargs_annotations):
                return anno_args[0]
251
252
253
254
255
256
257
258

        # otherwise, try to get from ProcessorKwargs
        module_name = type(processor).__module__
        mod = importlib.import_module(module_name)
        for name, obj in vars(mod).items():
            if name.endswith("ProcessorKwargs"):
                return obj

259
    except Exception:
260
        logger.exception("Failed to collect processor kwargs")
261
262
263
264
265
266
267
268
269

    return processing_utils.ProcessingKwargs


@lru_cache
def get_processor_kwargs_keys(
    kwargs_cls: type[processing_utils.ProcessingKwargs],
) -> set[str]:
    dynamic_kwargs: set[str] = set()
270
271
272
273
274
275
276
    modality_kwargs = {
        "text_kwargs",
        "images_kwargs",
        "videos_kwargs",
        "audio_kwargs",
        "common_kwargs",
    }
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297

    try:
        # get kwargs annotations in processor
        # merge text_kwargs / images_kwargs / videos_kwargs / audio_kwargs
        kwargs_type_annotations = get_type_hints(kwargs_cls)
        for kw_type in modality_kwargs:
            if kw_type in kwargs_type_annotations:
                # Use __annotations__ instead of get_type_hints() to avoid
                # NameError from unresolved forward references (e.g.
                # PILImageResampling). We only need key names, not types.
                kw_cls = kwargs_type_annotations[kw_type]
                kw_annotations: dict[str, Any] = {}
                for base in reversed(kw_cls.__mro__):
                    kw_annotations.update(getattr(base, "__annotations__", {}))
                for kw_name in kw_annotations:
                    dynamic_kwargs.add(kw_name)

    except Exception:
        logger.exception("Failed to collect processor kwargs")

    return dynamic_kwargs | modality_kwargs
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316


def cached_get_processor_without_dynamic_kwargs(
    processor_name: str,
    *args: Any,
    revision: str | None = None,
    trust_remote_code: bool = False,
    processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
    **kwargs: Any,
) -> _P:
    # Step 1: use default kwargs to get a temporary processor instance
    processor = cached_get_processor(
        processor_name,
        revision=revision,
        trust_remote_code=trust_remote_code,
        processor_cls=processor_cls,  # type: ignore[arg-type]
    )

    # Step 2: use temporary processor collect dynamic keys
317
318
319
    dynamic_keys = get_processor_kwargs_keys(
        get_processor_kwargs_type(processor)  # type: ignore[arg-type]
    )
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335

    # Step 3: use dynamic_keys filter kwargs
    filtered_kwargs = {k: v for k, v in kwargs.items() if k not in dynamic_keys}

    # Step 4: use filtered kwargs to get final processor instance
    final_processor = cached_get_processor(
        processor_name,
        revision=revision,
        trust_remote_code=trust_remote_code,
        processor_cls=processor_cls,  # type: ignore[arg-type]
        **filtered_kwargs,
    )

    return final_processor


336
def cached_processor_from_config(
337
    model_config: "ModelConfig",
338
    processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
339
340
    **kwargs: Any,
) -> _P:
341
    if is_gguf(model_config.model):
342
        assert not is_gguf(model_config.tokenizer), (
343
344
345
            "For multimodal GGUF models, the original tokenizer "
            "should be used to correctly load processor."
        )
346
347
        model = model_config.tokenizer
        revision = model_config.tokenizer_revision
348
349
350
351
    else:
        model = model_config.model
        revision = model_config.revision

352
    return cached_get_processor_without_dynamic_kwargs(
353
354
        model,
        revision=revision,
355
356
        trust_remote_code=model_config.trust_remote_code,
        processor_cls=processor_cls,  # type: ignore[arg-type]
357
        **_merge_mm_kwargs(model_config, processor_cls, **kwargs),
358
359
360
    )


361
362
363
def get_feature_extractor(
    processor_name: str,
    *args: Any,
364
    revision: str | None = None,
365
366
367
    trust_remote_code: bool = False,
    **kwargs: Any,
):
368
    """Load an audio feature extractor for the given model name
369
370
    via HuggingFace."""
    try:
371
        processor_name = convert_model_repo_to_path(processor_name)
372
373
374
        feature_extractor = AutoFeatureExtractor.from_pretrained(
            processor_name,
            *args,
375
            revision=revision,
376
            trust_remote_code=trust_remote_code,
377
378
            **kwargs,
        )
379
380
381
382
383
384
385
386
387
388
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the feature extractor. If the feature "
                "extractor is a custom extractor not yet available in the "
                "HuggingFace transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
389
390
                "`--trust-remote-code` flag in the CLI."
            )
391
392
393
394
395
396
397
398
399
400
            raise RuntimeError(err_msg) from e
        else:
            raise e
    return cast(FeatureExtractionMixin, feature_extractor)


cached_get_feature_extractor = lru_cache(get_feature_extractor)


def cached_feature_extractor_from_config(
401
    model_config: "ModelConfig",
402
403
404
405
    **kwargs: Any,
):
    return cached_get_feature_extractor(
        model_config.model,
406
        revision=model_config.revision,
407
        trust_remote_code=model_config.trust_remote_code,
408
        **_merge_mm_kwargs(model_config, AutoFeatureExtractor, **kwargs),
409
410
411
    )


412
413
414
def get_image_processor(
    processor_name: str,
    *args: Any,
415
    revision: str | None = None,
416
417
418
419
420
    trust_remote_code: bool = False,
    **kwargs: Any,
):
    """Load an image processor for the given model name via HuggingFace."""
    try:
421
        processor_name = convert_model_repo_to_path(processor_name)
422
423
424
        processor = AutoImageProcessor.from_pretrained(
            processor_name,
            *args,
425
            revision=revision,
426
            trust_remote_code=trust_remote_code,
427
428
            **kwargs,
        )
429
430
431
432
433
434
435
436
437
438
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the image processor. If the image processor is "
                "a custom processor not yet available in the HuggingFace "
                "transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
439
440
                "`--trust-remote-code` flag in the CLI."
            )
441
442
443
444
445
446
447
            raise RuntimeError(err_msg) from e
        else:
            raise e

    return cast(BaseImageProcessor, processor)


448
449
450
451
cached_get_image_processor = lru_cache(get_image_processor)


def cached_image_processor_from_config(
452
    model_config: "ModelConfig",
453
454
    **kwargs: Any,
):
455
    if is_gguf(model_config.model):
456
        assert not is_gguf(model_config.tokenizer), (
457
458
459
            "For multimodal GGUF models, the original tokenizer "
            "should be used to correctly load image processor."
        )
460
461
        model = model_config.tokenizer
        revision = model_config.tokenizer_revision
462
463
464
    else:
        model = model_config.model
        revision = model_config.revision
465
    return cached_get_image_processor(
466
467
        model,
        revision=revision,
468
        trust_remote_code=model_config.trust_remote_code,
469
        **_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs),
470
    )
471
472
473
474
475


def get_video_processor(
    processor_name: str,
    *args: Any,
476
    revision: str | None = None,
477
    trust_remote_code: bool = False,
478
    processor_cls_overrides: type[_V] | None = None,
479
480
481
482
    **kwargs: Any,
):
    """Load a video processor for the given model name via HuggingFace."""
    try:
483
        processor_name = convert_model_repo_to_path(processor_name)
484
485
486
487
488
489
        processor_cls = processor_cls_overrides or AutoVideoProcessor
        processor = processor_cls.from_pretrained(
            processor_name,
            *args,
            revision=revision,
            trust_remote_code=trust_remote_code,
490
491
            **kwargs,
        )
492
493
494
495
496
497
498
499
500
501
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoVideoProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the video processor. If the video processor is "
                "a custom processor not yet available in the HuggingFace "
                "transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
502
503
                "`--trust-remote-code` flag in the CLI."
            )
504
505
506
507
508
509
510
511
512
513
514
            raise RuntimeError(err_msg) from e
        else:
            raise e

    return cast(BaseVideoProcessor, processor)


cached_get_video_processor = lru_cache(get_video_processor)


def cached_video_processor_from_config(
515
    model_config: "ModelConfig",
516
    processor_cls: type[_V] | None = None,
517
518
519
520
521
522
523
524
525
    **kwargs: Any,
):
    return cached_get_video_processor(
        model_config.model,
        revision=model_config.revision,
        trust_remote_code=model_config.trust_remote_code,
        processor_cls_overrides=processor_cls,  # type: ignore[arg-type]
        **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
    )