processor.py 18.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
import importlib
import inspect
6
from functools import lru_cache
7
from typing import TYPE_CHECKING, Any, cast, get_args, get_type_hints
8

9
10
11
12
13
from transformers import (
    AutoFeatureExtractor,
    AutoImageProcessor,
    AutoProcessor,
    AutoVideoProcessor,
14
    processing_utils,
15
)
16
17
from transformers.feature_extraction_utils import FeatureExtractionMixin
from transformers.image_processing_utils import BaseImageProcessor
18
from transformers.processing_utils import ProcessorMixin
19
from transformers.video_processing_utils import BaseVideoProcessor
20
21
from typing_extensions import TypeVar

22
from vllm.logger import init_logger
23
from vllm.transformers_utils import processors
24
from vllm.transformers_utils.gguf_utils import is_gguf
25
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
26
from vllm.transformers_utils.utils import convert_model_repo_to_path
27
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
28

29
30
logger = init_logger(__name__)

31
if TYPE_CHECKING:
32
    from vllm.config import ModelConfig
33

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

def _transformers_v4_compatibility_import():
    """Some remote code processors still import `ChatTemplateLoadKwargs` which was a
    subset of `ProcessorChatTemplateKwargs` as defined in Transformers v4.
    In Transformers v5 these were merged into `ProcessorChatTemplateKwargs` and
    `ChatTemplateLoadKwargs` was removed. For backward compatibility, we add an alias
    for `ChatTemplateLoadKwargs` if it doesn't exist.

    This can be removed if `HCXVisionForCausalLM` is upstreamed to Transformers."""
    old_import = getattr(processing_utils, "ChatTemplateLoadKwargs", None)
    new_import = getattr(processing_utils, "ProcessorChatTemplateKwargs", None)
    if old_import is None and new_import is not None:
        processing_utils.ChatTemplateLoadKwargs = new_import


49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def _transformers_v4_compatibility_init() -> Any:
    """Some remote code processors may define `optional_attributes` in their
    `ProcessorMixin` subclass, and then pass these arbitrary attributes directly to
    `ProcessorMixin.__init__`, which is no longer allowed in Transformers v5. For
    backward compatibility, we intercept these optional attributes and set them on the
    processor instance before calling the original `ProcessorMixin.__init__`.

    This can be removed if `Molmo2ForConditionalGeneration` is upstreamed to
    Transformers."""
    # Transformers v4
    if hasattr(ProcessorMixin, "optional_attributes"):
        return
    # Transformers v5
    if hasattr(ProcessorMixin.__init__, "_vllm_patched"):
        return

    original_init = ProcessorMixin.__init__

    def __init__(self, *args, **kwargs):
        for optional_attribute in getattr(self, "optional_attributes", []):
            if optional_attribute in kwargs:
                setattr(self, optional_attribute, kwargs.pop(optional_attribute))

        original_init(self, *args, **kwargs)

    # Only patch if ProcessorMixin is not mocked (for docs builds)
    if not hasattr(ProcessorMixin, "_mock_name"):
        __init__._vllm_patched = True  # type: ignore[attr-defined]
        ProcessorMixin.__init__ = __init__


80
_transformers_v4_compatibility_import()
81
_transformers_v4_compatibility_init()
82

83
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
84
_V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor)
85
86
87
88
89
90
91
92
93
94
95
96
97


class HashableDict(dict):
    """
    A dictionary that can be hashed by lru_cache.
    """

    # NOTE: pythonic dict is not hashable,
    # we override on it directly for simplicity
    def __hash__(self) -> int:  # type: ignore[override]
        return hash(frozenset(self.items()))


98
99
100
101
102
103
104
105
106
class HashableList(list):
    """
    A list that can be hashed by lru_cache.
    """

    def __hash__(self) -> int:  # type: ignore[override]
        return hash(tuple(self))


107
def _get_processor_factory_fn(processor_cls: type | tuple[type, ...]):
108
109
110
111
112
113
114
    if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin:
        return AutoProcessor.from_pretrained
    if hasattr(processor_cls, "from_pretrained"):
        return processor_cls.from_pretrained

    return processor_cls

115

116
117
def _merge_mm_kwargs(
    model_config: "ModelConfig",
118
    processor_cls: type | tuple[type, ...],
119
120
121
122
123
124
125
126
127
128
129
130
131
    /,
    **kwargs,
):
    mm_config = model_config.get_multimodal_config()
    merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)

    factory = _get_processor_factory_fn(processor_cls)
    allowed_kwargs = get_allowed_kwarg_only_overrides(
        factory,
        merged_kwargs,
        requires_kw_only=False,
        allow_var_kwargs=True,
    )
132
133
134
    # NOTE: Pythonic dict is not hashable and will raise unhashable type
    # error when calling `cached_get_processor`, therefore we need to
    # wrap it to a hashable dict.
135
    for key, value in allowed_kwargs.items():
136
        if isinstance(value, dict):
137
            allowed_kwargs[key] = HashableDict(value)
138
        if isinstance(value, list):
139
140
141
            allowed_kwargs[key] = HashableList(value)

    return allowed_kwargs
142

143

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def get_processor_cls_name_from_config(
    processor_name: str,
    revision: str | None = "main",
) -> str | None:
    config_file = [
        "processor_config.json",
        "preprocessor_config.json",
        "tokenizer_config.json",
    ]
    for file in config_file:
        config = get_hf_file_to_dict(file, processor_name, revision=revision)
        if config and "processor_class" in config:
            return config["processor_class"]
    return None


160
161
def get_processor(
    processor_name: str,
162
    *args: Any,
163
    revision: str | None = None,
164
    trust_remote_code: bool = False,
165
    processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
166
    **kwargs: Any,
167
) -> _P:
168
    """Load a processor for the given model name via HuggingFace."""
169
170
    if revision is None:
        revision = "main"
171
    try:
172
        processor_name = convert_model_repo_to_path(processor_name)
173
174
175
176
177
178
179
180
181
182
183
        registered_cls_name = get_processor_cls_name_from_config(
            processor_name, revision=revision
        )
        registered_processor_cls = (
            getattr(processors, registered_cls_name, None)
            if registered_cls_name
            else None
        )
        registered_processor_cls = cast(type[_P] | None, registered_processor_cls)
        # Use registered processor class when it's available
        # and explicit processor_cls is not set.
184
        if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin:
185
186
            _processor_cls = registered_processor_cls or AutoProcessor
            processor = _processor_cls.from_pretrained(
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
                processor_name,
                *args,
                revision=revision,
                trust_remote_code=trust_remote_code,
                **kwargs,
            )
        elif issubclass(processor_cls, ProcessorMixin):
            processor = processor_cls.from_pretrained(
                processor_name,
                *args,
                revision=revision,
                trust_remote_code=trust_remote_code,
                **kwargs,
            )
        else:
            # Processors that are standalone classes unrelated to HF
            processor = processor_cls(*args, **kwargs)
204
205
206
207
208
209
210
211
212
213
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the processor. If the processor is "
                "a custom processor not yet available in the HuggingFace "
                "transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
214
215
                "`--trust-remote-code` flag in the CLI."
            )
216
217
218
219
            raise RuntimeError(err_msg) from e
        else:
            raise e

220
    if not isinstance(processor, processor_cls):
221
222
223
224
225
        raise TypeError(
            "Invalid type of HuggingFace processor. "
            f"Expected type: {processor_cls}, but "
            f"found type: {type(processor)}"
        )
226
227

    return processor
228
229


230
231
232
cached_get_processor = lru_cache(get_processor)


233
@lru_cache
234
235
236
def get_processor_kwargs_type(
    processor: ProcessorMixin,
) -> type[processing_utils.ProcessingKwargs]:
237
238
    try:
        # get kwargs annotations in processor
239
240
        call_params = inspect.signature(type(processor).__call__).parameters
        call_kwargs = call_params.get("kwargs")
241
        call_kwargs_annotations = call_kwargs.annotation if call_kwargs else None
242

243
244
245
246
247
248
        # if the processor has explicit kwargs annotation, use it
        if call_kwargs_annotations not in (None, inspect._empty):
            # get_type_hints will parse all type annotations at runtime,
            # and if an annotation refers to a type or
            # name that hasn’t been imported or defined, it will raise an error.
            # So we use __annotations__ to get the raw annotations directly.
249
250
251
252
253
254
255
256
257
            return get_args(call_kwargs_annotations)[0]

        # otherwise, try to get from ProcessorKwargs
        module_name = type(processor).__module__
        mod = importlib.import_module(module_name)
        for name, obj in vars(mod).items():
            if name.endswith("ProcessorKwargs"):
                return obj

258
    except Exception:
259
        logger.exception("Failed to collect processor kwargs")
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290

    return processing_utils.ProcessingKwargs


@lru_cache
def get_processor_kwargs_keys(
    kwargs_cls: type[processing_utils.ProcessingKwargs],
) -> set[str]:
    dynamic_kwargs: set[str] = set()
    modality_kwargs = {"text_kwargs", "images_kwargs", "videos_kwargs", "audio_kwargs"}

    try:
        # get kwargs annotations in processor
        # merge text_kwargs / images_kwargs / videos_kwargs / audio_kwargs
        kwargs_type_annotations = get_type_hints(kwargs_cls)
        for kw_type in modality_kwargs:
            if kw_type in kwargs_type_annotations:
                # Use __annotations__ instead of get_type_hints() to avoid
                # NameError from unresolved forward references (e.g.
                # PILImageResampling). We only need key names, not types.
                kw_cls = kwargs_type_annotations[kw_type]
                kw_annotations: dict[str, Any] = {}
                for base in reversed(kw_cls.__mro__):
                    kw_annotations.update(getattr(base, "__annotations__", {}))
                for kw_name in kw_annotations:
                    dynamic_kwargs.add(kw_name)

    except Exception:
        logger.exception("Failed to collect processor kwargs")

    return dynamic_kwargs | modality_kwargs
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309


def cached_get_processor_without_dynamic_kwargs(
    processor_name: str,
    *args: Any,
    revision: str | None = None,
    trust_remote_code: bool = False,
    processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
    **kwargs: Any,
) -> _P:
    # Step 1: use default kwargs to get a temporary processor instance
    processor = cached_get_processor(
        processor_name,
        revision=revision,
        trust_remote_code=trust_remote_code,
        processor_cls=processor_cls,  # type: ignore[arg-type]
    )

    # Step 2: use temporary processor collect dynamic keys
310
311
312
    dynamic_keys = get_processor_kwargs_keys(
        get_processor_kwargs_type(processor)  # type: ignore[arg-type]
    )
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328

    # Step 3: use dynamic_keys filter kwargs
    filtered_kwargs = {k: v for k, v in kwargs.items() if k not in dynamic_keys}

    # Step 4: use filtered kwargs to get final processor instance
    final_processor = cached_get_processor(
        processor_name,
        revision=revision,
        trust_remote_code=trust_remote_code,
        processor_cls=processor_cls,  # type: ignore[arg-type]
        **filtered_kwargs,
    )

    return final_processor


329
def cached_processor_from_config(
330
    model_config: "ModelConfig",
331
    processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
332
333
    **kwargs: Any,
) -> _P:
334
    if is_gguf(model_config.model):
335
        assert not is_gguf(model_config.tokenizer), (
336
337
338
            "For multimodal GGUF models, the original tokenizer "
            "should be used to correctly load processor."
        )
339
340
        model = model_config.tokenizer
        revision = model_config.tokenizer_revision
341
342
343
344
    else:
        model = model_config.model
        revision = model_config.revision

345
    return cached_get_processor_without_dynamic_kwargs(
346
347
        model,
        revision=revision,
348
349
        trust_remote_code=model_config.trust_remote_code,
        processor_cls=processor_cls,  # type: ignore[arg-type]
350
        **_merge_mm_kwargs(model_config, processor_cls, **kwargs),
351
352
353
    )


354
355
356
def get_feature_extractor(
    processor_name: str,
    *args: Any,
357
    revision: str | None = None,
358
359
360
    trust_remote_code: bool = False,
    **kwargs: Any,
):
361
    """Load an audio feature extractor for the given model name
362
363
    via HuggingFace."""
    try:
364
        processor_name = convert_model_repo_to_path(processor_name)
365
366
367
        feature_extractor = AutoFeatureExtractor.from_pretrained(
            processor_name,
            *args,
368
            revision=revision,
369
            trust_remote_code=trust_remote_code,
370
371
            **kwargs,
        )
372
373
374
375
376
377
378
379
380
381
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the feature extractor. If the feature "
                "extractor is a custom extractor not yet available in the "
                "HuggingFace transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
382
383
                "`--trust-remote-code` flag in the CLI."
            )
384
385
386
387
388
389
390
391
392
393
            raise RuntimeError(err_msg) from e
        else:
            raise e
    return cast(FeatureExtractionMixin, feature_extractor)


cached_get_feature_extractor = lru_cache(get_feature_extractor)


def cached_feature_extractor_from_config(
394
    model_config: "ModelConfig",
395
396
397
398
    **kwargs: Any,
):
    return cached_get_feature_extractor(
        model_config.model,
399
        revision=model_config.revision,
400
        trust_remote_code=model_config.trust_remote_code,
401
        **_merge_mm_kwargs(model_config, AutoFeatureExtractor, **kwargs),
402
403
404
    )


405
406
407
def get_image_processor(
    processor_name: str,
    *args: Any,
408
    revision: str | None = None,
409
410
411
412
413
    trust_remote_code: bool = False,
    **kwargs: Any,
):
    """Load an image processor for the given model name via HuggingFace."""
    try:
414
        processor_name = convert_model_repo_to_path(processor_name)
415
416
417
        processor = AutoImageProcessor.from_pretrained(
            processor_name,
            *args,
418
            revision=revision,
419
            trust_remote_code=trust_remote_code,
420
421
            **kwargs,
        )
422
423
424
425
426
427
428
429
430
431
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the image processor. If the image processor is "
                "a custom processor not yet available in the HuggingFace "
                "transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
432
433
                "`--trust-remote-code` flag in the CLI."
            )
434
435
436
437
438
439
440
            raise RuntimeError(err_msg) from e
        else:
            raise e

    return cast(BaseImageProcessor, processor)


441
442
443
444
cached_get_image_processor = lru_cache(get_image_processor)


def cached_image_processor_from_config(
445
    model_config: "ModelConfig",
446
447
    **kwargs: Any,
):
448
    if is_gguf(model_config.model):
449
        assert not is_gguf(model_config.tokenizer), (
450
451
452
            "For multimodal GGUF models, the original tokenizer "
            "should be used to correctly load image processor."
        )
453
454
        model = model_config.tokenizer
        revision = model_config.tokenizer_revision
455
456
457
    else:
        model = model_config.model
        revision = model_config.revision
458
    return cached_get_image_processor(
459
460
        model,
        revision=revision,
461
        trust_remote_code=model_config.trust_remote_code,
462
        **_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs),
463
    )
464
465
466
467
468


def get_video_processor(
    processor_name: str,
    *args: Any,
469
    revision: str | None = None,
470
    trust_remote_code: bool = False,
471
    processor_cls_overrides: type[_V] | None = None,
472
473
474
475
    **kwargs: Any,
):
    """Load a video processor for the given model name via HuggingFace."""
    try:
476
        processor_name = convert_model_repo_to_path(processor_name)
477
478
479
480
481
482
        processor_cls = processor_cls_overrides or AutoVideoProcessor
        processor = processor_cls.from_pretrained(
            processor_name,
            *args,
            revision=revision,
            trust_remote_code=trust_remote_code,
483
484
            **kwargs,
        )
485
486
487
488
489
490
491
492
493
494
    except ValueError as e:
        # If the error pertains to the processor class not existing or not
        # currently being imported, suggest using the --trust-remote-code flag.
        # Unlike AutoTokenizer, AutoVideoProcessor does not separate such errors
        if not trust_remote_code:
            err_msg = (
                "Failed to load the video processor. If the video processor is "
                "a custom processor not yet available in the HuggingFace "
                "transformers library, consider setting "
                "`trust_remote_code=True` in LLM or using the "
495
496
                "`--trust-remote-code` flag in the CLI."
            )
497
498
499
500
501
502
503
504
505
506
507
            raise RuntimeError(err_msg) from e
        else:
            raise e

    return cast(BaseVideoProcessor, processor)


cached_get_video_processor = lru_cache(get_video_processor)


def cached_video_processor_from_config(
508
    model_config: "ModelConfig",
509
    processor_cls: type[_V] | None = None,
510
511
512
513
514
515
516
517
518
    **kwargs: Any,
):
    return cached_get_video_processor(
        model_config.model,
        revision=model_config.revision,
        trust_remote_code=model_config.trust_remote_code,
        processor_cls_overrides=processor_cls,  # type: ignore[arg-type]
        **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
    )