test_common.py 15.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from collections.abc import Set as AbstractSet
5
6
7
8
from functools import partial

import numpy as np
import pytest
9
10
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
11
from mistral_common.protocol.instruct.request import ChatCompletionRequest
12
13
14
from PIL import Image

from vllm.config import ModelConfig
15
16
17
18
19
20
from vllm.config.multimodal import (
    AudioDummyOptions,
    BaseDummyOptions,
    ImageDummyOptions,
    VideoDummyOptions,
)
21
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
22
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
23
from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
24
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
25
26
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.tokenizers.mistral import MistralTokenizer
27
28

from ....multimodal.utils import random_audio, random_image, random_video
29
30
31
32
33
from ...registry import (
    _MULTIMODAL_EXAMPLE_MODELS,
    _TRANSFORMERS_BACKEND_MODELS,
    HF_EXAMPLE_MODELS,
)
34
35


36
37
38
39
40
41
def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
    """
    Patch the multimodal data for GLM4.1V model.
    """
    # Ensure video metadata is included
    if "video" in mm_data:
42
        # GLM4.1V doesn't support multiple videos
43
        video = mm_data["video"]
44
        num_frames = len(video)
45
46
47
48
49
50
51
52
53
54
55
        mm_data["video"] = (
            video,
            {
                "total_num_frames": num_frames,
                "fps": num_frames,
                "duration": 1,
                "frames_indices": [i for i in range(num_frames)],
                "video_backend": "opencv",
                "do_sample_frames": True,
            },
        )
56
57
58
    return mm_data


59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
    """
    Patch the multimodal data for Qwen3-VL model.
    """

    def create_metadata(frames: np.ndarray):
        num_frames = len(frames)
        return {
            "total_num_frames": num_frames,
            "fps": 2.0,
            "duration": num_frames / 2.0,
            "video_backend": "opencv",
            "frames_indices": list(range(num_frames)),
            "do_sample_frames": True,
        }

    # Ensure video metadata is included
    if "video" in mm_data:
        video = mm_data["video"]
        if isinstance(video, list):
            # multiple videos
            mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
        else:
            # single video
            mm_data["video"] = (video, create_metadata(video))
    return mm_data


87
88
89
90
91
92
93
94
95
96
97
98
99
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
    """
    Patch the multimodal data for GLM-ASR model.
    GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
    """
    if "audio" in mm_data:
        audio = mm_data["audio"]
        if isinstance(audio, list) and len(audio) > 1:
            # Limit to single audio to match text requirement
            mm_data["audio"] = [audio[0]]
    return mm_data


100
101
102
103
104
# For some multimodal models, tokenizer will always add bos_token
# at the beginning of prompt by default, causing hf_processor outputs
# incorrect token ids. So we need use `add_special_tokens=False` here
# to leave bos_token to be added by the processor.
_ADD_SPECIAL_TOKENS_OVERRIDES = {
105
    "nemotron_parse": False,
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
    "ovis": False,
    "ovis2_5": False,
    "paligemma": False,
    "ultravox": False,
    "whisper": False,
}

_IGNORE_MM_KEYS = {
    # In Ultravox, the audio_features can be different depending on padding
    # The slight difference should not be a problem though, since
    # attention_mask lets us ignore the difference.
    "ultravox": {"audio_features"},
}

MM_DATA_PATCHES = {
121
122
    # Ernie4.5-VL, GLM4.1V and Qwen3-VL requires video metadata
    "ernie4_5_moe_vl": qwen3_vl_patch_mm_data,
123
124
    "glm4v": glm4_1v_patch_mm_data,
    "glm4v_moe": glm4_1v_patch_mm_data,
125
    "glm_ocr": glm4_1v_patch_mm_data,
126
    "glmasr": glmasr_patch_mm_data,
127
    "molmo2": qwen3_vl_patch_mm_data,
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
    "qwen3_vl": qwen3_vl_patch_mm_data,
    "qwen3_vl_moe": qwen3_vl_patch_mm_data,
}


def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
    for model_arch in model_arch_list:
        model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
        yield model_info.default

        for extra_type, extra_model_id in model_info.extras.items():
            if "fp" in extra_type:
                continue  # Redundant to test quantized models

            yield extra_model_id


def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
    return list(_iter_model_ids_to_test(model_arch_list))


def get_model_ids_to_test():
    transformers_arch_ids = {
        model_id
        for info in _TRANSFORMERS_BACKEND_MODELS.values()
        for model_id in (info.default, *info.extras.values())
    }
    vllm_only_archs = {
        arch
        for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
        if not any(
            model_id in transformers_arch_ids
            for model_id in (info.default, *info.extras.values())
        )
    }

    return _get_model_ids_to_test(vllm_only_archs)


def get_text_token_prompts(
    processor: BaseMultiModalProcessor,
    mm_data: MultiModalDataDict,
):
    dummy_inputs = processor.dummy_inputs
172
    tokenizer: TokenizerLike = processor.info.get_tokenizer()
173
174
175
176
177
178
    model_config = processor.info.ctx.model_config

    model_type = model_config.hf_config.model_type
    if model_type in MM_DATA_PATCHES:
        mm_data = MM_DATA_PATCHES[model_type](mm_data)

179
    parsed_data = processor.info.parse_mm_data(mm_data)
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
    mm_counts = {k: len(vs) for k, vs in parsed_data.items()}

    text_prompt: str | None
    token_prompt: list[int]
    if isinstance(tokenizer, MistralTokenizer):
        images = parsed_data.get("image", [])
        request = ChatCompletionRequest(
            messages=[
                UserMessage(
                    content=[
                        TextChunk(text=""),
                        *(ImageChunk(image=image) for image in images),
                    ]
                ),
            ]
        )
        res = tokenizer.mistral.encode_chat_completion(request)

        # Mistral does not support decode_tokens with skip_special_tokens=False
        text_prompt = None
        token_prompt = res.tokens
    else:
        inputs = dummy_inputs.get_dummy_processor_inputs(
            model_config.max_model_len,
            mm_counts,
        )
        assert isinstance(inputs.prompt, str)

        text_prompt = inputs.prompt
209
        token_prompt = tokenizer.encode(
210
            text_prompt,
211
            add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type, True),
212
213
214
215
216
        )

    return text_prompt, token_prompt


217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
def random_vision_chunk(
    rng: np.random.RandomState,
    min_wh: int,
    max_wh: int,
    min_frames: int,
    max_frames: int,
) -> dict:
    num_frames = rng.randint(min_frames, max_frames + 1)
    if num_frames == 1:
        # Single image chunk
        wh = rng.randint(min_wh, max_wh + 1)
        image = random_image(rng, wh, wh + 1)
        return {"type": "image", "image": image}
    frames = []
    for _ in range(num_frames):
        wh = rng.randint(min_wh, max_wh + 1)
        frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)
        frames.append(frame)
    video_array = np.stack(frames, axis=0)
    return {"type": "video_chunk", "video_chunk": video_array}


239
def _test_processing_correctness(
240
    model_id_or_arch: str,
241
242
243
244
    hit_rate: float,
    num_batches: int,
    simplify_rate: float,
):
245
246
247
248
249
250
251
    if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
        # Use model architecture to get the default model id
        model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
        model_id = model_info.default
    else:
        model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
        model_id = model_id_or_arch
252
    model_info.check_available_online(on_fail="skip")
253
254
255
256
257
    model_info.check_transformers_version(
        on_fail="skip",
        check_max_version=False,
        check_version_reason="vllm",
    )
258

259
260
261
262
263
264
265
266
267
268
269
270
    model_config = ModelConfig(
        model_id,
        tokenizer=model_info.tokenizer or model_id,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
        enforce_eager=model_info.enforce_eager,
        dtype=model_info.dtype,
271
    )
272
273
274
    # Ensure that the cache can fit all of the data
    # (set after because ModelConfig would set it to 0 for encoder-decoder models)
    model_config.multimodal_config.mm_processor_cache_gb = 2048
275
276

    model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
277
    factories = model_cls._processor_factory
278
279
280
281
    ctx = InputProcessingContext(
        model_config,
        tokenizer=cached_tokenizer_from_config(model_config),
    )
282
    cache = MultiModalProcessorOnlyCache(model_config)
283

284
285
    processing_info = factories.info(ctx)
    supported_mm_limits = processing_info.get_supported_mm_limits()
286
287
    # Keep integer limits for local data generation
    limit_mm_per_prompt_ints = {
288
289
290
291
        modality: 3 if limit is None else limit
        for modality, limit in supported_mm_limits.items()
    }

292
293
294
295
296
297
298
299
300
301
302
303
304
305
    def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
        if modality == "video":
            return VideoDummyOptions(count=count)
        if modality == "image":
            return ImageDummyOptions(count=count)
        if modality == "audio":
            return AudioDummyOptions(count=count)
        return BaseDummyOptions(count=count)

    # Assign normalized DummyOptions to the model config
    model_config.get_multimodal_config().limit_per_prompt = {
        modality: _to_dummy_options(modality, count)
        for modality, count in limit_mm_per_prompt_ints.items()
    }
306

307
308
309
310
311
312
313
314
    baseline_processor = factories.build_processor(ctx, cache=None)
    cached_processor = factories.build_processor(ctx, cache=cache)

    rng = np.random.RandomState(0)

    input_to_hit = {
        "image": Image.new("RGB", size=(128, 128)),
        "video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
315
        "audio": (np.zeros((512,)), 16000),
316
        "vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},
317
318
    }
    input_factory = {
319
320
321
322
323
        "image": partial(random_image, rng, min_wh=128, max_wh=256),
        "video": partial(
            random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256
        ),
        "audio": partial(random_audio, rng, min_len=512, max_len=1024, sr=16000),
324
325
326
        "vision_chunk": partial(
            random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1
        ),
327
328
329
330
    }

    for batch_idx in range(num_batches):
        mm_data = {
331
332
333
334
            k: [
                (input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
                for _ in range(rng.randint(limit + 1))
            ]
335
            for k, limit in limit_mm_per_prompt_ints.items()
336
337
338
339
340
341
342
343
344
345
        }

        # Drop unnecessary keys and test single -> multi conversion
        if rng.rand() < simplify_rate:
            for k in list(mm_data.keys()):
                if not mm_data[k]:
                    del mm_data[k]
                elif len(mm_data[k]) == 1:
                    mm_data[k] = mm_data[k][0]

346
347
348
349
350
351
352
353
354
355
        _test_processing_correctness_one(
            model_config,
            mm_data,
            baseline_processor,
            cached_processor,
            batch_idx,
        )


def _test_processing_correctness_one(
356
357
358
359
360
361
    model_config: ModelConfig,
    mm_data: MultiModalDataDict,
    baseline_processor: BaseMultiModalProcessor,
    cached_processor: BaseMultiModalProcessor,
    batch_idx: int,
):
362
363
    model_type = model_config.hf_config.model_type

364
    text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
365
    mm_items = baseline_processor.info.parse_mm_data(mm_data)
366
    ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
367
368
369

    baseline_tokenized_result = baseline_processor.apply(
        token_prompt,
370
        mm_items=mm_items,
371
372
373
374
375
        hf_processor_mm_kwargs={},
    )

    cached_tokenized_result = cached_processor.apply(
        token_prompt,
376
        mm_items=mm_items,
377
378
379
        hf_processor_mm_kwargs={},
    )

380
    _assert_inputs_equal(
381
382
        baseline_tokenized_result,
        cached_tokenized_result,
383
        ignore_mm_keys=ignore_mm_keys,
384
        msg=f"Failed ({batch_idx=}, {token_prompt=}, {mm_data=})",
385
    )
386

387
388
389
    if text_prompt is not None:
        baseline_text_result = baseline_processor.apply(
            text_prompt,
390
            mm_items=mm_items,
391
392
393
394
            hf_processor_mm_kwargs={},
        )
        cached_text_result = cached_processor.apply(
            text_prompt,
395
            mm_items=mm_items,
396
397
398
399
400
401
402
403
404
405
406
407
408
409
            hf_processor_mm_kwargs={},
        )

        _assert_inputs_equal(
            baseline_text_result,
            cached_text_result,
            ignore_mm_keys=ignore_mm_keys,
            msg=f"Failed ({batch_idx=}, {text_prompt=}, {mm_data=})",
        )

        _assert_inputs_equal(
            baseline_text_result,
            baseline_tokenized_result,
            ignore_mm_keys=ignore_mm_keys,
410
            msg=f"Failed ({batch_idx=}, {text_prompt=}, {token_prompt=}, {mm_data=})",
411
412
413
414
415
416
        )

        _assert_inputs_equal(
            cached_text_result,
            cached_tokenized_result,
            ignore_mm_keys=ignore_mm_keys,
417
            msg=f"Failed ({batch_idx=}, {text_prompt=}, {token_prompt=}, {mm_data=})",
418
419
        )

420

421
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
422
423
424
425
426
427
428
429
430
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness(
    model_id: str,
    hit_rate: float,
    num_batches: int,
    simplify_rate: float,
):
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
431
    if model_id == "google/gemma-3n-E2B-it":
432
433
434
435
436
        pytest.skip("Fix later")
    if model_id == "OpenGVLab/InternVL2-2B":
        pytest.skip("Fix later")
    if model_id == "jinaai/jina-reranker-m0":
        pytest.skip("Fix later")
437
438
439
440
441
    if model_id in {"Qwen/Qwen-VL", "Qwen/Qwen-VL-Chat"}:
        pytest.skip(
            "Qwen-VL tokenizer requires downloading a font file from "
            "servers that often refuse connections in CI"
        )
442

443
444
445
446
447
448
    _test_processing_correctness(
        model_id,
        hit_rate=hit_rate,
        num_batches=num_batches,
        simplify_rate=simplify_rate,
    )
449
450


451
def _assert_inputs_equal(
452
453
    a: MultiModalInputs,
    b: MultiModalInputs,
454
    *,
455
    ignore_mm_keys: set[str] | None = None,
456
    msg: str = "",
457
):
458
459
460
    if ignore_mm_keys is None:
        ignore_mm_keys = set()

461
462
463
464
465
466
467
    a_rest = {k: v for k, v in a.items() if k != "mm_kwargs"}
    b_rest = {k: v for k, v in b.items() if k != "mm_kwargs"}

    assert a_rest == b_rest, msg

    a_data = a["mm_kwargs"].get_data()
    b_data = b["mm_kwargs"].get_data()
468
469

    for key in ignore_mm_keys:
470
471
        a_data.pop(key, None)
        b_data.pop(key, None)
472

473
    assert batched_tensors_equal(a_data, b_data), msg