test_common.py 15.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from collections.abc import Set as AbstractSet
5
6
7
8
from functools import partial

import numpy as np
import pytest
9
10
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
11
from mistral_common.protocol.instruct.request import ChatCompletionRequest
12
13
14
from PIL import Image

from vllm.config import ModelConfig
15
16
17
18
19
20
from vllm.config.multimodal import (
    AudioDummyOptions,
    BaseDummyOptions,
    ImageDummyOptions,
    VideoDummyOptions,
)
21
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
22
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
23
from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
24
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
25
26
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.tokenizers.mistral import MistralTokenizer
27
28

from ....multimodal.utils import random_audio, random_image, random_video
29
30
31
32
33
from ...registry import (
    _MULTIMODAL_EXAMPLE_MODELS,
    _TRANSFORMERS_BACKEND_MODELS,
    HF_EXAMPLE_MODELS,
)
34
35


36
37
38
39
40
41
def glm4_1v_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
    """
    Patch the multimodal data for GLM4.1V model.
    """
    # Ensure video metadata is included
    if "video" in mm_data:
42
        # GLM4.1V doesn't support multiple videos
43
        video = mm_data["video"]
44
        num_frames = len(video)
45
46
47
48
49
50
51
52
53
54
55
        mm_data["video"] = (
            video,
            {
                "total_num_frames": num_frames,
                "fps": num_frames,
                "duration": 1,
                "frames_indices": [i for i in range(num_frames)],
                "video_backend": "opencv",
                "do_sample_frames": True,
            },
        )
56
57
58
    return mm_data


59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
def qwen3_vl_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
    """
    Patch the multimodal data for Qwen3-VL model.
    """

    def create_metadata(frames: np.ndarray):
        num_frames = len(frames)
        return {
            "total_num_frames": num_frames,
            "fps": 2.0,
            "duration": num_frames / 2.0,
            "video_backend": "opencv",
            "frames_indices": list(range(num_frames)),
            "do_sample_frames": True,
        }

    # Ensure video metadata is included
    if "video" in mm_data:
        video = mm_data["video"]
        if isinstance(video, list):
            # multiple videos
            mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
        else:
            # single video
            mm_data["video"] = (video, create_metadata(video))
    return mm_data


87
88
89
90
91
92
93
94
95
96
97
98
99
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
    """
    Patch the multimodal data for GLM-ASR model.
    GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
    """
    if "audio" in mm_data:
        audio = mm_data["audio"]
        if isinstance(audio, list) and len(audio) > 1:
            # Limit to single audio to match text requirement
            mm_data["audio"] = [audio[0]]
    return mm_data


100
101
102
103
104
# For some multimodal models, tokenizer will always add bos_token
# at the beginning of prompt by default, causing hf_processor outputs
# incorrect token ids. So we need use `add_special_tokens=False` here
# to leave bos_token to be added by the processor.
_ADD_SPECIAL_TOKENS_OVERRIDES = {
105
    "nemotron_parse": False,
106
107
108
109
110
    "ovis": False,
    "ovis2_5": False,
    "paligemma": False,
    "ultravox": False,
    "whisper": False,
111
    "lfm2_vl": False,
112
113
114
115
116
117
118
119
120
121
}

_IGNORE_MM_KEYS = {
    # In Ultravox, the audio_features can be different depending on padding
    # The slight difference should not be a problem though, since
    # attention_mask lets us ignore the difference.
    "ultravox": {"audio_features"},
}

MM_DATA_PATCHES = {
122
123
    # Ernie4.5-VL, GLM4.1V and Qwen3-VL requires video metadata
    "ernie4_5_moe_vl": qwen3_vl_patch_mm_data,
124
125
    "glm4v": glm4_1v_patch_mm_data,
    "glm4v_moe": glm4_1v_patch_mm_data,
126
    "glm_ocr": glm4_1v_patch_mm_data,
127
    "glmasr": glmasr_patch_mm_data,
128
    "interns1_pro": qwen3_vl_patch_mm_data,
129
    "molmo2": qwen3_vl_patch_mm_data,
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
    "qwen3_vl": qwen3_vl_patch_mm_data,
    "qwen3_vl_moe": qwen3_vl_patch_mm_data,
}


def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
    for model_arch in model_arch_list:
        model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
        yield model_info.default

        for extra_type, extra_model_id in model_info.extras.items():
            if "fp" in extra_type:
                continue  # Redundant to test quantized models

            yield extra_model_id


def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
    return list(_iter_model_ids_to_test(model_arch_list))


def get_model_ids_to_test():
    transformers_arch_ids = {
        model_id
        for info in _TRANSFORMERS_BACKEND_MODELS.values()
        for model_id in (info.default, *info.extras.values())
    }
    vllm_only_archs = {
        arch
        for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
        if not any(
            model_id in transformers_arch_ids
            for model_id in (info.default, *info.extras.values())
        )
    }

    return _get_model_ids_to_test(vllm_only_archs)


def get_text_token_prompts(
    processor: BaseMultiModalProcessor,
    mm_data: MultiModalDataDict,
):
    dummy_inputs = processor.dummy_inputs
174
    tokenizer: TokenizerLike = processor.info.get_tokenizer()
175
176
177
178
179
180
    model_config = processor.info.ctx.model_config

    model_type = model_config.hf_config.model_type
    if model_type in MM_DATA_PATCHES:
        mm_data = MM_DATA_PATCHES[model_type](mm_data)

181
    parsed_data = processor.info.parse_mm_data(mm_data)
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
    mm_counts = {k: len(vs) for k, vs in parsed_data.items()}

    text_prompt: str | None
    token_prompt: list[int]
    if isinstance(tokenizer, MistralTokenizer):
        images = parsed_data.get("image", [])
        request = ChatCompletionRequest(
            messages=[
                UserMessage(
                    content=[
                        TextChunk(text=""),
                        *(ImageChunk(image=image) for image in images),
                    ]
                ),
            ]
        )
        res = tokenizer.mistral.encode_chat_completion(request)

        # Mistral does not support decode_tokens with skip_special_tokens=False
        text_prompt = None
        token_prompt = res.tokens
    else:
        inputs = dummy_inputs.get_dummy_processor_inputs(
            model_config.max_model_len,
            mm_counts,
        )
        assert isinstance(inputs.prompt, str)

        text_prompt = inputs.prompt
211
        token_prompt = tokenizer.encode(
212
            text_prompt,
213
            add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type, True),
214
215
216
217
218
        )

    return text_prompt, token_prompt


219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
def random_vision_chunk(
    rng: np.random.RandomState,
    min_wh: int,
    max_wh: int,
    min_frames: int,
    max_frames: int,
) -> dict:
    num_frames = rng.randint(min_frames, max_frames + 1)
    if num_frames == 1:
        # Single image chunk
        wh = rng.randint(min_wh, max_wh + 1)
        image = random_image(rng, wh, wh + 1)
        return {"type": "image", "image": image}
    frames = []
    for _ in range(num_frames):
        wh = rng.randint(min_wh, max_wh + 1)
        frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)
        frames.append(frame)
    video_array = np.stack(frames, axis=0)
    return {"type": "video_chunk", "video_chunk": video_array}


241
def _test_processing_correctness(
242
    model_id_or_arch: str,
243
244
245
246
    hit_rate: float,
    num_batches: int,
    simplify_rate: float,
):
247
248
249
250
251
252
253
    if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
        # Use model architecture to get the default model id
        model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
        model_id = model_info.default
    else:
        model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
        model_id = model_id_or_arch
254
    model_info.check_available_online(on_fail="skip")
255
256
257
258
259
    model_info.check_transformers_version(
        on_fail="skip",
        check_max_version=False,
        check_version_reason="vllm",
    )
260

261
262
263
264
265
266
267
268
269
270
271
272
    model_config = ModelConfig(
        model_id,
        tokenizer=model_info.tokenizer or model_id,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
        enforce_eager=model_info.enforce_eager,
        dtype=model_info.dtype,
273
    )
274
275
276
    # Ensure that the cache can fit all of the data
    # (set after because ModelConfig would set it to 0 for encoder-decoder models)
    model_config.multimodal_config.mm_processor_cache_gb = 2048
277
278

    model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
279
    factories = model_cls._processor_factory
280
281
282
283
    ctx = InputProcessingContext(
        model_config,
        tokenizer=cached_tokenizer_from_config(model_config),
    )
284
    cache = MultiModalProcessorOnlyCache(model_config)
285

286
287
    processing_info = factories.info(ctx)
    supported_mm_limits = processing_info.get_supported_mm_limits()
288
289
    # Keep integer limits for local data generation
    limit_mm_per_prompt_ints = {
290
291
292
293
        modality: 3 if limit is None else limit
        for modality, limit in supported_mm_limits.items()
    }

294
295
296
297
298
299
300
301
302
303
304
305
306
307
    def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
        if modality == "video":
            return VideoDummyOptions(count=count)
        if modality == "image":
            return ImageDummyOptions(count=count)
        if modality == "audio":
            return AudioDummyOptions(count=count)
        return BaseDummyOptions(count=count)

    # Assign normalized DummyOptions to the model config
    model_config.get_multimodal_config().limit_per_prompt = {
        modality: _to_dummy_options(modality, count)
        for modality, count in limit_mm_per_prompt_ints.items()
    }
308

309
310
311
312
313
314
315
316
    baseline_processor = factories.build_processor(ctx, cache=None)
    cached_processor = factories.build_processor(ctx, cache=cache)

    rng = np.random.RandomState(0)

    input_to_hit = {
        "image": Image.new("RGB", size=(128, 128)),
        "video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
317
        "audio": (np.zeros((512,)), 16000),
318
        "vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},
319
320
    }
    input_factory = {
321
322
323
324
325
        "image": partial(random_image, rng, min_wh=128, max_wh=256),
        "video": partial(
            random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256
        ),
        "audio": partial(random_audio, rng, min_len=512, max_len=1024, sr=16000),
326
327
328
        "vision_chunk": partial(
            random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1
        ),
329
330
331
332
    }

    for batch_idx in range(num_batches):
        mm_data = {
333
334
335
336
            k: [
                (input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
                for _ in range(rng.randint(limit + 1))
            ]
337
            for k, limit in limit_mm_per_prompt_ints.items()
338
339
340
341
342
343
344
345
346
347
        }

        # Drop unnecessary keys and test single -> multi conversion
        if rng.rand() < simplify_rate:
            for k in list(mm_data.keys()):
                if not mm_data[k]:
                    del mm_data[k]
                elif len(mm_data[k]) == 1:
                    mm_data[k] = mm_data[k][0]

348
349
350
351
352
353
354
355
356
357
        _test_processing_correctness_one(
            model_config,
            mm_data,
            baseline_processor,
            cached_processor,
            batch_idx,
        )


def _test_processing_correctness_one(
358
359
360
361
362
363
    model_config: ModelConfig,
    mm_data: MultiModalDataDict,
    baseline_processor: BaseMultiModalProcessor,
    cached_processor: BaseMultiModalProcessor,
    batch_idx: int,
):
364
365
    model_type = model_config.hf_config.model_type

366
    text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)
367
    mm_items = baseline_processor.info.parse_mm_data(mm_data)
368
    ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
369
370
371

    baseline_tokenized_result = baseline_processor.apply(
        token_prompt,
372
        mm_items=mm_items,
373
374
375
376
377
        hf_processor_mm_kwargs={},
    )

    cached_tokenized_result = cached_processor.apply(
        token_prompt,
378
        mm_items=mm_items,
379
380
381
        hf_processor_mm_kwargs={},
    )

382
    _assert_inputs_equal(
383
384
        baseline_tokenized_result,
        cached_tokenized_result,
385
        ignore_mm_keys=ignore_mm_keys,
386
        msg=f"Failed ({batch_idx=}, {token_prompt=}, {mm_data=})",
387
    )
388

389
390
391
    if text_prompt is not None:
        baseline_text_result = baseline_processor.apply(
            text_prompt,
392
            mm_items=mm_items,
393
394
395
396
            hf_processor_mm_kwargs={},
        )
        cached_text_result = cached_processor.apply(
            text_prompt,
397
            mm_items=mm_items,
398
399
400
401
402
403
404
405
406
407
408
409
410
411
            hf_processor_mm_kwargs={},
        )

        _assert_inputs_equal(
            baseline_text_result,
            cached_text_result,
            ignore_mm_keys=ignore_mm_keys,
            msg=f"Failed ({batch_idx=}, {text_prompt=}, {mm_data=})",
        )

        _assert_inputs_equal(
            baseline_text_result,
            baseline_tokenized_result,
            ignore_mm_keys=ignore_mm_keys,
412
            msg=f"Failed ({batch_idx=}, {text_prompt=}, {token_prompt=}, {mm_data=})",
413
414
415
416
417
418
        )

        _assert_inputs_equal(
            cached_text_result,
            cached_tokenized_result,
            ignore_mm_keys=ignore_mm_keys,
419
            msg=f"Failed ({batch_idx=}, {text_prompt=}, {token_prompt=}, {mm_data=})",
420
421
        )

422

423
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
424
425
426
427
428
429
430
431
432
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness(
    model_id: str,
    hit_rate: float,
    num_batches: int,
    simplify_rate: float,
):
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
433
    if model_id == "google/gemma-3n-E2B-it":
434
435
436
437
438
        pytest.skip("Fix later")
    if model_id == "OpenGVLab/InternVL2-2B":
        pytest.skip("Fix later")
    if model_id == "jinaai/jina-reranker-m0":
        pytest.skip("Fix later")
439
440
441
442
443
    if model_id in {"Qwen/Qwen-VL", "Qwen/Qwen-VL-Chat"}:
        pytest.skip(
            "Qwen-VL tokenizer requires downloading a font file from "
            "servers that often refuse connections in CI"
        )
444
445
446
447
448
449
450
    if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
        pytest.skip(
            "Voxtral Realtime doesn't make use of any place-holder"
            "tokens and hence cannot pass the processing "
            "correctness test as is. Let's revisit adapting this "
            "test once more realtime models exist."
        )
451
452
453
    if model_id == "internlm/Intern-S1-Pro":
        # FIXME(Isotr0py): Fix later.
        pytest.skip("Tokenization issue. Fix later")
454

455
456
457
458
459
460
    _test_processing_correctness(
        model_id,
        hit_rate=hit_rate,
        num_batches=num_batches,
        simplify_rate=simplify_rate,
    )
461
462


463
def _assert_inputs_equal(
464
465
    a: MultiModalInputs,
    b: MultiModalInputs,
466
    *,
467
    ignore_mm_keys: set[str] | None = None,
468
    msg: str = "",
469
):
470
471
472
    if ignore_mm_keys is None:
        ignore_mm_keys = set()

473
474
475
476
477
478
479
    a_rest = {k: v for k, v in a.items() if k != "mm_kwargs"}
    b_rest = {k: v for k, v in b.items() if k != "mm_kwargs"}

    assert a_rest == b_rest, msg

    a_data = a["mm_kwargs"].get_data()
    b_data = b["mm_kwargs"].get_data()
480
481

    for key in ignore_mm_keys:
482
483
        a_data.pop(key, None)
        b_data.pop(key, None)
484

485
    assert batched_tensors_equal(a_data, b_data), msg