test_common.py 53.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
"""Common tests for testing .generate() functionality for single / multiple
image, embedding, and video support for different VLMs in vLLM.
"""
6

7
8
import math
from collections import defaultdict
9
10
11
from pathlib import PosixPath

import pytest
12
from packaging.version import Version
13
14
from transformers import (
    AutoModel,
15
    AutoModelForCausalLM,
16
17
18
    AutoModelForImageTextToText,
    AutoModelForTextToWaveform,
)
19
from transformers import __version__ as TRANSFORMERS_VERSION
20
21

from vllm.platforms import current_platform
22
from vllm.utils.func_utils import identity
23

24
25
26
27
28
29
30
31
32
from ....conftest import (
    IMAGE_ASSETS,
    AudioTestAssets,
    HfRunner,
    ImageTestAssets,
    VideoTestAssets,
    VllmRunner,
)
from ....utils import create_new_process_for_each_test, large_gpu_mark, multi_gpu_marks
33
34
35
from ...utils import check_outputs_equal
from .vlm_utils import custom_inputs, model_utils, runners
from .vlm_utils.case_filtering import get_parametrized_options
36
37
38
39
40
41
from .vlm_utils.types import (
    CustomTestOptions,
    ExpandableVLMTestArgs,
    VLMTestInfo,
    VLMTestType,
)
42
43
44
45
46
47

COMMON_BROADCAST_SETTINGS = {
    "test_type": VLMTestType.IMAGE,
    "dtype": "half",
    "max_tokens": 5,
    "tensor_parallel_size": 2,
48
    "hf_model_kwargs": {"device_map": "auto"},
49
    "image_size_factors": [(0.25, 0.5, 1.0)],
50
51
52
    "distributed_executor_backend": (
        "ray",
        "mp",
53
    ),
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
}

### Test configuration for specific models
# NOTE: The convention of the test settings below is to lead each test key
# with the name of the model arch used in the test, using underscores in place
# of hyphens; this makes it more convenient to filter tests for a specific kind
# of model. For example....
#
# To run all test types for a specific key:
#     use the k flag to substring match with a leading square bracket; if the
#     model arch happens to be a substring of another one, you can add a
#     trailing hyphen. E.g.,
#                 - pytest $TEST_FILE -k "[llava-"
#     prevents matching on "[llava_next-" & will match just the enabled cases
#     for llava, i.e., single image, image embedding, and custom input tests.
#
# To run a test for a Test Info for just one of multiple models:
#     use the k flag to substring match the model name, e.g.,
#                 - pytest $TEST_FILE -k OpenGVLab/InternVL2-1B
#     prevents matching on nGVLab/InternVL2-2B.
#
# You can also combine substrings to match more granularly.
#     ex 1:
#        pytest $TEST_FILE -k "test_single_image and OpenGVLab/InternVL2-1B"
#     will run only test_single_image* for OpenGVLab/InternVL2-1B; this would
#     match both wrappers for single image tests, since it also matches
#     test_single_image_heavy (which forks if we have a distributed backend)
#     ex 2:
#        pytest $TEST_FILE -k  "[llava- or [intern_vl-"
#     will run all of the tests for only llava & internvl.
#
# NOTE you can add --collect-only to any of the above commands to see
# which cases would be selected and deselected by pytest. In general,
# this is a good idea for checking your command first, since tests are slow.

VLM_TEST_SETTINGS = {
90
91
92
    #### Core tests to always run in the CI
    "llava": VLMTestInfo(
        models=["llava-hf/llava-1.5-7b-hf"],
93
        test_type=(VLMTestType.EMBEDDING, VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
94
95
96
        prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
        convert_assets_to_embeddings=model_utils.get_llava_embeddings,
        max_model_len=4096,
97
        auto_cls=AutoModelForImageTextToText,
98
        vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
99
100
101
102
103
104
105
106
        custom_test_opts=[
            CustomTestOptions(
                inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
                    formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
                ),
                limit_mm_per_prompt={"image": 4},
            )
        ],
107
        vllm_runner_kwargs={"enable_mm_embeds": True},
108
        marks=[pytest.mark.core_model, pytest.mark.cpu_model],
109
    ),
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
    "paligemma": VLMTestInfo(
        models=["google/paligemma-3b-mix-224"],
        test_type=VLMTestType.IMAGE,
        prompt_formatter=identity,
        img_idx_to_prompt=lambda idx: "",
        # Paligemma uses its own sample prompts because the default one fails
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
                "stop_sign": "caption es",
                "cherry_blossom": "What is in the picture?",
            }
        ),
        auto_cls=AutoModelForImageTextToText,
        vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
    ),
Roger Wang's avatar
Roger Wang committed
125
126
    "qwen2_5_vl": VLMTestInfo(
        models=["Qwen/Qwen2.5-VL-3B-Instruct"],
127
128
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
129
130
        img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
        video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
131
        enforce_eager=False,
Roger Wang's avatar
Roger Wang committed
132
133
        max_model_len=4096,
        max_num_seqs=2,
134
        auto_cls=AutoModelForImageTextToText,
Roger Wang's avatar
Roger Wang committed
135
        vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
136
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
137
        marks=[pytest.mark.core_model, pytest.mark.cpu_model],
Roger Wang's avatar
Roger Wang committed
138
    ),
139
    "qwen2_5_omni": VLMTestInfo(
140
        models=["Qwen/Qwen2.5-Omni-3B"],
141
142
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
143
144
        img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>",
        video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>",
145
146
        max_model_len=4096,
        max_num_seqs=2,
147
        num_logprobs=6 if current_platform.is_cpu() else 5,
148
        auto_cls=AutoModelForTextToWaveform,
149
        vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
150
        patch_hf_runner=model_utils.qwen2_5_omni_patch_hf_runner,
151
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
152
153
        marks=[pytest.mark.core_model, pytest.mark.cpu_model],
    ),
154
155
156
157
158
159
160
    "qwen3_vl": VLMTestInfo(
        models=["Qwen/Qwen3-VL-4B-Instruct"],
        test_type=(
            VLMTestType.IMAGE,
            VLMTestType.MULTI_IMAGE,
            VLMTestType.VIDEO,
        ),
161
        enforce_eager=False,
162
163
164
165
166
167
168
169
170
171
        needs_video_metadata=True,
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
        img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",  # noqa: E501
        video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",  # noqa: E501
        max_model_len=4096,
        max_num_seqs=2,
        num_logprobs=20,
        auto_cls=AutoModelForImageTextToText,
        vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
        patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
172
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
173
174
175
        marks=[
            pytest.mark.core_model,
        ],
176
177
178
        vllm_runner_kwargs={"attention_backend": "TRITON_ATTN"}
        if current_platform.is_rocm()
        else {},
179
    ),
180
    "ultravox": VLMTestInfo(
181
        models=["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
182
        test_type=VLMTestType.AUDIO,
183
        prompt_formatter=lambda audio_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{audio_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",  # noqa: E501
184
185
186
187
188
189
190
        audio_idx_to_prompt=lambda idx: "<|audio|>",
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModel,
        hf_output_post_proc=model_utils.ultravox_trunc_hf_output,
        marks=[pytest.mark.core_model, pytest.mark.cpu_model],
    ),
191
192
193
194
195
196
    #### Transformers fallback to test
    ## To reduce test burden, we only test batching arbitrary image size
    # Dynamic image length and number of patches
    "llava-onevision-transformers": VLMTestInfo(
        models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
        test_type=VLMTestType.IMAGE,
197
        prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
198
        max_model_len=16384,
199
200
        hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
            "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
201
        ),
202
203
204
205
206
        auto_cls=AutoModelForImageTextToText,
        vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
        image_size_factors=[(0.25, 0.5, 1.0)],
        vllm_runner_kwargs={
            "model_impl": "transformers",
207
            "default_torch_num_threads": 1,
208
        },
209
        marks=[pytest.mark.core_model],
210
    ),
211
212
213
    # Gemma3 has bidirectional mask on images
    "gemma3-transformers": VLMTestInfo(
        models=["google/gemma-3-4b-it"],
214
215
216
        test_type=VLMTestType.IMAGE,
        prompt_formatter=lambda vid_prompt: f"<'<bos><start_of_turn>user\n{vid_prompt}<start_of_image><end_of_turn>\n<start_of_turn>model\n",  # noqa: E501
        max_model_len=4096,
217
218
219
220
221
222
        auto_cls=AutoModelForImageTextToText,
        vllm_output_post_proc=model_utils.gemma3_vllm_to_hf_output,
        image_size_factors=[(0.25, 0.5, 1.0)],
        vllm_runner_kwargs={
            "model_impl": "transformers",
        },
223
224
225
226
        marks=[
            pytest.mark.core_model,
            *([large_gpu_mark(min_gb=80)] if current_platform.is_rocm() else []),
        ],
227
    ),
228
229
230
    "idefics3-transformers": VLMTestInfo(
        models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
231
        prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:",  # noqa: E501
232
233
234
235
236
237
238
239
240
241
242
        img_idx_to_prompt=lambda idx: "<image>",
        max_model_len=8192,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
        hf_output_post_proc=model_utils.idefics3_trunc_hf_output,
        image_size_factors=[(0.25, 0.5, 1.0)],
        vllm_runner_kwargs={
            "model_impl": "transformers",
        },
        marks=[pytest.mark.core_model],
    ),
243
244
245
246
    # Pixel values from processor are not 4D or 5D arrays
    "qwen2_5_vl-transformers": VLMTestInfo(
        models=["Qwen/Qwen2.5-VL-3B-Instruct"],
        test_type=VLMTestType.IMAGE,
247
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
248
        img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
249
250
251
252
253
254
255
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
        vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
        image_size_factors=[(0.25, 0.2, 0.15)],
        vllm_runner_kwargs={
            "model_impl": "transformers",
256
257
258
259
260
261
262
263
264
265
266
            # TODO: [ROCm] Revert this once issue #30167 is resolved
            **(
                {
                    "mm_processor_kwargs": {
                        "min_pixels": 256 * 28 * 28,
                        "max_pixels": 1280 * 28 * 28,
                    },
                }
                if current_platform.is_rocm()
                else {}
            ),
267
        },
268
        marks=[large_gpu_mark(min_gb=80 if current_platform.is_rocm() else 32)],
269
    ),
270
    #### Extended model tests
271
272
273
    "aria": VLMTestInfo(
        models=["rhymes-ai/Aria"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
274
        prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ",  # noqa: E501
275
276
277
278
        img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
279
280
281
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
                "stop_sign": "<vlm_image>Please describe the image shortly.",
282
                "cherry_blossom": "<vlm_image>Please infer the season with reason.",
283
284
            }
        ),
285
        multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.",
286
287
288
289
290
        stop_str=["<|im_end|>"],
        image_size_factors=[(0.10, 0.15)],
        max_tokens=64,
        marks=[large_gpu_mark(min_gb=64)],
    ),
Jennifer Zhao's avatar
Jennifer Zhao committed
291
    "aya_vision": VLMTestInfo(
292
        models=["CohereLabs/aya-vision-8b"],
293
        test_type=(VLMTestType.IMAGE),
294
295
296
        prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",  # noqa: E501
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
297
298
                "stop_sign": "<image>What's the content in the center of the image?",
                "cherry_blossom": "<image>What is the season?",
299
300
            }
        ),
301
        multi_image_prompt="<image><image>Describe the two images in detail.",
302
303
304
305
306
307
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
        vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},
    ),
    "aya_vision-multi_image": VLMTestInfo(
308
        models=["CohereLabs/aya-vision-8b"],
309
        test_type=(VLMTestType.MULTI_IMAGE),
310
311
312
        prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",  # noqa: E501
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
313
314
                "stop_sign": "<image>What's the content in the center of the image?",
                "cherry_blossom": "<image>What is the season?",
315
316
            }
        ),
317
        multi_image_prompt="<image><image>Describe the two images in detail.",
318
        max_model_len=4096,
Jennifer Zhao's avatar
Jennifer Zhao committed
319
320
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
321
322
        vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},
        marks=[large_gpu_mark(min_gb=32)],
Jennifer Zhao's avatar
Jennifer Zhao committed
323
    ),
324
    "blip2": VLMTestInfo(
325
        models=["Salesforce/blip2-opt-2.7b"],
326
327
328
        test_type=VLMTestType.IMAGE,
        prompt_formatter=lambda img_prompt: f"Question: {img_prompt} Answer:",
        img_idx_to_prompt=lambda idx: "",
329
        auto_cls=AutoModelForImageTextToText,
330
        vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output,
331
332
        # FIXME: https://github.com/huggingface/transformers/pull/38510
        marks=[pytest.mark.skip("Model is broken")],
333
334
335
336
337
338
    ),
    "chameleon": VLMTestInfo(
        models=["facebook/chameleon-7b"],
        test_type=VLMTestType.IMAGE,
        prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
        max_model_len=4096,
339
        max_num_seqs=2,
340
        auto_cls=AutoModelForImageTextToText,
341
        # For chameleon, we only compare the sequences
342
343
        vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
        hf_output_post_proc=lambda hf_output, model: hf_output[:2],
344
345
346
347
        comparator=check_outputs_equal,
        max_tokens=8,
        dtype="bfloat16",
    ),
348
    "deepseek_vl_v2": VLMTestInfo(
349
        models=["Isotr0py/deepseek-vl2-tiny"],  # model repo using dynamic module
350
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
351
        prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ",  # noqa: E501
352
353
        max_model_len=4096,
        max_num_seqs=2,
354
355
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
356
                "stop_sign": "<image>\nWhat's the content in the center of the image?",
357
358
359
360
                "cherry_blossom": "<image>\nPlease infer the season with reason in details.",  # noqa: E501
            }
        ),
        multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?",  # noqa: E501
361
362
        patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
        hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
363
        stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"],
364
        image_size_factors=[(1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
365
    ),
366
367
368
369
370
371
372
    "fuyu": VLMTestInfo(
        models=["adept/fuyu-8b"],
        test_type=VLMTestType.IMAGE,
        prompt_formatter=lambda img_prompt: f"{img_prompt}\n",
        img_idx_to_prompt=lambda idx: "",
        max_model_len=2048,
        max_num_seqs=2,
373
        auto_cls=AutoModelForImageTextToText,
374
375
376
        use_tokenizer_eos=True,
        vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
        num_logprobs=10,
377
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
378
        marks=[large_gpu_mark(min_gb=32)],
379
    ),
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
    "gemma3": VLMTestInfo(
        models=["google/gemma-3-4b-it"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n",  # noqa: E501
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
                "stop_sign": "<start_of_image>What's the content in the center of the image?",  # noqa: E501
                "cherry_blossom": "<start_of_image>What is the season?",
            }
        ),
        multi_image_prompt="<start_of_image><start_of_image>Describe the two images in detail.",  # noqa: E501
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
        vllm_runner_kwargs={"mm_processor_kwargs": {"do_pan_and_scan": True}},
        patch_hf_runner=model_utils.gemma3_patch_hf_runner,
    ),
397
398
399
400
401
402
403
404
    "granite_vision": VLMTestInfo(
        models=["ibm-granite/granite-vision-3.3-2b"],
        test_type=(VLMTestType.IMAGE),
        prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}\n<|assistant|>\n",
        max_model_len=8192,
        auto_cls=AutoModelForImageTextToText,
        vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
    ),
405
    "glm4v": VLMTestInfo(
406
        models=["zai-org/glm-4v-9b"],
407
        test_type=VLMTestType.IMAGE,
408
        prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>",
409
410
411
412
413
414
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
                "stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?",  # noqa: E501
                "cherry_blossom": "<|begin_of_image|><|endoftext|><|end_of_image|>What is the season?",  # noqa: E501
            }
        ),
415
416
417
        max_model_len=2048,
        max_num_seqs=2,
        get_stop_token_ids=lambda tok: [151329, 151336, 151338],
418
419
420
421
422
423
        patch_hf_runner=model_utils.glm4v_patch_hf_runner,
        # The image embeddings match with HF but the outputs of the language
        # decoder are only consistent up to 2 decimal places.
        # So, we need to reduce the number of tokens for the test to pass.
        max_tokens=8,
        num_logprobs=10,
424
        auto_cls=AutoModelForCausalLM,
425
        marks=[large_gpu_mark(min_gb=32)],
426
    ),
427
    "glm4_1v": VLMTestInfo(
428
        models=["zai-org/GLM-4.1V-9B-Thinking"],
429
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
430
        prompt_formatter=lambda img_prompt: f"[gMASK]<|user|>\n{img_prompt}<|assistant|>\n",  # noqa: E501
431
432
        img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>",
        video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>",
433
434
435
436
        max_model_len=2048,
        max_num_seqs=2,
        get_stop_token_ids=lambda tok: [151329, 151336, 151338],
        num_logprobs=10,
437
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
438
        auto_cls=AutoModelForImageTextToText,
439
        marks=[large_gpu_mark(min_gb=32)],
440
441
    ),
    "glm4_1v-video": VLMTestInfo(
442
        models=["zai-org/GLM-4.1V-9B-Thinking"],
443
444
        # GLM4.1V require include video metadata for input
        test_type=VLMTestType.CUSTOM_INPUTS,
445
        prompt_formatter=lambda vid_prompt: f"[gMASK]<|user|>\n{vid_prompt}<|assistant|>\n",  # noqa: E501
446
447
448
449
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
        patch_hf_runner=model_utils.glm4_1v_patch_hf_runner,
450
451
452
453
454
455
        custom_test_opts=[
            CustomTestOptions(
                inputs=custom_inputs.video_with_metadata_glm4_1v(),
                limit_mm_per_prompt={"video": 1},
            )
        ],
456
        marks=[large_gpu_mark(min_gb=32)],
457
    ),
458
459
460
461
462
463
464
465
466
467
    "glm_ocr": VLMTestInfo(
        models=["zai-org/GLM-OCR"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: f"[gMASK]<|user|>\n{img_prompt}<|assistant|>\n",  # noqa: E501
        img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>",
        video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>",
        max_model_len=2048,
        max_num_seqs=2,
        get_stop_token_ids=lambda tok: [151329, 151336, 151338],
        num_logprobs=10,
468
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
469
470
471
        auto_cls=AutoModelForImageTextToText,
        marks=[large_gpu_mark(min_gb=32)],
    ),
472
    "h2ovl": VLMTestInfo(
473
        models=[
474
            "h2oai/h2ovl-mississippi-800m",
475
            "h2oai/h2ovl-mississippi-2b",
476
477
        ],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
478
        prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>",
479
480
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
481
                "stop_sign": "<image>\nWhat's the content in the center of the image?",
482
483
484
                "cherry_blossom": "<image>\nWhat is the season?",
            }
        ),
485
486
487
        multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.",  # noqa: E501
        max_model_len=8192,
        use_tokenizer_eos=True,
488
        num_logprobs=10,
489
        patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
490
    ),
491
    "idefics3": VLMTestInfo(
492
        models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
493
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
494
        prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}<end_of_utterance>\nAssistant:",  # noqa: E501
495
496
497
        img_idx_to_prompt=lambda idx: "<image>",
        max_model_len=8192,
        max_num_seqs=2,
498
        auto_cls=AutoModelForImageTextToText,
499
        hf_output_post_proc=model_utils.idefics3_trunc_hf_output,
500
    ),
501
502
503
504
    "intern_vl": VLMTestInfo(
        models=[
            "OpenGVLab/InternVL2-1B",
            "OpenGVLab/InternVL2-2B",
505
506
            # FIXME: Config cannot be loaded in transformers 4.52
            # "OpenGVLab/Mono-InternVL-2B",
507
508
        ],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
509
510
511
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n",  # noqa: E501
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
512
                "stop_sign": "<image>\nWhat's the content in the center of the image?",
513
514
515
                "cherry_blossom": "<image>\nWhat is the season?",
            }
        ),
516
517
518
519
520
        multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.",  # noqa: E501
        max_model_len=4096,
        use_tokenizer_eos=True,
        patch_hf_runner=model_utils.internvl_patch_hf_runner,
    ),
521
522
523
524
525
    "intern_vl-video": VLMTestInfo(
        models=[
            "OpenGVLab/InternVL3-1B",
        ],
        test_type=VLMTestType.VIDEO,
526
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n",  # noqa: E501
527
528
529
530
        video_idx_to_prompt=lambda idx: "<video>",
        max_model_len=8192,
        use_tokenizer_eos=True,
        patch_hf_runner=model_utils.internvl_patch_hf_runner,
531
        num_logprobs=10 if current_platform.is_rocm() else 5,
532
    ),
533
534
535
536
537
538
539
    "intern_vl-hf": VLMTestInfo(
        models=["OpenGVLab/InternVL3-1B-hf"],
        test_type=(
            VLMTestType.IMAGE,
            VLMTestType.MULTI_IMAGE,
            VLMTestType.VIDEO,
        ),
540
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n",  # noqa: E501
541
542
543
544
545
546
        img_idx_to_prompt=lambda idx: "<IMG_CONTEXT>",
        video_idx_to_prompt=lambda idx: "<video>",
        max_model_len=8192,
        use_tokenizer_eos=True,
        auto_cls=AutoModelForImageTextToText,
    ),
oscardev256's avatar
oscardev256 committed
547
    "isaac": VLMTestInfo(
548
549
550
551
        models=[
            "PerceptronAI/Isaac-0.1",
            "PerceptronAI/Isaac-0.2-2B-Preview",
        ],
oscardev256's avatar
oscardev256 committed
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: (
            f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n"
        ),
        img_idx_to_prompt=lambda idx: "<image>",
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
                "stop_sign": "<vlm_image>Please describe the image shortly.",
                "cherry_blossom": "<vlm_image>Please infer the season with reason.",
            }
        ),
        multi_image_prompt=(
            "Picture 1: <vlm_image>\n"
            "Picture 2: <vlm_image>\n"
            "Describe these two images with one paragraph respectively."
        ),
        enforce_eager=False,
        max_model_len=4096,
        max_num_seqs=2,
        hf_model_kwargs={"device_map": "auto"},
        patch_hf_runner=model_utils.isaac_patch_hf_runner,
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
    ),
575
576
577
    "kimi_vl": VLMTestInfo(
        models=["moonshotai/Kimi-VL-A3B-Instruct"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
578
        prompt_formatter=lambda img_prompt: f"<|im_user|>user<|im_middle|>{img_prompt}<|im_end|><|im_assistant|>assistant<|im_middle|>",  # noqa: E501
579
580
581
582
583
584
585
586
        img_idx_to_prompt=lambda _: "<|media_start|>image<|media_content|><|media_pad|><|media_end|>",  # noqa: E501
        max_model_len=8192,
        max_num_seqs=2,
        dtype="bfloat16",
        tensor_parallel_size=1,
        vllm_output_post_proc=model_utils.kimiv_vl_vllm_to_hf_output,
        marks=[large_gpu_mark(min_gb=48)],
    ),
587
588
    "llama4": VLMTestInfo(
        models=["meta-llama/Llama-4-Scout-17B-16E-Instruct"],
589
        prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|header_start|>user<|header_end|>\n\n{img_prompt}<|eot|><|header_start|>assistant<|header_end|>\n\n",  # noqa: E501
590
591
592
        img_idx_to_prompt=lambda _: "<|image|>",
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        distributed_executor_backend="mp",
593
        image_size_factors=[(0.25, 0.5, 1.0)],
594
595
596
597
598
        hf_model_kwargs={"device_map": "auto"},
        max_model_len=8192,
        max_num_seqs=4,
        dtype="bfloat16",
        auto_cls=AutoModelForImageTextToText,
599
600
        tensor_parallel_size=4,
        marks=multi_gpu_marks(num_gpus=4),
601
    ),
602
603
604
605
606
    "llava_next": VLMTestInfo(
        models=["llava-hf/llava-v1.6-mistral-7b-hf"],
        test_type=(VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
        prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
        max_model_len=10240,
607
        auto_cls=AutoModelForImageTextToText,
608
        vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
609
610
611
612
613
614
615
616
        custom_test_opts=[
            CustomTestOptions(
                inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
                    formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]"
                ),
                limit_mm_per_prompt={"image": 4},
            )
        ],
617
    ),
618
    "llava_onevision": VLMTestInfo(
619
620
        models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
        test_type=VLMTestType.CUSTOM_INPUTS,
621
        prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
622
623
        num_video_frames=16,
        max_model_len=16384,
624
625
        hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
            "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
626
        ),
627
        auto_cls=AutoModelForImageTextToText,
628
        vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
629
630
631
632
633
634
635
636
        custom_test_opts=[
            CustomTestOptions(
                inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
                    formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
                ),
                limit_mm_per_prompt={"video": 4},
            )
        ],
637
638
639
640
641
642
643
    ),
    "llava_next_video": VLMTestInfo(
        models=["llava-hf/LLaVA-NeXT-Video-7B-hf"],
        test_type=VLMTestType.VIDEO,
        prompt_formatter=lambda vid_prompt: f"USER: {vid_prompt} ASSISTANT:",
        num_video_frames=16,
        max_model_len=4096,
644
        max_num_seqs=2,
645
        auto_cls=AutoModelForImageTextToText,
646
647
        vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
    ),
648
649
650
651
652
653
    "mantis": VLMTestInfo(
        models=["TIGER-Lab/Mantis-8B-siglip-llama3"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: f"<|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",  # noqa: E501
        max_model_len=4096,
        get_stop_token_ids=lambda tok: [128009],
654
        auto_cls=AutoModelForImageTextToText,
655
656
657
        vllm_output_post_proc=model_utils.mantis_vllm_to_hf_output,
        patch_hf_runner=model_utils.mantis_patch_hf_runner,
    ),
658
    "minicpmv_25": VLMTestInfo(
659
        models=["openbmb/MiniCPM-Llama3-V-2_5"],
660
        test_type=VLMTestType.IMAGE,
661
662
663
664
665
        prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",  # noqa: E501
        img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
        max_model_len=4096,
        max_num_seqs=2,
        get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id],
666
        hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
667
        patch_hf_runner=model_utils.minicpmv_25_patch_hf_runner,
668
669
        # FIXME: https://huggingface.co/openbmb/MiniCPM-V-2_6/discussions/55
        marks=[pytest.mark.skip("HF import fails")],
670
    ),
671
672
    "minicpmo_26": VLMTestInfo(
        models=["openbmb/MiniCPM-o-2_6"],
673
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
674
675
676
677
        prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",  # noqa: E501
        img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
        max_model_len=4096,
        max_num_seqs=2,
678
679
        get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
            ["<|im_end|>", "<|endoftext|>"]
680
        ),
681
682
        hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
        patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
683
        # FIXME: https://huggingface.co/openbmb/MiniCPM-o-2_6/discussions/49
684
        marks=[pytest.mark.skip("HF import fails")],
685
    ),
686
687
    "minicpmv_26": VLMTestInfo(
        models=["openbmb/MiniCPM-V-2_6"],
688
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
689
690
691
692
        prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",  # noqa: E501
        img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
        max_model_len=4096,
        max_num_seqs=2,
693
694
        get_stop_token_ids=lambda tok: tok.convert_tokens_to_ids(
            ["<|im_end|>", "<|endoftext|>"]
695
        ),
696
697
698
        hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
        patch_hf_runner=model_utils.minicpmv_26_patch_hf_runner,
    ),
699
700
    "minimax_vl_01": VLMTestInfo(
        models=["MiniMaxAI/MiniMax-VL-01"],
701
        prompt_formatter=lambda img_prompt: f"<beginning_of_sentence>user: {img_prompt} assistant:<end_of_sentence>",  # noqa: E501
702
703
704
705
706
707
708
709
        img_idx_to_prompt=lambda _: "<image>",
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        max_model_len=8192,
        max_num_seqs=4,
        dtype="bfloat16",
        hf_output_post_proc=model_utils.minimax_vl_01_hf_output,
        patch_hf_runner=model_utils.minimax_vl_01_patch_hf_runner,
        auto_cls=AutoModelForImageTextToText,
710
711
712
713
714
715
716
717
718
719
720
        marks=[
            large_gpu_mark(min_gb=80),
            # TODO: [ROCm] Fix pickle issue with ROCm spawn and tp>1
            pytest.mark.skipif(
                current_platform.is_rocm(),
                reason=(
                    "ROCm: Model too large for single GPU; "
                    "multi-GPU blocked by HF _LazyConfigMapping pickle issue with spawn"
                ),
            ),
        ],
721
    ),
722
723
    "molmo": VLMTestInfo(
        models=["allenai/Molmo-7B-D-0924"],
724
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
725
        prompt_formatter=identity,
726
727
        max_model_len=4096,
        max_num_seqs=2,
728
        patch_hf_runner=model_utils.molmo_patch_hf_runner,
729
    ),
730
731
732
    "ovis1_6-gemma2": VLMTestInfo(
        models=["AIDC-AI/Ovis1.6-Gemma2-9B"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
733
        prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n",  # noqa: E501
734
        img_idx_to_prompt=lambda idx: "<image>\n",
735
736
737
738
739
740
741
742
        max_model_len=4096,
        max_num_seqs=2,
        dtype="half",
        # use sdpa mode for hf runner since ovis2 didn't work with flash_attn
        hf_model_kwargs={"llm_attn_implementation": "sdpa"},
        patch_hf_runner=model_utils.ovis_patch_hf_runner,
        marks=[large_gpu_mark(min_gb=32)],
    ),
743
744
745
    "ovis2": VLMTestInfo(
        models=["AIDC-AI/Ovis2-1B"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
746
        prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
747
        img_idx_to_prompt=lambda idx: "<image>\n",
748
749
750
751
752
        max_model_len=4096,
        max_num_seqs=2,
        dtype="half",
        # use sdpa mode for hf runner since ovis2 didn't work with flash_attn
        hf_model_kwargs={"llm_attn_implementation": "sdpa"},
753
        patch_hf_runner=model_utils.ovis_patch_hf_runner,
754
    ),
755
756
    "ovis2_5": VLMTestInfo(
        models=["AIDC-AI/Ovis2.5-2B"],
757
758
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
        prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
759
        img_idx_to_prompt=lambda idx: "<image>\n",
760
761
762
763
764
765
        video_idx_to_prompt=lambda idx: "<video>\n",
        max_model_len=4096,
        max_num_seqs=2,
        dtype="half",
        num_logprobs=10,
        patch_hf_runner=model_utils.ovis2_5_patch_hf_runner,
766
        hf_model_kwargs={"revision": "refs/pr/5"},
767
    ),
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
    "paddleocr_vl": VLMTestInfo(
        models=["PaddlePaddle/PaddleOCR-VL"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
        img_idx_to_prompt=lambda idx: (
            "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>"
        ),
        multi_image_prompt=(
            "Image-1: <|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>\n"
            "Image-2: <|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>\n"
            "Describe these two images separately."
        ),
        max_model_len=8192,
        max_num_seqs=2,
        auto_cls=AutoModelForCausalLM,
783
        patch_hf_runner=model_utils.paddleocr_vl_patch_hf_runner,
784
        image_size_factors=[(0.25,)],
785
786
787
788
789
790
        marks=[
            pytest.mark.skipif(
                Version(TRANSFORMERS_VERSION) == Version("4.57.3"),
                reason="This model is broken in Transformers v4.57.3",
            )
        ],
791
    ),
792
793
794
    "phi3v": VLMTestInfo(
        models=["microsoft/Phi-3.5-vision-instruct"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
795
        prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n",  # noqa: E501
796
797
798
        img_idx_to_prompt=lambda idx: f"<|image_{idx}|>\n",
        max_model_len=4096,
        max_num_seqs=2,
799
        runner="generate",
800
801
        # use sdpa mode for hf runner since phi3v didn't work with flash_attn
        hf_model_kwargs={"_attn_implementation": "sdpa"},
802
803
804
805
        use_tokenizer_eos=True,
        vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
        num_logprobs=10,
    ),
806
807
808
809
810
811
812
    "pixtral_hf": VLMTestInfo(
        models=["nm-testing/pixtral-12b-FP8-dynamic"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: f"<s>[INST]{img_prompt}[/INST]",
        img_idx_to_prompt=lambda idx: "[IMG]",
        max_model_len=8192,
        max_num_seqs=2,
813
        auto_cls=AutoModelForImageTextToText,
814
815
816
817
818
819
820
        marks=[
            large_gpu_mark(min_gb=48),
            pytest.mark.skipif(
                current_platform.is_rocm(),
                reason="Model produces a vector of <UNK> output in HF on ROCm",
            ),
        ],
821
    ),
822
    "qwen_vl": VLMTestInfo(
823
824
825
826
827
828
829
830
831
        models=["Qwen/Qwen-VL"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=identity,
        img_idx_to_prompt=lambda idx: f"Picture {idx}: <img></img>\n",
        max_model_len=1024,
        max_num_seqs=2,
        vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
        prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
    ),
832
833
    "qwen2_vl": VLMTestInfo(
        models=["Qwen/Qwen2-VL-2B-Instruct"],
834
835
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
836
837
        img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
        video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
838
        multi_image_prompt="Picture 1: <vlm_image>\nPicture 2: <vlm_image>\nDescribe these two images with one paragraph respectively.",  # noqa: E501
839
840
        max_model_len=4096,
        max_num_seqs=2,
841
        auto_cls=AutoModelForImageTextToText,
842
        vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
843
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
844
845
        marks=[pytest.mark.cpu_model],
    ),
846
847
848
    "skywork_r1v": VLMTestInfo(
        models=["Skywork/Skywork-R1V-38B"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
849
850
851
        prompt_formatter=lambda img_prompt: f"<|begin▁of▁sentence|><|User|>\n{img_prompt}<|Assistant|><think>\n",  # noqa: E501
        single_image_prompts=IMAGE_ASSETS.prompts(
            {
852
                "stop_sign": "<image>\nWhat's the content in the center of the image?",
853
854
855
                "cherry_blossom": "<image>\nWhat is the season?",
            }
        ),
856
        multi_image_prompt="<image>\n<image>\nDescribe the two images in short.",
857
858
859
860
861
        max_model_len=4096,
        use_tokenizer_eos=True,
        patch_hf_runner=model_utils.skyworkr1v_patch_hf_runner,
        marks=[large_gpu_mark(min_gb=80)],
    ),
862
863
864
    "smolvlm": VLMTestInfo(
        models=["HuggingFaceTB/SmolVLM2-2.2B-Instruct"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
865
        prompt_formatter=lambda img_prompt: f"<|im_start|>User:{img_prompt}<end_of_utterance>\nAssistant:",  # noqa: E501
866
867
868
869
870
        img_idx_to_prompt=lambda idx: "<image>",
        max_model_len=8192,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
        hf_output_post_proc=model_utils.smolvlm_trunc_hf_output,
871
        num_logprobs=10,
872
    ),
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
    "tarsier": VLMTestInfo(
        models=["omni-research/Tarsier-7b"],
        test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
        prompt_formatter=lambda img_prompt: f"USER: {img_prompt} ASSISTANT:",
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
        patch_hf_runner=model_utils.tarsier_patch_hf_runner,
    ),
    "tarsier2": VLMTestInfo(
        models=["omni-research/Tarsier2-Recap-7b"],
        test_type=(
            VLMTestType.IMAGE,
            VLMTestType.MULTI_IMAGE,
            VLMTestType.VIDEO,
        ),
889
        prompt_formatter=lambda img_prompt: f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
890
891
        img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
        video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
892
893
894
        max_model_len=4096,
        max_num_seqs=2,
        auto_cls=AutoModelForImageTextToText,
895
        image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
896
897
        marks=[pytest.mark.skip("Model initialization hangs")],
    ),
898
    ### Tensor parallel / multi-gpu broadcast tests
899
    "chameleon-broadcast": VLMTestInfo(
900
901
902
        models=["facebook/chameleon-7b"],
        prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
        max_model_len=4096,
903
        auto_cls=AutoModelForImageTextToText,
904
905
        vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
        hf_output_post_proc=lambda hf_output, model: hf_output[:2],
906
        comparator=check_outputs_equal,
907
        marks=multi_gpu_marks(num_gpus=2),
908
        **COMMON_BROADCAST_SETTINGS,  # type: ignore
909
    ),
910
    "llava-broadcast": VLMTestInfo(
911
912
913
        models=["llava-hf/llava-1.5-7b-hf"],
        prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
        max_model_len=4096,
914
        auto_cls=AutoModelForImageTextToText,
915
        vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
916
        marks=multi_gpu_marks(num_gpus=2),
917
        **COMMON_BROADCAST_SETTINGS,  # type: ignore
918
    ),
919
    "llava_next-broadcast": VLMTestInfo(
920
921
922
        models=["llava-hf/llava-v1.6-mistral-7b-hf"],
        prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
        max_model_len=10240,
923
        auto_cls=AutoModelForImageTextToText,
924
        vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
925
        marks=multi_gpu_marks(num_gpus=2),
926
        **COMMON_BROADCAST_SETTINGS,  # type: ignore
927
928
929
930
    ),
    ### Custom input edge-cases for specific models
    "intern_vl-diff-patches": VLMTestInfo(
        models=["OpenGVLab/InternVL2-2B"],
931
        prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n",  # noqa: E501
932
933
934
935
936
937
938
939
        test_type=VLMTestType.CUSTOM_INPUTS,
        max_model_len=4096,
        use_tokenizer_eos=True,
        patch_hf_runner=model_utils.internvl_patch_hf_runner,
        custom_test_opts=[
            CustomTestOptions(
                inputs=inp,
                limit_mm_per_prompt={"image": 2},
940
941
            )
            for inp in custom_inputs.different_patch_input_cases_internvl()
942
943
        ],
    ),
944
    "llava_onevision-multiple-images": VLMTestInfo(
945
946
947
948
        models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
        test_type=VLMTestType.CUSTOM_INPUTS,
        max_model_len=16384,
        max_num_seqs=2,
949
        auto_cls=AutoModelForImageTextToText,
950
951
        hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
            "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
952
        ),
953
        vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
954
955
956
957
958
959
960
961
        custom_test_opts=[
            CustomTestOptions(
                inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
                    formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n",  # noqa: E501
                ),
                limit_mm_per_prompt={"image": 4},
            )
        ],
962
    ),
963
964
965
966
967
968
    # regression test for https://github.com/vllm-project/vllm/issues/15122
    "qwen2_5_vl-windows-attention": VLMTestInfo(
        models=["Qwen/Qwen2.5-VL-3B-Instruct"],
        test_type=VLMTestType.CUSTOM_INPUTS,
        max_model_len=4096,
        max_num_seqs=2,
969
        auto_cls=AutoModelForImageTextToText,
970
        vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
971
972
973
974
975
976
        custom_test_opts=[
            CustomTestOptions(
                inputs=custom_inputs.windows_attention_image_qwen2_5_vl(),
                limit_mm_per_prompt={"image": 1},
            )
        ],
977
    ),
978
979
980
}


981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
def _mark_splits(
    test_settings: dict[str, VLMTestInfo],
    *,
    num_groups: int,
) -> dict[str, VLMTestInfo]:
    name_by_test_info_id = {id(v): k for k, v in test_settings.items()}
    test_infos_by_model = defaultdict[str, list[VLMTestInfo]](list)

    for info in test_settings.values():
        for model in info.models:
            test_infos_by_model[model].append(info)

    models = sorted(test_infos_by_model.keys())
    split_size = math.ceil(len(models) / num_groups)

    new_test_settings = dict[str, VLMTestInfo]()

    for i in range(num_groups):
999
        models_in_group = models[i * split_size : (i + 1) * split_size]
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015

        for model in models_in_group:
            for info in test_infos_by_model[model]:
                new_marks = (info.marks or []) + [pytest.mark.split(group=i)]
                new_info = info._replace(marks=new_marks)
                new_test_settings[name_by_test_info_id[id(info)]] = new_info

    missing_keys = test_settings.keys() - new_test_settings.keys()
    assert not missing_keys, f"Missing keys: {missing_keys}"

    return new_test_settings


VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)


1016
1017
1018
1019
1020
1021
### Test wrappers
# Wrappers around the core test running func for:
# - single image
# - multi-image
# - image embeddings
# - video
1022
# - audio
1023
# - custom inputs
1024
1025
1026
1027
1028
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.IMAGE,
1029
        create_new_process_for_each_test=False,
1030
1031
    ),
)
1032
1033
1034
1035
1036
1037
1038
1039
def test_single_image_models(
    tmp_path: PosixPath,
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    image_assets: ImageTestAssets,
):
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_single_image_test(
        tmp_path=tmp_path,
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        image_assets=image_assets,
    )


1051
1052
1053
1054
1055
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.MULTI_IMAGE,
1056
        create_new_process_for_each_test=False,
1057
1058
    ),
)
1059
1060
1061
1062
1063
1064
1065
1066
def test_multi_image_models(
    tmp_path: PosixPath,
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    image_assets: ImageTestAssets,
):
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_multi_image_test(
        tmp_path=tmp_path,
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        image_assets=image_assets,
    )


1078
1079
1080
1081
1082
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.EMBEDDING,
1083
        create_new_process_for_each_test=False,
1084
1085
    ),
)
1086
1087
1088
1089
1090
1091
1092
def test_image_embedding_models(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    image_assets: ImageTestAssets,
):
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_embedding_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        image_assets=image_assets,
    )


1103
1104
1105
1106
1107
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.VIDEO,
1108
        create_new_process_for_each_test=False,
1109
1110
    ),
)
1111
1112
1113
1114
1115
1116
1117
def test_video_models(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    video_assets: VideoTestAssets,
):
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_video_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        video_assets=video_assets,
    )


1128
1129
1130
1131
1132
1133
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.AUDIO,
        create_new_process_for_each_test=False,
1134
1135
    ),
)
1136
1137
1138
1139
1140
1141
1142
def test_audio_models(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    audio_assets: AudioTestAssets,
):
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_audio_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        audio_assets=audio_assets,
    )


1153
1154
1155
1156
1157
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.CUSTOM_INPUTS,
1158
        create_new_process_for_each_test=False,
1159
1160
    ),
)
1161
1162
1163
def test_custom_inputs_models(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
1164
1165
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
):
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_custom_inputs_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
    )


#### Tests filtering for things running each test as a new process
1177
1178
1179
1180
1181
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.IMAGE,
1182
        create_new_process_for_each_test=True,
1183
1184
    ),
)
1185
@create_new_process_for_each_test()
1186
1187
1188
1189
1190
1191
1192
1193
def test_single_image_models_heavy(
    tmp_path: PosixPath,
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    image_assets: ImageTestAssets,
):
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_single_image_test(
        tmp_path=tmp_path,
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        image_assets=image_assets,
    )


1205
1206
1207
1208
1209
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.MULTI_IMAGE,
1210
        create_new_process_for_each_test=True,
1211
1212
    ),
)
1213
@create_new_process_for_each_test()
1214
1215
1216
1217
1218
1219
1220
1221
def test_multi_image_models_heavy(
    tmp_path: PosixPath,
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    image_assets: ImageTestAssets,
):
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_multi_image_test(
        tmp_path=tmp_path,
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        image_assets=image_assets,
    )


1233
1234
1235
1236
1237
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.EMBEDDING,
1238
        create_new_process_for_each_test=True,
1239
1240
    ),
)
1241
@create_new_process_for_each_test()
1242
1243
1244
1245
1246
1247
1248
def test_image_embedding_models_heavy(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    image_assets: ImageTestAssets,
):
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_embedding_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        image_assets=image_assets,
    )


1259
1260
1261
1262
1263
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.VIDEO,
1264
        create_new_process_for_each_test=True,
1265
1266
    ),
)
1267
1268
1269
1270
1271
1272
1273
def test_video_models_heavy(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    video_assets: VideoTestAssets,
):
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_video_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        video_assets=video_assets,
    )


1284
1285
1286
1287
1288
1289
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.AUDIO,
        create_new_process_for_each_test=True,
1290
1291
    ),
)
1292
1293
1294
1295
1296
1297
1298
def test_audio_models_heavy(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
    audio_assets: AudioTestAssets,
):
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_audio_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
        audio_assets=audio_assets,
    )


1309
1310
1311
1312
1313
@pytest.mark.parametrize(
    "model_type,test_case",
    get_parametrized_options(
        VLM_TEST_SETTINGS,
        test_type=VLMTestType.CUSTOM_INPUTS,
1314
        create_new_process_for_each_test=True,
1315
1316
    ),
)
1317
@create_new_process_for_each_test()
1318
1319
1320
def test_custom_inputs_models_heavy(
    model_type: str,
    test_case: ExpandableVLMTestArgs,
1321
1322
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
1323
1324
1325
1326
1327
1328
1329
1330
):
    model_test_info = VLM_TEST_SETTINGS[model_type]
    runners.run_custom_inputs_test(
        model_test_info=model_test_info,
        test_case=test_case,
        hf_runner=hf_runner,
        vllm_runner=vllm_runner,
    )