test_idefics3.py 2.84 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
3
"""Tests for Idefics3's multimodal preprocessing kwargs."""
import pytest
4
from transformers import Idefics3Config
5

6
from vllm.multimodal import MULTIMODAL_REGISTRY
7
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
8

9
10
from ....conftest import _ImageAssets
from ...utils import build_model_context
11
12


13
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
14
15
16
17
18
19
20
21
22
# yapf: disable
@pytest.mark.parametrize(
    ("mm_processor_kwargs", "expected_toks_per_img"),
    [
        ({"size": {"longest_edge": 364}}, 169),
        ({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
    ])
# yapf: enable
@pytest.mark.parametrize("num_imgs", [1, 2])
23
24
25
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
    image_assets: _ImageAssets,
26
    model_id: str,
27
28
29
30
31
    mm_processor_kwargs: dict[str, object],
    expected_toks_per_img: int,
    num_imgs: int,
    kwargs_on_init: bool,
):
32
33
34
35
36
    """Ensure input_processor_for_idefics3 handles num_crops properly."""
    # Same as the previous test - don't initialize mm_processor_kwargs
    # in this test and assume that the kwargs will be correctly expanded by
    # the partial when calling the custom input processor.
    ctx = build_model_context(
37
        model_id,
38
        mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
39
        limit_mm_per_prompt={"image": num_imgs},
40
    )
41
    tokenizer = cached_tokenizer_from_config(ctx.model_config)
42
43
44
45
    processor = MULTIMODAL_REGISTRY.create_processor(
        ctx.model_config,
        tokenizer=tokenizer,
    )
46
    hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
47
48
49
50
51
52

    # Build the image str / prompt based on the number of images we pass
    placeholders = "<image>" if num_imgs == 1 else "\n".join(
        f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
    prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:"  # noqa: E501

53
54
55
56
57
58
    # Build mm_data
    image_size = ctx.get_hf_config(Idefics3Config).vision_config.image_size
    dummy_image_size = (image_size * 4, image_size * 4)
    dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
    mm_data = {"image": [dummy_image] * num_imgs}

59
60
    processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)

61
    # Ensure the placeholders format are correct
62
    hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
63
64
65
    hf_processed_inputs = hf_processor(text=prompt, images=mm_data["image"])
    assert processed_inputs["prompt_token_ids"] == hf_processed_inputs[
        "input_ids"][0]
66
67
68
69
70

    # Ensure we have the right number of placeholders per num_crops size
    image_token_id = ctx.get_hf_config().image_token_id
    img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
    assert img_tok_count == expected_toks_per_img * num_imgs