test_phi4mm.py 9.27 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4

import os
5
from collections.abc import Sequence
6
7

import pytest
8
import regex as re
9
10
11
from huggingface_hub import snapshot_download
from transformers import AutoTokenizer

12
from vllm.assets.image import ImageAsset
13
from vllm.logprobs import SampleLogprobs
14
from vllm.lora.request import LoRARequest
15
from vllm.multimodal.image import convert_image_mode, rescale_image_size
16
from vllm.multimodal.media.audio import load_audio
17

18
19
20
21
22
23
24
from ....conftest import (
    IMAGE_ASSETS,
    HfRunner,
    PromptAudioInput,
    PromptImageInput,
    VllmRunner,
)
25
26
27
from ....utils import large_gpu_test
from ...utils import check_logprobs_close

28
29
30
31
32
33
34
35
36
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
    {
        "stop_sign": "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n",  # noqa: E501
        "cherry_blossom": "<|user|>\n<|image_1|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n",  # noqa: E501
    }
)
HF_MULTIIMAGE_IMAGE_PROMPT = (
    "<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n"  # noqa: E501
)
37
38
39
40
41

model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
# Since the vision-lora and speech-lora co-exist with the base model,
# we have to manually specify the path of the lora weights.
vision_lora_path = os.path.join(model_path, "vision-lora")
42
43
44
speech_question = os.path.join(
    model_path, "examples", "what_is_shown_in_this_image.wav"
)
45
46
47
models = [model_path]


48
def vllm_to_hf_output(
49
    vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str
50
):
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
    """Sanitize vllm output to be comparable with hf output."""
    _, output_str, out_logprobs = vllm_output

    output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
    assert output_str_without_image[0] == " "
    output_str_without_image = output_str_without_image[1:]

    hf_output_str = output_str_without_image + "<|end|><|endoftext|>"

    tokenizer = AutoTokenizer.from_pretrained(model)
    hf_output_ids = tokenizer.encode(output_str_without_image)
    assert hf_output_ids[0] == 1
    hf_output_ids = hf_output_ids[1:]

    return hf_output_ids, hf_output_str, out_logprobs


target_dtype = "half"


def run_test(
    hf_runner: type[HfRunner],
    vllm_runner: type[VllmRunner],
74
    inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]],
75
76
77
78
79
80
81
82
    model: str,
    *,
    max_model_len: int,
    dtype: str,
    max_tokens: int,
    num_logprobs: int,
    mm_limit: int,
    tensor_parallel_size: int,
83
    distributed_executor_backend: str | None = None,
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
):
    """Inference result should be the same between hf and vllm.

    All the image fixtures for the test are from IMAGE_ASSETS.
    For huggingface runner, we provide the PIL images as input.
    For vllm runner, we provide MultiModalDataDict objects
    and corresponding MultiModalConfig as input.
    Note, the text input is also adjusted to abide by vllm contract.
    The text output is sanitized to be able to compare with hf.
    """
    # NOTE: take care of the order. run vLLM first, and then run HF.
    # vLLM needs a fresh new process without cuda initialization.
    # if we run HF first, the cuda initialization will be done and it
    # will hurt multiprocessing backend with fork method (the default method).
    # max_model_len should be greater than image_feature_size
    with vllm_runner(
100
101
102
103
104
105
106
107
108
109
110
111
        model,
        runner="generate",
        max_model_len=max_model_len,
        max_num_seqs=2,
        dtype=dtype,
        limit_mm_per_prompt={"image": mm_limit},
        tensor_parallel_size=tensor_parallel_size,
        distributed_executor_backend=distributed_executor_backend,
        enable_lora=True,
        max_lora_rank=320,
        gpu_memory_utilization=0.8,  # set to 0.8 to avoid OOM in CI
        enforce_eager=True,
112
113
114
    ) as vllm_model:
        lora_request = LoRARequest("vision", 1, vision_lora_path)
        vllm_outputs_per_case = [
115
116
117
118
119
120
121
122
            vllm_model.generate_greedy_logprobs(
                prompts,
                max_tokens,
                num_logprobs=num_logprobs,
                images=images,
                audios=audios,
                lora_request=lora_request,
            )
123
            for prompts, images, audios in inputs
124
125
        ]

126
127
128
129
    # This error occurs inside `get_peft_model`
    # FIXME: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/75
    pytest.skip("HF impl is not compatible with current transformers")

130
    hf_model_kwargs = {"_attn_implementation": "sdpa"}
131
    with hf_runner(model, dtype=dtype, model_kwargs=hf_model_kwargs) as hf_model:
132
133
134
        hf_processor = hf_model.processor
        eos_token_id = hf_processor.tokenizer.eos_token_id

135
136
137
        def patch_hf_processor(
            *args, text="", images=None, audio=None, sampling_rate=None, **kwargs
        ):
138
139
140
            audios = None
            if audio is not None and sampling_rate is not None:
                audios = [(audio, sampling_rate)]
141
142
143
            return hf_processor(
                *args, text=text, images=images, audios=audios, **kwargs
            )
144
145
146

        hf_model.processor = patch_hf_processor

147
        hf_outputs_per_case = [
148
149
150
151
152
153
154
155
156
            hf_model.generate_greedy_logprobs_limit(
                prompts,
                max_tokens,
                num_logprobs=num_logprobs,
                images=images,
                audios=audios,
                eos_token_id=eos_token_id,
                num_logits_to_keep=0,
            )
157
            for prompts, images, audios in inputs
158
159
        ]

160
    for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
        check_logprobs_close(
            outputs_0_lst=hf_outputs,
            outputs_1_lst=vllm_outputs,
            name_0="hf",
            name_1="vllm",
        )


@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
    "size_factors",
    [
        # Single-scale
        [1.0],
        # Single-scale, batched
        [1.0, 1.0, 1.0],
        # Multi-scale
178
        [0.25, 0.5, 1.0],
179
180
181
    ],
)
@pytest.mark.parametrize("dtype", [target_dtype])
182
@pytest.mark.parametrize("max_model_len", [12800])
183
184
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
185
186
187
188
189
190
191
192
193
194
195
def test_models(
    hf_runner,
    vllm_runner,
    image_assets,
    model,
    size_factors,
    dtype: str,
    max_model_len: int,
    max_tokens: int,
    num_logprobs: int,
) -> None:
196
197
    images = [asset.pil_image for asset in image_assets]

198
199
200
201
202
203
204
205
    inputs_per_image = [
        (
            [prompt for _ in size_factors],
            [rescale_image_size(image, factor) for factor in size_factors],
            None,
        )
        for image, prompt in zip(images, HF_IMAGE_PROMPTS)
    ]
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

    run_test(
        hf_runner,
        vllm_runner,
        inputs_per_image,
        model,
        dtype=dtype,
        max_model_len=max_model_len,
        max_tokens=max_tokens,
        num_logprobs=num_logprobs,
        mm_limit=1,
        tensor_parallel_size=1,
    )


@large_gpu_test(min_gb=48)
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
    "size_factors",
    [
        # No image
        # [],
        # Single-scale
        [1.0],
        # Single-scale, batched
        [1.0, 1.0, 1.0],
        # Multi-scale
        [0.25, 0.5, 1.0],
    ],
)
@pytest.mark.parametrize("dtype", [target_dtype])
237
@pytest.mark.parametrize("max_model_len", [25600])
238
239
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
240
241
242
243
244
245
246
247
248
249
250
def test_multi_images_models(
    hf_runner,
    vllm_runner,
    image_assets,
    model,
    size_factors,
    dtype: str,
    max_model_len: int,
    max_tokens: int,
    num_logprobs: int,
) -> None:
251
252
253
    images = [asset.pil_image for asset in image_assets]

    inputs_per_case = [
254
255
        (
            [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
256
257
258
259
            [
                [rescale_image_size(image, factor) for image in images]
                for factor in size_factors
            ],
260
261
            None,
        ),
262
263
264
265
266
267
268
269
270
271
272
273
274
275
    ]

    run_test(
        hf_runner,
        vllm_runner,
        inputs_per_case,
        model,
        dtype=dtype,
        max_model_len=max_model_len,
        max_tokens=max_tokens,
        num_logprobs=num_logprobs,
        mm_limit=2,
        tensor_parallel_size=1,
    )
276
277
278
279


@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize("dtype", [target_dtype])
280
@pytest.mark.parametrize("max_model_len", [12800])
281
282
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
283
284
285
286
287
288
289
290
291
def test_vision_speech_models(
    hf_runner,
    vllm_runner,
    model,
    dtype: str,
    max_model_len: int,
    max_tokens: int,
    num_logprobs: int,
) -> None:
292
    # use the example speech question so that the model outputs are reasonable
293
    audio = load_audio(speech_question, sr=None)
294
    image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315

    inputs_vision_speech = [
        (
            ["<|user|><|image_1|><|audio_1|><|end|><|assistant|>"],
            [image],
            [audio],
        ),
    ]

    run_test(
        hf_runner,
        vllm_runner,
        inputs_vision_speech,
        model,
        dtype=dtype,
        max_model_len=max_model_len,
        max_tokens=max_tokens,
        num_logprobs=num_logprobs,
        mm_limit=1,
        tensor_parallel_size=1,
    )