test_chat_utils.py 79.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import warnings
5
from collections.abc import Mapping
6
from typing import Literal
7
8

import pytest
9
import torch
10
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
11

12
from vllm.assets.audio import AudioAsset
13
from vllm.assets.image import ImageAsset
14
from vllm.assets.video import VideoAsset
15
from vllm.config import ModelConfig
16
17
from vllm.entrypoints.chat_utils import (
    _try_extract_ast,
18
    apply_mistral_chat_template,
19
20
21
22
23
24
25
    load_chat_template,
    parse_chat_messages,
    parse_chat_messages_futures,
    resolve_chat_template_content_format,
    resolve_chat_template_kwargs,
    resolve_hf_chat_template,
)
26
from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
27
28
29
30
31
from vllm.multimodal.utils import (
    encode_audio_base64,
    encode_image_base64,
    encode_video_base64,
)
32
from vllm.tokenizers import MistralTokenizer, get_tokenizer
33
from vllm.utils.serial_utils import tensor2base64
34

35
from ..models.registry import HF_EXAMPLE_MODELS
36
37
38
39
from ..utils import VLLM_PATH

EXAMPLES_DIR = VLLM_PATH / "examples"

40
PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
41
ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
42
QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
43
QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
44
QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
45
QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
46
QWEN3_MODEL_ID = "Qwen/Qwen3-8B"
47
LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
48
HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
49
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
50
51


52
@pytest.fixture(scope="function")
53
def phi3v_model_config():
54
55
56
57
58
59
60
61
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "image": 2,
        },
    )
62
63


64
65
@pytest.fixture(scope="function")
def phi3v_model_config_mm_interleaved():
66
67
68
69
70
71
72
73
74
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        interleave_mm_strings=True,
        limit_mm_per_prompt={
            "image": 2,
        },
    )
75
76


77
78
79
80
81
82
83
84
85
86
87
88
89
@pytest.fixture(scope="function")
def phi3v_model_config_image_embeds():
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "image": 2,
        },
        enable_mm_embeds=True,
    )


90
91
92
93
94
95
96
97
98
99
100
101
@pytest.fixture(scope="function")
def qwen2_audio_model_config():
    return ModelConfig(
        QWEN2AUDIO_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "audio": 1,
        },
    )


102
103
104
105
106
107
108
109
110
111
112
113
114
@pytest.fixture(scope="function")
def audio_embeds_model_config():
    return ModelConfig(
        QWEN2AUDIO_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "audio": 2,
        },
        enable_mm_embeds=True,
    )


115
116
@pytest.fixture(scope="function")
def qwen25omni_model_config_mm_interleaved():
117
118
119
120
121
122
123
124
125
126
    return ModelConfig(
        QWEN25OMNI_MODEL_ID,
        runner="generate",
        interleave_mm_strings=True,
        limit_mm_per_prompt={
            "image": 2,
            "audio": 1,
            "video": 1,
        },
    )
127
128


129
130
@pytest.fixture(scope="function")
def mistral_model_config():
131
132
133
134
135
136
137
    return ModelConfig(
        MISTRAL_MODEL_ID,
        runner="generate",
        limit_mm_per_prompt={
            "image": 2,
        },
    )
138
139


140
141
@pytest.fixture(scope="module")
def image_url():
142
    image = ImageAsset("cherry_blossom")
143
144
145
146
    base64 = encode_image_base64(image.pil_image)
    return f"data:image/jpeg;base64,{base64}"


147
148
@pytest.fixture(scope="module")
def video_url():
149
    video = VideoAsset("baby_reading", 1)
150
151
152
153
154
155
    base64 = encode_video_base64(video.np_ndarrays)
    return f"data:video/jpeg;base64,{base64}"


@pytest.fixture(scope="module")
def audio_url():
156
    audio = AudioAsset("mary_had_lamb")
157
158
159
160
    base64 = encode_audio_base64(*audio.audio_and_sample_rate)
    return f"data:audio/ogg;base64,{base64}"


161
def _assert_mm_data_is_image_input(
162
    mm_data: MultiModalDataDict | None,
163
    image_count: int,
164
    skipped_image_indices: list | None = None,
165
166
167
168
169
170
171
) -> None:
    assert mm_data is not None
    assert set(mm_data.keys()) == {"image"}

    image_data = mm_data.get("image")
    assert image_data is not None

172
    assert isinstance(image_data, list) and len(image_data) == image_count
173
174
175
    if skipped_image_indices is not None:
        for i in skipped_image_indices:
            assert image_data[i] is None
176
177


178
def _assert_mm_uuids(
179
    mm_uuids: MultiModalUUIDDict | None,
180
    media_count: int,
181
    expected_uuids: list[str | None],
182
183
184
185
186
187
188
189
190
    modality: str = "image",
) -> None:
    if len(expected_uuids) > 0:
        assert mm_uuids is not None
        assert modality in mm_uuids

        image_uuids = mm_uuids.get(modality)
        assert image_uuids is not None

191
        assert isinstance(image_uuids, list) and len(image_uuids) == media_count
192
193
194
195
196
197

        assert image_uuids == expected_uuids
    else:
        assert mm_uuids is None


198
199
200
201
202
ModalityType = Literal["image", "video", "audio"]
MultiModalDataCounts = Mapping[ModalityType, int]


def _assert_mm_data_inputs(
203
    mm_data: MultiModalDataDict | None,
204
    data_count: MultiModalDataCounts,
205
    skipped_media_indices: dict[str, list] | None = None,  # modality -> list[int]
206
207
208
209
210
211
212
213
214
) -> None:
    assert mm_data is not None
    assert set(data_count.keys()) == (set(mm_data.keys()))

    for modality, n in data_count.items():
        modality_data = mm_data.get(modality)
        assert modality_data is not None
        assert isinstance(modality_data, list) and len(modality_data) == n

215
        if skipped_media_indices is not None:
216
            skipped_media_indices_for_modality = skipped_media_indices.get(modality)
217
218
219
220
            assert skipped_media_indices_for_modality is not None
            for i in skipped_media_indices_for_modality:
                assert modality_data[i] is None

221

222
223
224
225
def test_parse_chat_messages_single_image(
    phi3v_model_config,
    image_url,
):
226
    conversation, mm_data, mm_uuids = parse_chat_messages(
227
228
229
230
231
232
233
234
235
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
236
        phi3v_model_config,
237
238
        content_format="string",
    )
239

240
241
242
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
243
    _assert_mm_data_is_image_input(mm_data, 1)
244
245
246
247
248
249
250
251
252
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])


def test_parse_chat_messages_single_image_with_uuid(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
253
254
255
256
257
258
259
260
261
262
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid,
263
                    },
264
265
266
267
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
268
        phi3v_model_config,
269
270
271
        content_format="string",
    )

272
273
274
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
275
276
277
278
    _assert_mm_data_is_image_input(mm_data, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


279
280
281
282
283
284
def test_parse_chat_messages_single_empty_image_with_uuid(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
285
286
287
288
289
290
291
292
293
294
295
296
297
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
298
        phi3v_model_config,
299
300
301
        content_format="string",
    )

302
303
304
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
305
306
307
308
    _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


309
310
311
312
313
314
def test_parse_chat_messages_single_image_with_bad_uuid_format(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
315
316
317
318
319
320
321
322
323
324
325
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                            "uuid": image_uuid,
                        },
                        "bad_uuid_key": image_uuid,
326
                    },
327
328
329
330
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
331
        phi3v_model_config,
332
333
334
        content_format="string",
    )

335
336
337
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
338
339
340
341
342
343
344
345
346
347
348
349
    _assert_mm_data_is_image_input(mm_data, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])


def test_parse_chat_messages_multiple_images_with_uuids(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
350
351
352
353
354
355
356
357
358
359
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid1,
360
                    },
361
362
363
364
365
366
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid2,
367
                    },
368
369
370
371
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
372
        phi3v_model_config,
373
374
375
        content_format="string",
    )

376
377
378
379
380
381
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
382
383
384
385
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


386
387
388
389
390
391
392
393
def test_parse_chat_messages_multiple_empty_images_with_uuids(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid1,
                    },
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
412
        phi3v_model_config,
413
414
415
        content_format="string",
    )

416
417
418
419
420
421
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
422
423
424
425
426
427
428
429
430
431
432
433
    _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[0, 1])
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


def test_parse_chat_messages_mixed_empty_images_with_uuids(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
434
435
436
437
438
439
440
441
442
443
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid1,
444
                    },
445
446
447
448
449
450
451
452
453
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
454
        phi3v_model_config,
455
456
457
        content_format="string",
    )

458
459
460
461
462
463
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
464
465
466
467
    _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[1])
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


468
469
470
471
472
473
474
@pytest.mark.asyncio
async def test_parse_chat_messages_single_image_with_uuid_async(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
475
476
477
478
479
480
481
482
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
483
                    },
484
485
486
487
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
488
        phi3v_model_config,
489
490
491
        content_format="string",
    )

492
493
494
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
495
496
497
498
    _assert_mm_data_is_image_input(await mm_future, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


499
500
501
502
503
504
505
@pytest.mark.asyncio
async def test_parse_chat_messages_empty_image_with_uuid_async(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
506
507
508
509
510
511
512
513
514
515
516
517
518
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
519
        phi3v_model_config,
520
521
522
        content_format="string",
    )

523
524
525
526
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
    _assert_mm_data_is_image_input(await mm_future, 1, skipped_image_indices=[0])
527
528
529
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


530
531
532
533
534
535
536
537
538
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_uuids_async(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
539
540
541
542
543
544
545
546
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid1,
547
                    },
548
549
550
551
552
553
554
555
556
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
557
        phi3v_model_config,
558
559
560
        content_format="string",
    )

561
562
563
564
565
566
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
567
568
569
570
    _assert_mm_data_is_image_input(await mm_future, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


571
572
573
574
575
576
577
578
579
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid1,
                    },
                    {
                        "type": "image_pil",
                        "image_pil": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
598
        phi3v_model_config,
599
600
601
        content_format="string",
    )

602
603
604
605
606
607
608
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
    _assert_mm_data_is_image_input(await mm_future, 2, skipped_image_indices=[0, 1])
609
610
611
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


612
613
614
615
616
617
618
619
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
    phi3v_model_config,
    image_url,
):
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
620
621
622
623
624
625
626
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
627
                    },
628
629
630
631
632
633
634
635
636
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
637
        phi3v_model_config,
638
639
640
        content_format="string",
    )

641
642
643
644
645
646
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
647
648
    _assert_mm_data_is_image_input(await mm_future, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, image_uuid2])
649
650


651
652
653
654
def test_parse_chat_messages_empty_system(
    mistral_model_config,
):
    # Test string format
655
    conversation, _, _ = parse_chat_messages(
656
        [
657
            {"role": "system", "content": ""},
658
659
            {
                "role": "user",
660
                "content": [{"type": "text", "text": "Who are you?"}],
661
662
            },
        ],
663
        mistral_model_config,
664
665
        content_format="string",
    )
666
    assert conversation == [
667
668
        {"role": "system", "content": ""},
        {"role": "user", "content": "Who are you?"},
669
    ]
670
671

    # Test openai format
672
    conversation, _, _ = parse_chat_messages(
673
        [
674
            {"role": "system", "content": ""},
675
676
            {
                "role": "user",
677
                "content": [{"type": "text", "text": "Who are you?"}],
678
679
            },
        ],
680
        mistral_model_config,
681
682
683
        content_format="openai",
    )
    assert conversation == [
684
685
        {"role": "system", "content": [{"type": "text", "text": ""}]},
        {"role": "user", "content": [{"type": "text", "text": "Who are you?"}]},
686
    ]
687
688


689
@pytest.mark.asyncio
690
691
692
693
async def test_parse_chat_messages_single_image_async(
    phi3v_model_config,
    image_url,
):
694
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
695
696
697
698
699
700
701
702
703
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
704
        phi3v_model_config,
705
706
        content_format="string",
    )
707

708
709
710
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
711
    _assert_mm_data_is_image_input(await mm_future, 1)
712
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
713
714
715
716
717
718


def test_parse_chat_messages_multiple_images(
    phi3v_model_config,
    image_url,
):
719
    conversation, mm_data, mm_uuids = parse_chat_messages(
720
721
722
723
724
725
726
727
728
729
730
731
732
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
733
        phi3v_model_config,
734
735
        content_format="string",
    )
736

737
738
739
740
741
742
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
743
    _assert_mm_data_is_image_input(mm_data, 2)
744
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
745
746


747
748
749
750
751
def test_parse_chat_messages_empty_pil_image_with_uuid(
    phi3v_model_config,
):
    uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
752
753
754
755
756
757
758
759
760
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_pil", "image_pil": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
761
        phi3v_model_config,
762
763
764
        content_format="string",
    )

765
766
767
768
769
770
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
771
772
773
774
775
    _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


def test_parse_chat_messages_empty_image_embeds_with_uuid(
776
    phi3v_model_config_image_embeds,
777
778
779
):
    uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
780
781
782
783
784
785
786
787
788
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_embeds", "image_embeds": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
789
        phi3v_model_config_image_embeds,
790
791
792
        content_format="string",
    )

793
794
795
796
797
798
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
799

800
801
    assert mm_data is not None
    assert "image" in mm_data
802
803
804
805
    assert isinstance(mm_data["image"], list)
    assert len(mm_data["image"]) == 1
    assert mm_data["image"][0] is None

806
807
808
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
def test_parse_chat_messages_empty_audio_embeds_with_uuid(
    audio_embeds_model_config,
):
    """Test audio_embeds with UUID (no actual embeds data)."""
    uuid = "test-audio-uuid-123"

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {"type": "audio_embeds", "audio_embeds": None, "uuid": uuid},
                ],
            }
        ],
825
        audio_embeds_model_config,
826
827
828
829
830
831
        content_format="string",
    )

    # Should have audio in mm_data as None (UUID provided)
    assert mm_data is not None
    assert "audio" in mm_data
832
833
834
835
    assert isinstance(mm_data["audio"], list)
    assert len(mm_data["audio"]) == 1
    assert mm_data["audio"][0] is None

836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
    # UUID should be recorded
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[uuid])


def test_parse_chat_messages_audio_embeds_with_string(
    audio_embeds_model_config,
):
    """Test audio_embeds with base64 string embedding data."""

    import torch

    # Create a sample audio embedding tensor
    audio_embedding = torch.randn(1, 128, 768)

    # Encode it as base64
851
    base64_audio_embedding = tensor2base64(audio_embedding)
852
853
854
855
856
857
858
859
860
861
862
863
864
865

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {
                        "type": "audio_embeds",
                        "audio_embeds": base64_audio_embedding,
                    },
                ],
            }
        ],
866
        audio_embeds_model_config,
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
        content_format="string",
    )

    # Should have audio embedding in mm_data (single tensor, not a list)
    assert mm_data is not None
    assert "audio" in mm_data
    assert isinstance(mm_data["audio"], torch.Tensor)
    assert mm_data["audio"].shape == audio_embedding.shape
    # No UUID provided
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


@pytest.mark.asyncio
async def test_parse_chat_messages_audio_embeds_async(
    audio_embeds_model_config,
):
    """Test audio_embeds with async futures."""

    import torch

    # Create a sample audio embedding tensor
    audio_embedding = torch.randn(1, 128, 768)

    # Encode it as base64
893
    base64_audio_embedding = tensor2base64(audio_embedding)
894
895
896
897
898
899
900
901
902
903
904
905
906
907

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {
                        "type": "audio_embeds",
                        "audio_embeds": base64_audio_embedding,
                    },
                ],
            }
        ],
908
        audio_embeds_model_config,
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
        content_format="string",
    )

    # Should have audio embedding in mm_data (single tensor, not a list)
    mm_data = await mm_future
    assert mm_data is not None
    assert "audio" in mm_data
    assert isinstance(mm_data["audio"], torch.Tensor)
    assert mm_data["audio"].shape == audio_embedding.shape
    # No UUID provided
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
def test_parse_chat_messages_multiple_image_embeds(
    phi3v_model_config_image_embeds,
):
    """Test that multiple image_embeds in a single message are now supported.

    This test validates the fix for the limitation that previously only allowed
    one message with {'type': 'image_embeds'}. Now multiple image embeddings
    can be provided in a single request, similar to regular images.
    """
    # Create two sample image embedding tensors
    image_embedding_1 = torch.randn(256, 1024)
    image_embedding_2 = torch.randn(128, 1024)

    # Encode them as base64 using the convenience function
    base64_image_embedding_1 = tensor2base64(image_embedding_1)
    base64_image_embedding_2 = tensor2base64(image_embedding_2)

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_embeds",
                        "image_embeds": base64_image_embedding_1,
                    },
                    {
                        "type": "image_embeds",
                        "image_embeds": base64_image_embedding_2,
                    },
                    {"type": "text", "text": "Describe these two images."},
                ],
            }
        ],
        phi3v_model_config_image_embeds,
        content_format="string",
    )

    # Verify conversation structure
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nDescribe these two images.",
        }
    ]

    # Verify mm_data contains a list of embeddings (not a single embedding)
    assert mm_data is not None
    assert "image" in mm_data
    assert isinstance(mm_data["image"], list)
    assert len(mm_data["image"]) == 2

    # Verify each embedding has the correct shape
    assert isinstance(mm_data["image"][0], torch.Tensor)
    assert mm_data["image"][0].shape == image_embedding_1.shape
    assert isinstance(mm_data["image"][1], torch.Tensor)
    assert mm_data["image"][1].shape == image_embedding_2.shape

    # Verify UUIDs (None since we didn't provide any)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


def test_parse_chat_messages_multiple_image_embeds_with_uuids(
    phi3v_model_config_image_embeds,
):
    """Test multiple image_embeds with UUIDs.

    This validates that UUIDs are properly tracked for multiple embeddings.
    """
    uuid1 = "image-uuid-1"
    uuid2 = "image-uuid-2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_embeds",
                        "image_embeds": None,
                        "uuid": uuid1,
                    },
                    {
                        "type": "image_embeds",
                        "image_embeds": None,
                        "uuid": uuid2,
                    },
                    {"type": "text", "text": "Compare these images."},
                ],
            }
        ],
        phi3v_model_config_image_embeds,
        content_format="string",
    )

    # Verify conversation structure
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nCompare these images.",
        }
    ]

    # Verify mm_data contains a list with None values (UUID references)
    assert mm_data is not None
    assert "image" in mm_data
    assert isinstance(mm_data["image"], list)
    assert len(mm_data["image"]) == 2
    assert mm_data["image"][0] is None
    assert mm_data["image"][1] is None

    # Verify UUIDs are correctly tracked
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[uuid1, uuid2])


@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_image_embeds_async(
    phi3v_model_config_image_embeds,
):
    """Test multiple image_embeds with async parsing.

    This validates the AsyncMultiModalItemTracker also supports multiple embeddings.
    """
    # Create two sample image embedding tensors
    image_embedding_1 = torch.randn(200, 768)
    image_embedding_2 = torch.randn(150, 768)

    # Encode them as base64 using the convenience function
    base64_image_embedding_1 = tensor2base64(image_embedding_1)
    base64_image_embedding_2 = tensor2base64(image_embedding_2)

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_embeds",
                        "image_embeds": base64_image_embedding_1,
                    },
                    {
                        "type": "image_embeds",
                        "image_embeds": base64_image_embedding_2,
                    },
                    {"type": "text", "text": "What do these images show?"},
                ],
            }
        ],
        phi3v_model_config_image_embeds,
        content_format="string",
    )

    # Verify conversation structure
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat do these images show?",
        }
    ]

    # Await the future and verify mm_data
    mm_data = await mm_future
    assert mm_data is not None
    assert "image" in mm_data
    assert isinstance(mm_data["image"], list)
    assert len(mm_data["image"]) == 2

    # Verify each embedding has the correct shape
    assert isinstance(mm_data["image"][0], torch.Tensor)
    assert mm_data["image"][0].shape == image_embedding_1.shape
    assert isinstance(mm_data["image"][1], torch.Tensor)
    assert mm_data["image"][1].shape == image_embedding_2.shape

    # Verify UUIDs
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


1101
1102
@pytest.mark.asyncio
async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
1103
    phi3v_model_config_image_embeds,
1104
1105
1106
):
    uuid = "abcd"
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
1107
1108
1109
1110
1111
1112
1113
1114
1115
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_embeds", "image_embeds": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
1116
        phi3v_model_config_image_embeds,
1117
1118
1119
        content_format="string",
    )

1120
1121
1122
1123
1124
1125
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
1126
1127
1128
    mm_data = await mm_future
    assert mm_data is not None
    assert "image" in mm_data
1129
1130
1131
1132
    assert isinstance(mm_data["image"], list)
    assert len(mm_data["image"]) == 1
    assert mm_data["image"][0] is None

1133
1134
1135
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
def test_parse_chat_messages_empty_dict_image_embeds(
    phi3v_model_config_image_embeds,
):
    """Test that empty dictionary for image_embeds is handled without errors."""
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_embeds", "image_embeds": {}},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
        phi3v_model_config_image_embeds,
        content_format="string",
    )

    # Verify conversation structure
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]

    # Verify mm_data contains an empty dictionary of embeddings
    assert mm_data is not None
    assert "image" in mm_data
    assert isinstance(mm_data["image"], dict)
    assert len(mm_data["image"]) == 0

    # Verify UUIDs (None since we didn't provide any)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])


def test_parse_chat_messages_multiple_dict_image_embeds(
    phi3v_model_config_image_embeds,
):
    """Test that multiple dictionaries for image_embeds is handled without errors."""
    # Create two sample image embedding tensors
    batch_size = 2
    image_embedding_1 = torch.randn(batch_size, 256, 1024)
    image_embedding_2 = torch.randn(batch_size, 3)

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_embeds",
                        "image_embeds": {
                            "image_embedding_1": tensor2base64(p),
                            "image_embedding_2": tensor2base64(i),
                        },
                    }
                    for p, i in zip(image_embedding_1, image_embedding_2)
                ]
                + [
                    {"type": "text", "text": "Describe these two images."},
                ],
            }
        ],
        phi3v_model_config_image_embeds,
        content_format="string",
    )

    # Verify conversation structure
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nDescribe these two images.",
        }
    ]

    # Verify mm_data contains a dictionary of multi-embeddings
    assert mm_data is not None
    assert "image" in mm_data
    assert isinstance(mm_data["image"], dict)
    assert len(mm_data["image"]) == batch_size

    # Verify each embedding has the correct shape
    assert isinstance(mm_data["image"]["image_embedding_1"], torch.Tensor)
    assert mm_data["image"]["image_embedding_1"].shape == image_embedding_1.shape
    assert isinstance(mm_data["image"]["image_embedding_2"], torch.Tensor)
    assert mm_data["image"]["image_embedding_2"].shape == image_embedding_2.shape

    # Verify UUIDs (None since we didn't provide any)
    _assert_mm_uuids(mm_uuids, batch_size, expected_uuids=[None, None])


1228
@pytest.mark.asyncio
1229
1230
1231
1232
async def test_parse_chat_messages_multiple_images_async(
    phi3v_model_config,
    image_url,
):
1233
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
1247
        phi3v_model_config,
1248
1249
        content_format="string",
    )
1250

1251
1252
1253
1254
1255
1256
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
1257
    _assert_mm_data_is_image_input(await mm_future, 2)
1258
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1259
1260
1261
1262
1263
1264


def test_parse_chat_messages_placeholder_already_in_prompt(
    phi3v_model_config,
    image_url,
):
1265
    conversation, mm_data, mm_uuids = parse_chat_messages(
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "text",
                        "text": "What's in <|image_1|> and how does it compare to <|image_2|>?",  # noqa: E501
                    },
                ],
            }
        ],
1279
        phi3v_model_config,
1280
1281
        content_format="string",
    )
1282
1283
1284
1285
1286
1287
    assert conversation == [
        {
            "role": "user",
            "content": "What's in <|image_1|> and how does it compare to <|image_2|>?",
        }
    ]
1288
    _assert_mm_data_is_image_input(mm_data, 2)
1289
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1290
1291


1292
1293
1294
1295
def test_parse_chat_messages_placeholder_one_already_in_prompt(
    phi3v_model_config,
    image_url,
):
1296
    conversation, mm_data, mm_uuids = parse_chat_messages(
1297
1298
1299
1300
1301
1302
1303
1304
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "text",
1305
1306
                        "text": "What's in <|image_1|> and how does it compare to "
                        "the other one?",
1307
1308
1309
1310
                    },
                ],
            }
        ],
1311
        phi3v_model_config,
1312
1313
        content_format="string",
    )
1314

1315
1316
1317
    assert conversation == [
        {
            "role": "user",
1318
1319
            "content": "<|image_2|>\nWhat's in <|image_1|> and how does it compare to "
            "the other one?",
1320
1321
        }
    ]
1322
    _assert_mm_data_is_image_input(mm_data, 2)
1323
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1324
1325


1326
1327
1328
1329
def test_parse_chat_messages_multiple_images_across_messages(
    phi3v_model_config,
    image_url,
):
1330
    conversation, mm_data, mm_uuids = parse_chat_messages(
1331
1332
        [
            {
1333
                "role": "user",
1334
                "content": [
1335
1336
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in this image?"},
1337
1338
                ],
            },
1339
            {"role": "assistant", "content": "Some stuff."},
1340
            {
1341
                "role": "user",
1342
                "content": [
1343
1344
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What about this one?"},
1345
1346
1347
                ],
            },
        ],
1348
        phi3v_model_config,
1349
1350
        content_format="string",
    )
1351
1352

    assert conversation == [
1353
1354
1355
        {"role": "user", "content": "<|image_1|>\nWhat's in this image?"},
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "<|image_2|>\nWhat about this one?"},
1356
    ]
1357
    _assert_mm_data_is_image_input(mm_data, 2)
1358
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1359
1360


1361
def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
1362
    phi3v_model_config,
1363
    image_url,
1364
):
1365
1366
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
1367
1368
        [
            {
1369
                "role": "user",
1370
1371
1372
                "content": [
                    {
                        "type": "image_url",
1373
                        "image_url": {"url": image_url},
1374
1375
                        "uuid": image_uuid,
                    },
1376
                    {"type": "text", "text": "What's in this image?"},
1377
                ],
1378
            },
1379
            {"role": "assistant", "content": "Some stuff."},
1380
            {
1381
                "role": "user",
1382
1383
1384
                "content": [
                    {
                        "type": "image_url",
1385
                        "image_url": {"url": image_url},
1386
1387
                        "uuid": image_uuid,
                    },
1388
                    {"type": "text", "text": "What about this one?"},
1389
1390
1391
                ],
            },
        ],
1392
        phi3v_model_config,
1393
1394
1395
1396
        content_format="string",
    )

    assert conversation == [
1397
1398
1399
        {"role": "user", "content": "<|image_1|>\nWhat's in this image?"},
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "<|image_2|>\nWhat about this one?"},
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
    ]
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])


def test_parse_chat_messages_context_text_format(
    phi3v_model_config,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
1412
                "content": [{"type": "text", "text": "What's in this text?"}],
1413
            },
1414
1415
            {"role": "assistant", "content": "Some stuff."},
            {"role": "user", "content": "What about this one?"},
1416
        ],
1417
        phi3v_model_config,
1418
1419
        content_format="openai",
    )
1420
1421
1422
1423

    assert conversation == [
        {
            "role": "user",
1424
            "content": [{"type": "text", "text": "What's in this text?"}],
1425
1426
1427
        },
        {
            "role": "assistant",
1428
            "content": [{"type": "text", "text": "Some stuff."}],
1429
1430
1431
        },
        {
            "role": "user",
1432
            "content": [{"type": "text", "text": "What about this one?"}],
1433
1434
        },
    ]
1435
1436
    assert mm_data is None
    assert mm_uuids is None
1437
1438


1439
1440
1441
1442
def test_parse_chat_messages_rejects_too_many_images_in_one_message(
    phi3v_model_config,
    image_url,
):
1443
1444
1445
    with warnings.catch_warnings():
        warnings.filterwarnings(
            "ignore",
1446
1447
            message="coroutine 'async_get_and_parse_image' was never awaited",
        )
1448
        with pytest.raises(ValueError, match="At most"):
1449
            parse_chat_messages(
1450
1451
1452
1453
1454
1455
1456
                [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1457
                            },
1458
1459
1460
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1461
                            },
1462
1463
1464
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1465
                            },
1466
1467
1468
1469
                            {"type": "text", "text": "What's in these images?"},
                        ],
                    }
                ],
1470
                phi3v_model_config,
1471
1472
                content_format="string",
            )
1473
1474


1475
1476
1477
1478
def test_parse_chat_messages_rejects_too_many_images_across_messages(
    phi3v_model_config,
    image_url,
):
1479
1480
1481
    with warnings.catch_warnings():
        warnings.filterwarnings(
            "ignore",
1482
1483
            message="coroutine 'async_get_and_parse_image' was never awaited",
        )
1484
        with pytest.raises(ValueError, match="At most"):
1485
            parse_chat_messages(
1486
1487
                [
                    {
1488
                        "role": "user",
1489
1490
1491
                        "content": [
                            {
                                "type": "image_url",
1492
                                "image_url": {"url": image_url},
1493
                            },
1494
                            {"type": "text", "text": "What's in this image?"},
1495
1496
                        ],
                    },
1497
                    {"role": "assistant", "content": "Some stuff."},
1498
                    {
1499
                        "role": "user",
1500
1501
1502
                        "content": [
                            {
                                "type": "image_url",
1503
                                "image_url": {"url": image_url},
1504
1505
1506
                            },
                            {
                                "type": "image_url",
1507
                                "image_url": {"url": image_url},
1508
                            },
1509
                            {"type": "text", "text": "What about these two?"},
1510
1511
1512
                        ],
                    },
                ],
1513
                phi3v_model_config,
1514
1515
                content_format="string",
            )
1516
1517
1518
1519
1520
1521


def test_parse_chat_messages_multiple_images_uncommon_input(
    phi3v_model_config,
    image_url,
):
1522
    conversation, mm_data, mm_uuids = parse_chat_messages(
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
        [
            {
                "role": "user",
                "content": [
                    "What's in these images?",
                    {"image_url": image_url},
                    {"image_url": image_url},
                ],
            }
        ],
1533
        phi3v_model_config,
1534
1535
        content_format="string",
    )
1536

1537
1538
1539
1540
1541
1542
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
1543
    _assert_mm_data_is_image_input(mm_data, 2)
1544
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1545
1546


1547
1548
1549
1550
def test_parse_chat_messages_multiple_images_interleave(
    phi3v_model_config_mm_interleaved,
    image_url,
):
1551
    conversation, mm_data, mm_uuids = parse_chat_messages(
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
                    },
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "and this one"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1567
        phi3v_model_config_mm_interleaved,
1568
1569
1570
        content_format="string",
    )

1571
1572
1573
1574
1575
1576
1577
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1578
    _assert_mm_data_is_image_input(mm_data, 2)
1579
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1580
1581
1582
1583
1584
1585
1586


@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_interleave_async(
    phi3v_model_config_mm_interleaved,
    image_url,
):
1587
    conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
                    },
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "and this one"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1603
        phi3v_model_config_mm_interleaved,
1604
1605
1606
        content_format="string",
    )

1607
1608
1609
1610
1611
1612
1613
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1614
    _assert_mm_data_is_image_input(await mm_data, 2)
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
    phi3v_model_config_mm_interleaved,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1625
1626
1627
1628
1629
1630
1631
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
1632
                    },
1633
1634
1635
1636
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
1637
                    },
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
                    {"type": "text", "text": "and this one"},
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1648
        phi3v_model_config_mm_interleaved,
1649
1650
1651
        content_format="string",
    )

1652
1653
1654
1655
1656
1657
1658
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1659
1660
    _assert_mm_data_is_image_input(await mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1661
1662
1663
1664
1665
1666


def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
    phi3v_model_config_mm_interleaved,
    image_url,
):
1667
    conversation, mm_data, mm_uuids = parse_chat_messages(
1668
1669
        [
            {
1670
                "role": "user",
1671
                "content": [
1672
1673
1674
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Be accurate."},
1675
1676
                ],
            },
1677
            {"role": "assistant", "content": "Some stuff."},
1678
            {
1679
                "role": "user",
1680
                "content": [
1681
1682
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
1683
1684
1685
                ],
            },
        ],
1686
        phi3v_model_config_mm_interleaved,
1687
1688
1689
        content_format="string",
    )

1690
1691
1692
1693
1694
    assert conversation == [
        {
            "role": "user",
            "content": "What's on this image?\n<|image_1|>\nBe accurate.",
        },
1695
1696
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "What's on this image?\n<|image_2|>"},
1697
    ]
1698
    _assert_mm_data_is_image_input(mm_data, 2)
1699
1700
1701
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


1702
def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave(
1703
1704
1705
1706
1707
1708
1709
    phi3v_model_config_mm_interleaved,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1710
                "role": "user",
1711
                "content": [
1712
                    {"type": "text", "text": "What's on this image?"},
1713
1714
                    {
                        "type": "image_url",
1715
                        "image_url": {"url": image_url},
1716
1717
                        "uuid": image_uuid,
                    },
1718
                    {"type": "text", "text": "Be accurate."},
1719
1720
                ],
            },
1721
            {"role": "assistant", "content": "Some stuff."},
1722
            {
1723
                "role": "user",
1724
                "content": [
1725
                    {"type": "text", "text": "What's on this image?"},
1726
1727
                    {
                        "type": "image_url",
1728
                        "image_url": {"url": image_url},
1729
1730
1731
1732
1733
                        "uuid": image_uuid,
                    },
                ],
            },
        ],
1734
        phi3v_model_config_mm_interleaved,
1735
1736
1737
1738
1739
1740
1741
1742
        content_format="string",
    )

    assert conversation == [
        {
            "role": "user",
            "content": "What's on this image?\n<|image_1|>\nBe accurate.",
        },
1743
1744
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "What's on this image?\n<|image_2|>"},
1745
1746
1747
    ]
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1748
1749
1750


def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1751
1752
1753
1754
1755
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
1756
    conversation, mm_data, mm_uuids = parse_chat_messages(
1757
1758
        [
            {
1759
                "role": "user",
1760
                "content": [
1761
1762
1763
1764
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Now listen to this audio"},
                    {"type": "audio_url", "audio_url": {"url": audio_url}},
1765
1766
                ],
            },
1767
            {"role": "assistant", "content": "Some stuff."},
1768
            {
1769
                "role": "user",
1770
                "content": [
1771
1772
1773
1774
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "And what's in the video?"},
                    {"type": "video_url", "video_url": {"url": video_url}},
1775
1776
1777
                ],
            },
        ],
1778
        qwen25omni_model_config_mm_interleaved,
1779
1780
1781
1782
1783
        content_format="string",
    )

    assert conversation == [
        {
1784
            "role": "user",
1785
1786
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1787
        },
1788
        {"role": "assistant", "content": "Some stuff."},
1789
        {
1790
            "role": "user",
1791
1792
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1793
1794
        },
    ]
1795
1796

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1797
    _assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=[None, None])
1798
1799
1800
1801
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


1802
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave(
1803
1804
1805
1806
1807
1808
1809
1810
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1811
                "role": "user",
1812
                "content": [
1813
                    {"type": "text", "text": "What's on this image?"},
1814
1815
                    {
                        "type": "image_url",
1816
                        "image_url": {"url": image_url},
1817
1818
                        "uuid": "image_123",
                    },
1819
                    {"type": "text", "text": "Now listen to this audio"},
1820
1821
                    {
                        "type": "audio_url",
1822
                        "audio_url": {"url": audio_url},
1823
1824
1825
1826
                        "uuid": "audio_123",
                    },
                ],
            },
1827
            {"role": "assistant", "content": "Some stuff."},
1828
            {
1829
                "role": "user",
1830
                "content": [
1831
                    {"type": "text", "text": "What's on this image?"},
1832
1833
                    {
                        "type": "image_url",
1834
                        "image_url": {"url": image_url},
1835
1836
                        "uuid": "image_123",
                    },
1837
                    {"type": "text", "text": "And what's in the video?"},
1838
1839
                    {
                        "type": "video_url",
1840
                        "video_url": {"url": video_url},
1841
1842
1843
1844
1845
                        "uuid": "video_123",
                    },
                ],
            },
        ],
1846
        qwen25omni_model_config_mm_interleaved,
1847
1848
1849
1850
1851
        content_format="string",
    )

    assert conversation == [
        {
1852
            "role": "user",
1853
1854
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1855
        },
1856
        {"role": "assistant", "content": "Some stuff."},
1857
        {
1858
            "role": "user",
1859
1860
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1861
1862
1863
1864
        },
    ]

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1865
1866
1867
1868
1869
    _assert_mm_uuids(
        mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
    )
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
1870
1871


1872
1873
1874
1875
1876
1877
1878
1879
1880
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave(  # noqa: E501
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1881
                "role": "user",
1882
                "content": [
1883
                    {"type": "text", "text": "What's on this image?"},
1884
1885
1886
1887
1888
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": "image_123",
                    },
1889
                    {"type": "text", "text": "Now listen to this audio"},
1890
1891
1892
1893
1894
1895
1896
                    {
                        "type": "audio_url",
                        "audio_url": None,
                        "uuid": "audio_123",
                    },
                ],
            },
1897
            {"role": "assistant", "content": "Some stuff."},
1898
            {
1899
                "role": "user",
1900
                "content": [
1901
                    {"type": "text", "text": "What's on this image?"},
1902
1903
1904
1905
1906
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": "image_123",
                    },
1907
                    {"type": "text", "text": "And what's in the video?"},
1908
1909
1910
1911
1912
1913
1914
1915
                    {
                        "type": "video_url",
                        "video_url": None,
                        "uuid": "video_123",
                    },
                ],
            },
        ],
1916
        qwen25omni_model_config_mm_interleaved,
1917
1918
1919
1920
1921
        content_format="string",
    )

    assert conversation == [
        {
1922
            "role": "user",
1923
1924
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1925
        },
1926
        {"role": "assistant", "content": "Some stuff."},
1927
        {
1928
            "role": "user",
1929
1930
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1931
1932
1933
        },
    ]

1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
    _assert_mm_data_inputs(
        mm_data,
        {"image": 2, "video": 1, "audio": 1},
        skipped_media_indices={"image": [0, 1], "video": [0], "audio": [0]},
    )
    _assert_mm_uuids(
        mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
    )
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
1944
1945


1946
1947
1948
1949
1950
1951
1952
1953
1954
def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave(  # noqa: E501
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1955
                "role": "user",
1956
                "content": [
1957
                    {"type": "text", "text": "What's on this image?"},
1958
1959
                    {
                        "type": "image_url",
1960
                        "image_url": {"url": image_url},
1961
1962
                        "uuid": "image_123",
                    },
1963
1964
                    {"type": "text", "text": "Now listen to this audio"},
                    {"type": "audio_url", "audio_url": {"url": audio_url}},
1965
1966
                ],
            },
1967
            {"role": "assistant", "content": "Some stuff."},
1968
            {
1969
                "role": "user",
1970
                "content": [
1971
1972
1973
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "And what's in the video?"},
1974
1975
                    {
                        "type": "video_url",
1976
                        "video_url": {"url": video_url},
1977
1978
1979
1980
1981
                        "uuid": "video_123",
                    },
                ],
            },
        ],
1982
        qwen25omni_model_config_mm_interleaved,
1983
1984
1985
1986
1987
        content_format="string",
    )

    assert conversation == [
        {
1988
            "role": "user",
1989
1990
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1991
        },
1992
        {"role": "assistant", "content": "Some stuff."},
1993
        {
1994
            "role": "user",
1995
1996
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1997
1998
1999
2000
        },
    ]

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
2001
2002
    _assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=["image_123", None])
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
2003
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
2004
2005
2006
2007
2008
2009
2010


def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
    phi3v_model_config_mm_interleaved,
    image_url,
):
    with pytest.raises(
2011
2012
2013
        ValueError,
        match=r"Found more '<|image_1|>' placeholders in input prompt "
        "than actual multimodal data items.",
2014
    ):
2015
        parse_chat_messages(
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
            [
                {
                    "role": "user",
                    "content": [
                        {"type": "image_url", "image_url": {"url": image_url}},
                        {"type": "image_url", "image_url": {"url": image_url}},
                        {
                            "type": "text",
                            "text": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
                            "Do they have differences?",
                        },
                    ],
                }
            ],
2030
            phi3v_model_config_mm_interleaved,
2031
2032
2033
2034
            content_format="string",
        )


2035
2036
2037
2038
2039
@pytest.mark.parametrize(
    "model",
    [
        QWEN2VL_MODEL_ID,  # tokenizer.chat_template is of type str
        HERMES_MODEL_ID,  # tokenizer.chat_template is of type dict
2040
2041
    ],
)
2042
2043
2044
@pytest.mark.parametrize("use_tools", [True, False])
def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
    """checks that chat_template is a dict type for HF models."""
2045
2046
2047
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
        enforce_eager=model_info.enforce_eager,
        dtype=model_info.dtype,
    )

    # Build the tokenizer
2063
    tokenizer = get_tokenizer(
2064
2065
        model,
        trust_remote_code=model_config.trust_remote_code,
2066
2067
    )

2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
    tools = (
        [
            {
                "type": "function",
                "function": {
                    "name": "dummy_function_name",
                    "description": "This is a dummy function",
                    "parameters": sample_json_schema,
                },
            }
        ]
        if use_tools
        else None
    )
2082
2083

    # Test detecting the tokenizer's chat_template
2084
    chat_template = resolve_hf_chat_template(
2085
2086
2087
        tokenizer,
        chat_template=None,
        tools=tools,
2088
        model_config=model_config,
2089
2090
2091
2092
    )
    assert isinstance(chat_template, str)


2093
2094
2095
2096
2097
2098
@pytest.mark.parametrize(
    "model, expected_kwargs",
    [
        (
            QWEN2VL_MODEL_ID,
            {
2099
2100
2101
2102
                "add_vision_id",
                "add_generation_prompt",
                "continue_final_message",
                "tools",
2103
2104
2105
2106
2107
            },
        ),
        (
            QWEN3_MODEL_ID,
            {
2108
2109
2110
2111
                "enable_thinking",
                "add_generation_prompt",
                "continue_final_message",
                "tools",
2112
2113
2114
2115
            },
        ),
    ],
)
2116
def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwargs):
2117
2118
2119
2120
    """checks that chat_template is a dict type for HF models."""
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
    tools = [
        {
            "type": "function",
            "function": {
                "name": "dummy_function_name",
                "description": "This is a dummy function",
                "parameters": sample_json_schema,
            },
        }
    ]
2131
2132
2133
2134
2135
2136
2137

    chat_template_kwargs = {
        # both unused
        "unsed_kwargs_1": 123,
        "unsed_kwargs_2": "abc",
        # should not appear
        "chat_template": "{% Hello world! %}",
2138
        "tokenize": True,
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
        # used by tokenizer
        "continue_final_message": True,
        "tools": tools,
        # both used by Qwen2-VL and Qwen3
        "add_generation_prompt": True,
        # only used by Qwen2-VL
        "add_vision_id": True,
        # only used by Qwen3
        "enable_thinking": True,
    }

2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
        enforce_eager=model_info.enforce_eager,
        dtype=model_info.dtype,
    )

    # Build the tokenizer
2165
    tokenizer = get_tokenizer(
2166
2167
        model,
        trust_remote_code=model_config.trust_remote_code,
2168
2169
2170
2171
2172
2173
2174
    )

    # Test detecting the tokenizer's chat_template
    chat_template = resolve_hf_chat_template(
        tokenizer,
        chat_template=None,
        tools=tools,
2175
        model_config=model_config,
2176
    )
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
    with pytest.raises(
        ValueError, match="Found unexpected chat template kwargs from request"
    ):
        # should raise error if `chat_template_kwargs` contains
        # `chat_template` or `tokenize`
        resolve_chat_template_kwargs(
            tokenizer,
            chat_template=chat_template,
            chat_template_kwargs=chat_template_kwargs,
        )
2187
2188
2189
2190
    resolved_chat_template_kwargs = resolve_chat_template_kwargs(
        tokenizer,
        chat_template=chat_template,
        chat_template_kwargs=chat_template_kwargs,
2191
        raise_on_unexpected=False,
2192
2193
2194
    )
    assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs

2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
    # Additional test: Verify HF base parameters work with **kwargs tokenizers
    # This validates the fix for tokenizers like Kimi K2 that use **kwargs
    # to receive standard HuggingFace parameters instead of declaring them explicitly
    from vllm.entrypoints.chat_utils import _get_hf_base_chat_template_params

    hf_base_params = _get_hf_base_chat_template_params()
    # Verify common HF parameters are in the base class
    assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset(
        hf_base_params
    ), f"Expected HF base params not found in {hf_base_params}"

    # Test with a mock tokenizer that uses **kwargs (like Kimi K2)
    class MockTokenizerWithKwargs:
        def apply_chat_template(self, conversation, **kwargs):
            return "mocked_output"

    mock_tokenizer = MockTokenizerWithKwargs()
    mock_kwargs = {
        "add_generation_prompt": True,
        "tools": tools,
        "continue_final_message": False,
        "unknown_param": "should_be_filtered",
    }
    resolved_mock = resolve_chat_template_kwargs(
        mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False
    )
    # HF base params should pass through even with **kwargs tokenizer
    assert "add_generation_prompt" in resolved_mock
    assert "tools" in resolved_mock
    assert "continue_final_message" in resolved_mock
    # Unknown params should be filtered out
    assert "unknown_param" not in resolved_mock

2228

2229
2230
# NOTE: Qwen2-Audio default chat template is specially defined inside
# processor class instead of using `tokenizer_config.json`
2231
2232
@pytest.mark.parametrize(
    ("model", "expected_format"),
2233
2234
2235
2236
2237
2238
2239
2240
    [
        (PHI3V_MODEL_ID, "string"),
        (QWEN2VL_MODEL_ID, "openai"),
        (QWEN25VL_MODEL_ID, "openai"),
        (ULTRAVOX_MODEL_ID, "string"),
        (QWEN2AUDIO_MODEL_ID, "openai"),
        (LLAMA_GUARD_MODEL_ID, "openai"),
    ],
2241
2242
)
def test_resolve_content_format_hf_defined(model, expected_format):
2243
2244
2245
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
        enforce_eager=model_info.enforce_eager,
        dtype=model_info.dtype,
    )
2259

2260
    tokenizer = get_tokenizer(
2261
2262
        model,
        trust_remote_code=model_config.trust_remote_code,
2263
2264
    )

2265
    # Test detecting the tokenizer's chat_template
2266
    chat_template = resolve_hf_chat_template(
2267
2268
2269
        tokenizer,
        chat_template=None,
        tools=None,
2270
        model_config=model_config,
2271
    )
2272
2273
2274
2275
2276
2277
2278
2279
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
2280
2281
2282
2283
        None,  # Test detecting the tokenizer's chat_template
        None,
        "auto",
        tokenizer,
2284
        model_config=model_config,
2285
2286
2287
2288
2289
2290
2291
    )

    assert resolved_format == expected_format


@pytest.mark.parametrize(
    ("model", "expected_format"),
2292
2293
2294
2295
2296
2297
2298
2299
2300
    [
        ("Salesforce/blip2-opt-2.7b", "string"),
        ("facebook/chameleon-7b", "string"),
        ("deepseek-ai/deepseek-vl2-tiny", "string"),
        ("adept/fuyu-8b", "string"),
        ("google/paligemma-3b-mix-224", "string"),
        ("Qwen/Qwen-VL", "string"),
        ("Qwen/Qwen-VL-Chat", "string"),
    ],
2301
2302
2303
2304
2305
)
def test_resolve_content_format_fallbacks(model, expected_format):
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
        enforce_eager=model_info.enforce_eager,
        dtype=model_info.dtype,
    )
2319

2320
    tokenizer = get_tokenizer(
2321
2322
        model_config.tokenizer,
        trust_remote_code=model_config.trust_remote_code,
2323
2324
2325
2326
2327
2328
2329
    )

    # Test detecting the tokenizer's chat_template
    chat_template = resolve_hf_chat_template(
        tokenizer,
        chat_template=None,
        tools=None,
2330
        model_config=model_config,
2331
2332
2333
2334
2335
2336
2337
2338
2339
    )
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
2340
        None,  # Test detecting the tokenizer's chat_template
2341
        None,
2342
2343
        "auto",
        tokenizer,
2344
        model_config=model_config,
2345
2346
2347
2348
2349
2350
2351
    )

    assert resolved_format == expected_format


@pytest.mark.parametrize(
    ("template_path", "expected_format"),
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
    [
        ("template_alpaca.jinja", "string"),
        ("template_baichuan.jinja", "string"),
        ("template_chatglm.jinja", "string"),
        ("template_chatglm2.jinja", "string"),
        ("template_chatml.jinja", "string"),
        ("template_dse_qwen2_vl.jinja", "openai"),
        ("template_falcon_180b.jinja", "string"),
        ("template_falcon.jinja", "string"),
        ("template_inkbot.jinja", "string"),
        ("template_teleflm.jinja", "string"),
        ("template_vlm2vec_phi3v.jinja", "openai"),
        ("template_vlm2vec_qwen2vl.jinja", "openai"),
        ("tool_chat_template_granite_20b_fc.jinja", "string"),
        ("tool_chat_template_hermes.jinja", "string"),
        ("tool_chat_template_internlm2_tool.jinja", "string"),
        ("tool_chat_template_llama3.1_json.jinja", "openai"),
        ("tool_chat_template_llama3.2_json.jinja", "openai"),
        ("tool_chat_template_mistral_parallel.jinja", "string"),
        ("tool_chat_template_mistral.jinja", "string"),
    ],
2373
2374
)
def test_resolve_content_format_examples(template_path, expected_format):
2375
2376
2377
2378
2379
    model_config = ModelConfig(
        PHI3V_MODEL_ID,  # Dummy
        tokenizer=PHI3V_MODEL_ID,  # Dummy
        trust_remote_code=True,
    )
2380

2381
    dummy_tokenizer = get_tokenizer(
2382
2383
        PHI3V_MODEL_ID,  # Dummy
        trust_remote_code=model_config.trust_remote_code,
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
    )
    dummy_tokenizer.chat_template = None

    chat_template = load_chat_template(EXAMPLES_DIR / template_path)
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
        chat_template,
2397
        None,
2398
2399
        "auto",
        dummy_tokenizer,
2400
        model_config=model_config,
2401
2402
2403
    )

    assert resolved_format == expected_format
Julien Denize's avatar
Julien Denize committed
2404
2405


2406
def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
    messages = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "thinking",
                    "closed": True,
                    "thinking": "Only return the answer when you are confident.",
                },
            ],
        },
        {"role": "user", "content": "What is 2+2?"},
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
                {
                    "type": "text",
                    "text": "The answer is 4.",
                },
            ],
        },
    ]
Julien Denize's avatar
Julien Denize committed
2432

2433
    conversation_with_thinking, _, _ = parse_chat_messages(
Julien Denize's avatar
Julien Denize committed
2434
        messages,
2435
        mistral_model_config,
Julien Denize's avatar
Julien Denize committed
2436
2437
2438
        content_format="openai",
    )

2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
    expected_conversation = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "text",
                    "text": "Only return the answer when you are confident.",
                },
            ],
        },
        {
            "role": "user",
            "content": [{"type": "text", "text": "What is 2+2?"}],
        },
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "text", "text": "2+2 = 4"},
                {"type": "text", "text": "The answer is 4."},
            ],
        },
    ]
Julien Denize's avatar
Julien Denize committed
2463
2464
2465
2466
2467

    assert conversation_with_thinking == expected_conversation


def test_apply_mistral_chat_template_thinking_chunk():
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
    messages = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "thinking",
                    "closed": True,
                    "thinking": "Only return the answer when you are confident.",
                },
            ],
        },
        {"role": "user", "content": "What is 2+2?"},
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
                {
                    "type": "text",
                    "text": "The answer is 4.",
                },
            ],
        },
        {"role": "user", "content": "Thanks, what is 3+3?"},
    ]
Julien Denize's avatar
Julien Denize committed
2494
    mistral_tokenizer = MistralTokenizer.from_pretrained(
2495
        "mistralai/Magistral-Small-2509"
2496
    )
Julien Denize's avatar
Julien Denize committed
2497

2498
2499
2500
    tokens_ids = apply_mistral_chat_template(
        mistral_tokenizer, messages, chat_template=None, tools=None
    )
Julien Denize's avatar
Julien Denize committed
2501
2502

    string_tokens = mistral_tokenizer.mistral.decode(
2503
2504
        tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
    )
Julien Denize's avatar
Julien Denize committed
2505
2506
2507
2508
2509
2510

    expected_tokens = (
        r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
        r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
        r"[INST]What is 2+2?[/INST]"
        r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
2511
2512
        r"[INST]Thanks, what is 3+3?[/INST]"
    )
Julien Denize's avatar
Julien Denize committed
2513
2514

    assert string_tokens == expected_tokens
2515
2516
2517
2518
2519
2520
2521


def test_parse_chat_messages_single_empty_audio_with_uuid(
    qwen2_audio_model_config,
):
    audio_uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "input_audio",
                        "input_audio": {},
                        "uuid": audio_uuid,
                    },
                    {"type": "text", "text": "What does the audio say?"},
                ],
            }
        ],
2535
        qwen2_audio_model_config,
2536
2537
2538
        content_format="string",
    )

2539
2540
2541
    assert conversation == [
        {
            "role": "user",
2542
2543
            "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
            "audio say?",
2544
2545
        }
    ]
2546
    _assert_mm_data_inputs(mm_data, {"audio": 1})
2547
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
2548
2549
2550
2551
2552
2553
2554
2555


@pytest.mark.asyncio
async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
    qwen2_audio_model_config,
):
    audio_uuid = "abcd"
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "input_audio",
                        "input_audio": {},
                        "uuid": audio_uuid,
                    },
                    {"type": "text", "text": "What does the audio say?"},
                ],
            }
        ],
2569
        qwen2_audio_model_config,
2570
2571
2572
        content_format="string",
    )

2573
2574
2575
    assert conversation == [
        {
            "role": "user",
2576
2577
            "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
            "audio say?",
2578
2579
        }
    ]
2580
    _assert_mm_data_inputs(await mm_future, {"audio": 1})
2581
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])