test_chat_utils.py 73.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import warnings
5
from collections.abc import Mapping
6
from typing import Literal
7
8

import pytest
9
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
10

11
from vllm.assets.audio import AudioAsset
12
from vllm.assets.image import ImageAsset
13
from vllm.assets.video import VideoAsset
14
from vllm.config import ModelConfig
15
16
from vllm.entrypoints.chat_utils import (
    _try_extract_ast,
17
    apply_mistral_chat_template,
18
19
20
21
22
23
24
    load_chat_template,
    parse_chat_messages,
    parse_chat_messages_futures,
    resolve_chat_template_content_format,
    resolve_chat_template_kwargs,
    resolve_hf_chat_template,
)
25
from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
26
27
28
29
30
from vllm.multimodal.utils import (
    encode_audio_base64,
    encode_image_base64,
    encode_video_base64,
)
31
from vllm.tokenizers import MistralTokenizer, get_tokenizer
32

33
from ..models.registry import HF_EXAMPLE_MODELS
34
35
36
37
from ..utils import VLLM_PATH

EXAMPLES_DIR = VLLM_PATH / "examples"

38
PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
39
ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
40
QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
41
QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
42
QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
43
QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
44
QWEN3_MODEL_ID = "Qwen/Qwen3-8B"
45
LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
46
HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
47
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
48
49


50
@pytest.fixture(scope="function")
51
def phi3v_model_config():
52
53
54
55
56
57
58
59
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "image": 2,
        },
    )
60
61


62
63
@pytest.fixture(scope="function")
def phi3v_model_config_mm_interleaved():
64
65
66
67
68
69
70
71
72
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        interleave_mm_strings=True,
        limit_mm_per_prompt={
            "image": 2,
        },
    )
73
74


75
76
77
78
79
80
81
82
83
84
85
86
87
@pytest.fixture(scope="function")
def phi3v_model_config_image_embeds():
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "image": 2,
        },
        enable_mm_embeds=True,
    )


88
89
@pytest.fixture(scope="module")
def phi3v_tokenizer():
90
    return get_tokenizer(PHI3V_MODEL_ID)
91
92


93
94
95
96
97
98
99
100
101
102
103
104
@pytest.fixture(scope="function")
def qwen2_audio_model_config():
    return ModelConfig(
        QWEN2AUDIO_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "audio": 1,
        },
    )


105
106
107
108
109
110
111
112
113
114
115
116
117
@pytest.fixture(scope="function")
def audio_embeds_model_config():
    return ModelConfig(
        QWEN2AUDIO_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "audio": 2,
        },
        enable_mm_embeds=True,
    )


118
119
@pytest.fixture(scope="module")
def qwen2_audio_tokenizer():
120
    return get_tokenizer(QWEN2AUDIO_MODEL_ID)
121
122


123
124
@pytest.fixture(scope="function")
def qwen25omni_model_config_mm_interleaved():
125
126
127
128
129
130
131
132
133
134
    return ModelConfig(
        QWEN25OMNI_MODEL_ID,
        runner="generate",
        interleave_mm_strings=True,
        limit_mm_per_prompt={
            "image": 2,
            "audio": 1,
            "video": 1,
        },
    )
135
136
137
138


@pytest.fixture(scope="module")
def qwen25omni_tokenizer():
139
    return get_tokenizer(QWEN25OMNI_MODEL_ID)
140
141


142
143
@pytest.fixture(scope="function")
def mistral_model_config():
144
145
146
147
148
149
150
    return ModelConfig(
        MISTRAL_MODEL_ID,
        runner="generate",
        limit_mm_per_prompt={
            "image": 2,
        },
    )
151
152
153
154


@pytest.fixture(scope="module")
def mistral_tokenizer():
155
    return get_tokenizer(MISTRAL_MODEL_ID)
156
157


158
159
@pytest.fixture(scope="module")
def image_url():
160
    image = ImageAsset("cherry_blossom")
161
162
163
164
    base64 = encode_image_base64(image.pil_image)
    return f"data:image/jpeg;base64,{base64}"


165
166
@pytest.fixture(scope="module")
def video_url():
167
    video = VideoAsset("baby_reading", 1)
168
169
170
171
172
173
    base64 = encode_video_base64(video.np_ndarrays)
    return f"data:video/jpeg;base64,{base64}"


@pytest.fixture(scope="module")
def audio_url():
174
    audio = AudioAsset("mary_had_lamb")
175
176
177
178
    base64 = encode_audio_base64(*audio.audio_and_sample_rate)
    return f"data:audio/ogg;base64,{base64}"


179
def _assert_mm_data_is_image_input(
180
    mm_data: MultiModalDataDict | None,
181
    image_count: int,
182
    skipped_image_indices: list | None = None,
183
184
185
186
187
188
189
) -> None:
    assert mm_data is not None
    assert set(mm_data.keys()) == {"image"}

    image_data = mm_data.get("image")
    assert image_data is not None

190
    assert isinstance(image_data, list) and len(image_data) == image_count
191
192
193
    if skipped_image_indices is not None:
        for i in skipped_image_indices:
            assert image_data[i] is None
194
195


196
def _assert_mm_uuids(
197
    mm_uuids: MultiModalUUIDDict | None,
198
    media_count: int,
199
    expected_uuids: list[str | None],
200
201
202
203
204
205
206
207
208
    modality: str = "image",
) -> None:
    if len(expected_uuids) > 0:
        assert mm_uuids is not None
        assert modality in mm_uuids

        image_uuids = mm_uuids.get(modality)
        assert image_uuids is not None

209
        assert isinstance(image_uuids, list) and len(image_uuids) == media_count
210
211
212
213
214
215

        assert image_uuids == expected_uuids
    else:
        assert mm_uuids is None


216
217
218
219
220
ModalityType = Literal["image", "video", "audio"]
MultiModalDataCounts = Mapping[ModalityType, int]


def _assert_mm_data_inputs(
221
    mm_data: MultiModalDataDict | None,
222
    data_count: MultiModalDataCounts,
223
    skipped_media_indices: dict[str, list] | None = None,  # modality -> list[int]
224
225
226
227
228
229
230
231
232
) -> None:
    assert mm_data is not None
    assert set(data_count.keys()) == (set(mm_data.keys()))

    for modality, n in data_count.items():
        modality_data = mm_data.get(modality)
        assert modality_data is not None
        assert isinstance(modality_data, list) and len(modality_data) == n

233
        if skipped_media_indices is not None:
234
            skipped_media_indices_for_modality = skipped_media_indices.get(modality)
235
236
237
238
            assert skipped_media_indices_for_modality is not None
            for i in skipped_media_indices_for_modality:
                assert modality_data[i] is None

239

240
241
242
243
244
def test_parse_chat_messages_single_image(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
245
    conversation, mm_data, mm_uuids = parse_chat_messages(
246
247
248
249
250
251
252
253
254
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
255
256
257
258
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
259

260
261
262
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
263
    _assert_mm_data_is_image_input(mm_data, 1)
264
265
266
267
268
269
270
271
272
273
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])


def test_parse_chat_messages_single_image_with_uuid(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
274
275
276
277
278
279
280
281
282
283
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid,
284
                    },
285
286
287
288
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
289
290
291
292
293
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

294
295
296
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
297
298
299
300
    _assert_mm_data_is_image_input(mm_data, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


301
302
303
304
305
306
307
def test_parse_chat_messages_single_empty_image_with_uuid(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
308
309
310
311
312
313
314
315
316
317
318
319
320
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
321
322
323
324
325
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

326
327
328
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
329
330
331
332
    _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


333
334
335
336
337
338
339
def test_parse_chat_messages_single_image_with_bad_uuid_format(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
340
341
342
343
344
345
346
347
348
349
350
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                            "uuid": image_uuid,
                        },
                        "bad_uuid_key": image_uuid,
351
                    },
352
353
354
355
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
356
357
358
359
360
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

361
362
363
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
364
365
366
367
368
369
370
371
372
373
374
375
376
    _assert_mm_data_is_image_input(mm_data, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])


def test_parse_chat_messages_multiple_images_with_uuids(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
377
378
379
380
381
382
383
384
385
386
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid1,
387
                    },
388
389
390
391
392
393
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid2,
394
                    },
395
396
397
398
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
399
400
401
402
403
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

404
405
406
407
408
409
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
410
411
412
413
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


414
415
416
417
418
419
420
421
422
def test_parse_chat_messages_multiple_empty_images_with_uuids(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid1,
                    },
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
441
442
443
444
445
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

446
447
448
449
450
451
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
452
453
454
455
456
457
458
459
460
461
462
463
464
    _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[0, 1])
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


def test_parse_chat_messages_mixed_empty_images_with_uuids(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
465
466
467
468
469
470
471
472
473
474
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid1,
475
                    },
476
477
478
479
480
481
482
483
484
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
485
486
487
488
489
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

490
491
492
493
494
495
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
496
497
498
499
    _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[1])
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


500
501
502
503
504
505
506
507
@pytest.mark.asyncio
async def test_parse_chat_messages_single_image_with_uuid_async(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
508
509
510
511
512
513
514
515
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
516
                    },
517
518
519
520
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
521
522
523
524
525
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

526
527
528
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
529
530
531
532
    _assert_mm_data_is_image_input(await mm_future, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


533
534
535
536
537
538
539
540
@pytest.mark.asyncio
async def test_parse_chat_messages_empty_image_with_uuid_async(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
541
542
543
544
545
546
547
548
549
550
551
552
553
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
554
555
556
557
558
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

559
560
561
562
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
    _assert_mm_data_is_image_input(await mm_future, 1, skipped_image_indices=[0])
563
564
565
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


566
567
568
569
570
571
572
573
574
575
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_uuids_async(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
576
577
578
579
580
581
582
583
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid1,
584
                    },
585
586
587
588
589
590
591
592
593
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
594
595
596
597
598
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

599
600
601
602
603
604
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
605
606
607
608
    _assert_mm_data_is_image_input(await mm_future, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


609
610
611
612
613
614
615
616
617
618
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid1,
                    },
                    {
                        "type": "image_pil",
                        "image_pil": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
637
638
639
640
641
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

642
643
644
645
646
647
648
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
    _assert_mm_data_is_image_input(await mm_future, 2, skipped_image_indices=[0, 1])
649
650
651
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


652
653
654
655
656
657
658
659
660
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
661
662
663
664
665
666
667
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
668
                    },
669
670
671
672
673
674
675
676
677
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
678
679
680
681
682
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

683
684
685
686
687
688
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
689
690
    _assert_mm_data_is_image_input(await mm_future, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, image_uuid2])
691
692


693
694
695
696
697
def test_parse_chat_messages_empty_system(
    mistral_model_config,
    mistral_tokenizer,
):
    # Test string format
698
    conversation, _, _ = parse_chat_messages(
699
        [
700
            {"role": "system", "content": ""},
701
702
            {
                "role": "user",
703
                "content": [{"type": "text", "text": "Who are you?"}],
704
705
            },
        ],
706
707
708
709
        mistral_model_config,
        mistral_tokenizer,
        content_format="string",
    )
710
    assert conversation == [
711
712
        {"role": "system", "content": ""},
        {"role": "user", "content": "Who are you?"},
713
    ]
714
715

    # Test openai format
716
    conversation, _, _ = parse_chat_messages(
717
        [
718
            {"role": "system", "content": ""},
719
720
            {
                "role": "user",
721
                "content": [{"type": "text", "text": "Who are you?"}],
722
723
724
725
726
727
728
            },
        ],
        mistral_model_config,
        mistral_tokenizer,
        content_format="openai",
    )
    assert conversation == [
729
730
        {"role": "system", "content": [{"type": "text", "text": ""}]},
        {"role": "user", "content": [{"type": "text", "text": "Who are you?"}]},
731
    ]
732
733


734
@pytest.mark.asyncio
735
736
737
738
739
async def test_parse_chat_messages_single_image_async(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
740
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
741
742
743
744
745
746
747
748
749
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
750
751
752
753
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
754

755
756
757
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
758
    _assert_mm_data_is_image_input(await mm_future, 1)
759
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
760
761
762
763
764
765
766


def test_parse_chat_messages_multiple_images(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
767
    conversation, mm_data, mm_uuids = parse_chat_messages(
768
769
770
771
772
773
774
775
776
777
778
779
780
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
781
782
783
784
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
785

786
787
788
789
790
791
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
792
    _assert_mm_data_is_image_input(mm_data, 2)
793
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
794
795


796
797
798
799
800
801
def test_parse_chat_messages_empty_pil_image_with_uuid(
    phi3v_model_config,
    phi3v_tokenizer,
):
    uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
802
803
804
805
806
807
808
809
810
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_pil", "image_pil": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
811
812
813
814
815
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

816
817
818
819
820
821
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
822
823
824
825
826
    _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


def test_parse_chat_messages_empty_image_embeds_with_uuid(
827
    phi3v_model_config_image_embeds,
828
829
830
831
    phi3v_tokenizer,
):
    uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
832
833
834
835
836
837
838
839
840
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_embeds", "image_embeds": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
841
        phi3v_model_config_image_embeds,
842
843
844
845
        phi3v_tokenizer,
        content_format="string",
    )

846
847
848
849
850
851
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
852
853
854
855
856
857
    assert mm_data is not None
    assert "image" in mm_data
    assert mm_data["image"] is None
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
def test_parse_chat_messages_empty_audio_embeds_with_uuid(
    audio_embeds_model_config,
    qwen2_audio_tokenizer,
):
    """Test audio_embeds with UUID (no actual embeds data)."""
    uuid = "test-audio-uuid-123"

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {"type": "audio_embeds", "audio_embeds": None, "uuid": uuid},
                ],
            }
        ],
        audio_embeds_model_config,
        qwen2_audio_tokenizer,
        content_format="string",
    )

    # Should have audio in mm_data as None (UUID provided)
    assert mm_data is not None
    assert "audio" in mm_data
    assert mm_data["audio"] is None
    # UUID should be recorded
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[uuid])


def test_parse_chat_messages_audio_embeds_with_string(
    audio_embeds_model_config,
    qwen2_audio_tokenizer,
):
    """Test audio_embeds with base64 string embedding data."""
    import base64
    import io

    import torch

    # Create a sample audio embedding tensor
    audio_embedding = torch.randn(1, 128, 768)

    # Encode it as base64
    buffer = io.BytesIO()
    torch.save(audio_embedding, buffer)
    buffer.seek(0)
    binary_data = buffer.read()
    base64_audio_embedding = base64.b64encode(binary_data).decode("utf-8")

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {
                        "type": "audio_embeds",
                        "audio_embeds": base64_audio_embedding,
                    },
                ],
            }
        ],
        audio_embeds_model_config,
        qwen2_audio_tokenizer,
        content_format="string",
    )

    # Should have audio embedding in mm_data (single tensor, not a list)
    assert mm_data is not None
    assert "audio" in mm_data
    assert isinstance(mm_data["audio"], torch.Tensor)
    assert mm_data["audio"].shape == audio_embedding.shape
    # No UUID provided
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


@pytest.mark.asyncio
async def test_parse_chat_messages_audio_embeds_async(
    audio_embeds_model_config,
    qwen2_audio_tokenizer,
):
    """Test audio_embeds with async futures."""
    import base64
    import io

    import torch

    # Create a sample audio embedding tensor
    audio_embedding = torch.randn(1, 128, 768)

    # Encode it as base64
    buffer = io.BytesIO()
    torch.save(audio_embedding, buffer)
    buffer.seek(0)
    binary_data = buffer.read()
    base64_audio_embedding = base64.b64encode(binary_data).decode("utf-8")

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {
                        "type": "audio_embeds",
                        "audio_embeds": base64_audio_embedding,
                    },
                ],
            }
        ],
        audio_embeds_model_config,
        qwen2_audio_tokenizer,
        content_format="string",
    )

    # Should have audio embedding in mm_data (single tensor, not a list)
    mm_data = await mm_future
    assert mm_data is not None
    assert "audio" in mm_data
    assert isinstance(mm_data["audio"], torch.Tensor)
    assert mm_data["audio"].shape == audio_embedding.shape
    # No UUID provided
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


990
991
@pytest.mark.asyncio
async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
992
    phi3v_model_config_image_embeds,
993
994
995
996
    phi3v_tokenizer,
):
    uuid = "abcd"
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
997
998
999
1000
1001
1002
1003
1004
1005
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_embeds", "image_embeds": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
1006
        phi3v_model_config_image_embeds,
1007
1008
1009
1010
        phi3v_tokenizer,
        content_format="string",
    )

1011
1012
1013
1014
1015
1016
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
1017
1018
1019
1020
1021
1022
1023
    mm_data = await mm_future
    assert mm_data is not None
    assert "image" in mm_data
    assert mm_data["image"] is None
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


1024
@pytest.mark.asyncio
1025
1026
1027
1028
1029
async def test_parse_chat_messages_multiple_images_async(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
1030
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
1044
1045
1046
1047
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
1048

1049
1050
1051
1052
1053
1054
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
1055
    _assert_mm_data_is_image_input(await mm_future, 2)
1056
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1057
1058
1059
1060
1061
1062
1063


def test_parse_chat_messages_placeholder_already_in_prompt(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
1064
    conversation, mm_data, mm_uuids = parse_chat_messages(
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "text",
                        "text": "What's in <|image_1|> and how does it compare to <|image_2|>?",  # noqa: E501
                    },
                ],
            }
        ],
1078
1079
1080
1081
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
1082
1083
1084
1085
1086
1087
    assert conversation == [
        {
            "role": "user",
            "content": "What's in <|image_1|> and how does it compare to <|image_2|>?",
        }
    ]
1088
    _assert_mm_data_is_image_input(mm_data, 2)
1089
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1090
1091


1092
1093
1094
1095
1096
def test_parse_chat_messages_placeholder_one_already_in_prompt(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
1097
    conversation, mm_data, mm_uuids = parse_chat_messages(
1098
1099
1100
1101
1102
1103
1104
1105
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "text",
1106
1107
                        "text": "What's in <|image_1|> and how does it compare to "
                        "the other one?",
1108
1109
1110
1111
                    },
                ],
            }
        ],
1112
1113
1114
1115
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
1116

1117
1118
1119
    assert conversation == [
        {
            "role": "user",
1120
1121
            "content": "<|image_2|>\nWhat's in <|image_1|> and how does it compare to "
            "the other one?",
1122
1123
        }
    ]
1124
    _assert_mm_data_is_image_input(mm_data, 2)
1125
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1126
1127


1128
1129
1130
1131
1132
def test_parse_chat_messages_multiple_images_across_messages(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
1133
    conversation, mm_data, mm_uuids = parse_chat_messages(
1134
1135
        [
            {
1136
                "role": "user",
1137
                "content": [
1138
1139
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in this image?"},
1140
1141
                ],
            },
1142
            {"role": "assistant", "content": "Some stuff."},
1143
            {
1144
                "role": "user",
1145
                "content": [
1146
1147
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What about this one?"},
1148
1149
1150
                ],
            },
        ],
1151
1152
1153
1154
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
1155
1156

    assert conversation == [
1157
1158
1159
        {"role": "user", "content": "<|image_1|>\nWhat's in this image?"},
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "<|image_2|>\nWhat about this one?"},
1160
    ]
1161
    _assert_mm_data_is_image_input(mm_data, 2)
1162
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1163
1164


1165
def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
1166
1167
    phi3v_model_config,
    phi3v_tokenizer,
1168
    image_url,
1169
):
1170
1171
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
1172
1173
        [
            {
1174
                "role": "user",
1175
1176
1177
                "content": [
                    {
                        "type": "image_url",
1178
                        "image_url": {"url": image_url},
1179
1180
                        "uuid": image_uuid,
                    },
1181
                    {"type": "text", "text": "What's in this image?"},
1182
                ],
1183
            },
1184
            {"role": "assistant", "content": "Some stuff."},
1185
            {
1186
                "role": "user",
1187
1188
1189
                "content": [
                    {
                        "type": "image_url",
1190
                        "image_url": {"url": image_url},
1191
1192
                        "uuid": image_uuid,
                    },
1193
                    {"type": "text", "text": "What about this one?"},
1194
1195
1196
1197
1198
1199
1200
1201
1202
                ],
            },
        ],
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )

    assert conversation == [
1203
1204
1205
        {"role": "user", "content": "<|image_1|>\nWhat's in this image?"},
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "<|image_2|>\nWhat about this one?"},
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
    ]
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])


def test_parse_chat_messages_context_text_format(
    phi3v_model_config,
    phi3v_tokenizer,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
1219
                "content": [{"type": "text", "text": "What's in this text?"}],
1220
            },
1221
1222
            {"role": "assistant", "content": "Some stuff."},
            {"role": "user", "content": "What about this one?"},
1223
        ],
1224
1225
1226
1227
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="openai",
    )
1228
1229
1230
1231

    assert conversation == [
        {
            "role": "user",
1232
            "content": [{"type": "text", "text": "What's in this text?"}],
1233
1234
1235
        },
        {
            "role": "assistant",
1236
            "content": [{"type": "text", "text": "Some stuff."}],
1237
1238
1239
        },
        {
            "role": "user",
1240
            "content": [{"type": "text", "text": "What about this one?"}],
1241
1242
        },
    ]
1243
1244
    assert mm_data is None
    assert mm_uuids is None
1245
1246


1247
1248
1249
1250
1251
def test_parse_chat_messages_rejects_too_many_images_in_one_message(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
1252
1253
1254
    with warnings.catch_warnings():
        warnings.filterwarnings(
            "ignore",
1255
1256
            message="coroutine 'async_get_and_parse_image' was never awaited",
        )
1257
        with pytest.raises(ValueError, match="At most"):
1258
            parse_chat_messages(
1259
1260
1261
1262
1263
1264
1265
                [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1266
                            },
1267
1268
1269
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1270
                            },
1271
1272
1273
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1274
                            },
1275
1276
1277
1278
                            {"type": "text", "text": "What's in these images?"},
                        ],
                    }
                ],
1279
1280
1281
1282
                phi3v_model_config,
                phi3v_tokenizer,
                content_format="string",
            )
1283
1284


1285
1286
1287
1288
1289
def test_parse_chat_messages_rejects_too_many_images_across_messages(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
1290
1291
1292
    with warnings.catch_warnings():
        warnings.filterwarnings(
            "ignore",
1293
1294
            message="coroutine 'async_get_and_parse_image' was never awaited",
        )
1295
        with pytest.raises(ValueError, match="At most"):
1296
            parse_chat_messages(
1297
1298
                [
                    {
1299
                        "role": "user",
1300
1301
1302
                        "content": [
                            {
                                "type": "image_url",
1303
                                "image_url": {"url": image_url},
1304
                            },
1305
                            {"type": "text", "text": "What's in this image?"},
1306
1307
                        ],
                    },
1308
                    {"role": "assistant", "content": "Some stuff."},
1309
                    {
1310
                        "role": "user",
1311
1312
1313
                        "content": [
                            {
                                "type": "image_url",
1314
                                "image_url": {"url": image_url},
1315
1316
1317
                            },
                            {
                                "type": "image_url",
1318
                                "image_url": {"url": image_url},
1319
                            },
1320
                            {"type": "text", "text": "What about these two?"},
1321
1322
1323
                        ],
                    },
                ],
1324
1325
1326
1327
                phi3v_model_config,
                phi3v_tokenizer,
                content_format="string",
            )
1328
1329
1330
1331
1332
1333
1334


def test_parse_chat_messages_multiple_images_uncommon_input(
    phi3v_model_config,
    phi3v_tokenizer,
    image_url,
):
1335
    conversation, mm_data, mm_uuids = parse_chat_messages(
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
        [
            {
                "role": "user",
                "content": [
                    "What's in these images?",
                    {"image_url": image_url},
                    {"image_url": image_url},
                ],
            }
        ],
1346
1347
1348
1349
        phi3v_model_config,
        phi3v_tokenizer,
        content_format="string",
    )
1350

1351
1352
1353
1354
1355
1356
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
1357
    _assert_mm_data_is_image_input(mm_data, 2)
1358
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1359
1360


1361
1362
1363
1364
1365
def test_parse_chat_messages_multiple_images_interleave(
    phi3v_model_config_mm_interleaved,
    phi3v_tokenizer,
    image_url,
):
1366
    conversation, mm_data, mm_uuids = parse_chat_messages(
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
                    },
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "and this one"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1382
1383
1384
1385
1386
        phi3v_model_config_mm_interleaved,
        phi3v_tokenizer,
        content_format="string",
    )

1387
1388
1389
1390
1391
1392
1393
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1394
    _assert_mm_data_is_image_input(mm_data, 2)
1395
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1396
1397
1398
1399
1400
1401
1402
1403


@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_interleave_async(
    phi3v_model_config_mm_interleaved,
    phi3v_tokenizer,
    image_url,
):
1404
    conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
                    },
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "and this one"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1420
1421
1422
1423
1424
        phi3v_model_config_mm_interleaved,
        phi3v_tokenizer,
        content_format="string",
    )

1425
1426
1427
1428
1429
1430
1431
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1432
    _assert_mm_data_is_image_input(await mm_data, 2)
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
    phi3v_model_config_mm_interleaved,
    phi3v_tokenizer,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1444
1445
1446
1447
1448
1449
1450
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
1451
                    },
1452
1453
1454
1455
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
1456
                    },
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
                    {"type": "text", "text": "and this one"},
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1467
1468
1469
1470
1471
        phi3v_model_config_mm_interleaved,
        phi3v_tokenizer,
        content_format="string",
    )

1472
1473
1474
1475
1476
1477
1478
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1479
1480
    _assert_mm_data_is_image_input(await mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1481
1482
1483
1484
1485
1486
1487


def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
    phi3v_model_config_mm_interleaved,
    phi3v_tokenizer,
    image_url,
):
1488
    conversation, mm_data, mm_uuids = parse_chat_messages(
1489
1490
        [
            {
1491
                "role": "user",
1492
                "content": [
1493
1494
1495
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Be accurate."},
1496
1497
                ],
            },
1498
            {"role": "assistant", "content": "Some stuff."},
1499
            {
1500
                "role": "user",
1501
                "content": [
1502
1503
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
1504
1505
1506
                ],
            },
        ],
1507
1508
1509
1510
1511
        phi3v_model_config_mm_interleaved,
        phi3v_tokenizer,
        content_format="string",
    )

1512
1513
1514
1515
1516
    assert conversation == [
        {
            "role": "user",
            "content": "What's on this image?\n<|image_1|>\nBe accurate.",
        },
1517
1518
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "What's on this image?\n<|image_2|>"},
1519
    ]
1520
    _assert_mm_data_is_image_input(mm_data, 2)
1521
1522
1523
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


1524
def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave(
1525
1526
1527
1528
1529
1530
1531
1532
    phi3v_model_config_mm_interleaved,
    phi3v_tokenizer,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1533
                "role": "user",
1534
                "content": [
1535
                    {"type": "text", "text": "What's on this image?"},
1536
1537
                    {
                        "type": "image_url",
1538
                        "image_url": {"url": image_url},
1539
1540
                        "uuid": image_uuid,
                    },
1541
                    {"type": "text", "text": "Be accurate."},
1542
1543
                ],
            },
1544
            {"role": "assistant", "content": "Some stuff."},
1545
            {
1546
                "role": "user",
1547
                "content": [
1548
                    {"type": "text", "text": "What's on this image?"},
1549
1550
                    {
                        "type": "image_url",
1551
                        "image_url": {"url": image_url},
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
                        "uuid": image_uuid,
                    },
                ],
            },
        ],
        phi3v_model_config_mm_interleaved,
        phi3v_tokenizer,
        content_format="string",
    )

    assert conversation == [
        {
            "role": "user",
            "content": "What's on this image?\n<|image_1|>\nBe accurate.",
        },
1567
1568
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "What's on this image?\n<|image_2|>"},
1569
1570
1571
    ]
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1572
1573
1574


def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1575
1576
1577
1578
1579
1580
    qwen25omni_model_config_mm_interleaved,
    qwen25omni_tokenizer,
    image_url,
    video_url,
    audio_url,
):
1581
    conversation, mm_data, mm_uuids = parse_chat_messages(
1582
1583
        [
            {
1584
                "role": "user",
1585
                "content": [
1586
1587
1588
1589
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Now listen to this audio"},
                    {"type": "audio_url", "audio_url": {"url": audio_url}},
1590
1591
                ],
            },
1592
            {"role": "assistant", "content": "Some stuff."},
1593
            {
1594
                "role": "user",
1595
                "content": [
1596
1597
1598
1599
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "And what's in the video?"},
                    {"type": "video_url", "video_url": {"url": video_url}},
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        qwen25omni_tokenizer,
        content_format="string",
    )

    assert conversation == [
        {
1610
            "role": "user",
1611
1612
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1613
        },
1614
        {"role": "assistant", "content": "Some stuff."},
1615
        {
1616
            "role": "user",
1617
1618
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1619
1620
        },
    ]
1621
1622

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1623
    _assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=[None, None])
1624
1625
1626
1627
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


1628
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave(
1629
1630
1631
1632
1633
1634
1635
1636
1637
    qwen25omni_model_config_mm_interleaved,
    qwen25omni_tokenizer,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1638
                "role": "user",
1639
                "content": [
1640
                    {"type": "text", "text": "What's on this image?"},
1641
1642
                    {
                        "type": "image_url",
1643
                        "image_url": {"url": image_url},
1644
1645
                        "uuid": "image_123",
                    },
1646
                    {"type": "text", "text": "Now listen to this audio"},
1647
1648
                    {
                        "type": "audio_url",
1649
                        "audio_url": {"url": audio_url},
1650
1651
1652
1653
                        "uuid": "audio_123",
                    },
                ],
            },
1654
            {"role": "assistant", "content": "Some stuff."},
1655
            {
1656
                "role": "user",
1657
                "content": [
1658
                    {"type": "text", "text": "What's on this image?"},
1659
1660
                    {
                        "type": "image_url",
1661
                        "image_url": {"url": image_url},
1662
1663
                        "uuid": "image_123",
                    },
1664
                    {"type": "text", "text": "And what's in the video?"},
1665
1666
                    {
                        "type": "video_url",
1667
                        "video_url": {"url": video_url},
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
                        "uuid": "video_123",
                    },
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        qwen25omni_tokenizer,
        content_format="string",
    )

    assert conversation == [
        {
1680
            "role": "user",
1681
1682
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1683
        },
1684
        {"role": "assistant", "content": "Some stuff."},
1685
        {
1686
            "role": "user",
1687
1688
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1689
1690
1691
1692
        },
    ]

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1693
1694
1695
1696
1697
    _assert_mm_uuids(
        mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
    )
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
1698
1699


1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave(  # noqa: E501
    qwen25omni_model_config_mm_interleaved,
    qwen25omni_tokenizer,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1710
                "role": "user",
1711
                "content": [
1712
                    {"type": "text", "text": "What's on this image?"},
1713
1714
1715
1716
1717
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": "image_123",
                    },
1718
                    {"type": "text", "text": "Now listen to this audio"},
1719
1720
1721
1722
1723
1724
1725
                    {
                        "type": "audio_url",
                        "audio_url": None,
                        "uuid": "audio_123",
                    },
                ],
            },
1726
            {"role": "assistant", "content": "Some stuff."},
1727
            {
1728
                "role": "user",
1729
                "content": [
1730
                    {"type": "text", "text": "What's on this image?"},
1731
1732
1733
1734
1735
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": "image_123",
                    },
1736
                    {"type": "text", "text": "And what's in the video?"},
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
                    {
                        "type": "video_url",
                        "video_url": None,
                        "uuid": "video_123",
                    },
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        qwen25omni_tokenizer,
        content_format="string",
    )

    assert conversation == [
        {
1752
            "role": "user",
1753
1754
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1755
        },
1756
        {"role": "assistant", "content": "Some stuff."},
1757
        {
1758
            "role": "user",
1759
1760
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1761
1762
1763
        },
    ]

1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
    _assert_mm_data_inputs(
        mm_data,
        {"image": 2, "video": 1, "audio": 1},
        skipped_media_indices={"image": [0, 1], "video": [0], "audio": [0]},
    )
    _assert_mm_uuids(
        mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
    )
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
1774
1775


1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave(  # noqa: E501
    qwen25omni_model_config_mm_interleaved,
    qwen25omni_tokenizer,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1786
                "role": "user",
1787
                "content": [
1788
                    {"type": "text", "text": "What's on this image?"},
1789
1790
                    {
                        "type": "image_url",
1791
                        "image_url": {"url": image_url},
1792
1793
                        "uuid": "image_123",
                    },
1794
1795
                    {"type": "text", "text": "Now listen to this audio"},
                    {"type": "audio_url", "audio_url": {"url": audio_url}},
1796
1797
                ],
            },
1798
            {"role": "assistant", "content": "Some stuff."},
1799
            {
1800
                "role": "user",
1801
                "content": [
1802
1803
1804
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "And what's in the video?"},
1805
1806
                    {
                        "type": "video_url",
1807
                        "video_url": {"url": video_url},
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
                        "uuid": "video_123",
                    },
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        qwen25omni_tokenizer,
        content_format="string",
    )

    assert conversation == [
        {
1820
            "role": "user",
1821
1822
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1823
        },
1824
        {"role": "assistant", "content": "Some stuff."},
1825
        {
1826
            "role": "user",
1827
1828
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1829
1830
1831
1832
        },
    ]

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1833
1834
    _assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=["image_123", None])
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
1835
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
1836
1837
1838
1839
1840
1841
1842
1843


def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
    phi3v_model_config_mm_interleaved,
    phi3v_tokenizer,
    image_url,
):
    with pytest.raises(
1844
1845
1846
        ValueError,
        match=r"Found more '<|image_1|>' placeholders in input prompt "
        "than actual multimodal data items.",
1847
    ):
1848
        parse_chat_messages(
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
            [
                {
                    "role": "user",
                    "content": [
                        {"type": "image_url", "image_url": {"url": image_url}},
                        {"type": "image_url", "image_url": {"url": image_url}},
                        {
                            "type": "text",
                            "text": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
                            "Do they have differences?",
                        },
                    ],
                }
            ],
1863
1864
1865
1866
1867
1868
            phi3v_model_config_mm_interleaved,
            phi3v_tokenizer,
            content_format="string",
        )


1869
1870
1871
1872
1873
@pytest.mark.parametrize(
    "model",
    [
        QWEN2VL_MODEL_ID,  # tokenizer.chat_template is of type str
        HERMES_MODEL_ID,  # tokenizer.chat_template is of type dict
1874
1875
    ],
)
1876
1877
1878
@pytest.mark.parametrize("use_tools", [True, False])
def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
    """checks that chat_template is a dict type for HF models."""
1879
1880
1881
1882
1883
1884
1885
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
1886
        revision=model_info.revision,
1887
1888
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
1889
1890
1891
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
1892
        enforce_eager=model_info.enforce_eager,
1893
1894
        dtype=model_info.dtype,
    )
1895

1896
1897
    # Build the tokenizer
    tokenizer = get_tokenizer(
1898
        model,
1899
        trust_remote_code=model_config.trust_remote_code,
1900
1901
    )

1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
    tools = (
        [
            {
                "type": "function",
                "function": {
                    "name": "dummy_function_name",
                    "description": "This is a dummy function",
                    "parameters": sample_json_schema,
                },
            }
        ]
        if use_tools
        else None
    )
1916
1917

    # Test detecting the tokenizer's chat_template
1918
    chat_template = resolve_hf_chat_template(
1919
1920
1921
        tokenizer,
        chat_template=None,
        tools=tools,
1922
        model_config=model_config,
1923
1924
1925
1926
    )
    assert isinstance(chat_template, str)


1927
1928
1929
1930
1931
1932
@pytest.mark.parametrize(
    "model, expected_kwargs",
    [
        (
            QWEN2VL_MODEL_ID,
            {
1933
1934
1935
1936
                "add_vision_id",
                "add_generation_prompt",
                "continue_final_message",
                "tools",
1937
1938
1939
1940
1941
            },
        ),
        (
            QWEN3_MODEL_ID,
            {
1942
1943
1944
1945
                "enable_thinking",
                "add_generation_prompt",
                "continue_final_message",
                "tools",
1946
1947
1948
1949
            },
        ),
    ],
)
1950
def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwargs):
1951
1952
1953
1954
    """checks that chat_template is a dict type for HF models."""
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
    tools = [
        {
            "type": "function",
            "function": {
                "name": "dummy_function_name",
                "description": "This is a dummy function",
                "parameters": sample_json_schema,
            },
        }
    ]
1965
1966
1967
1968
1969
1970
1971

    chat_template_kwargs = {
        # both unused
        "unsed_kwargs_1": 123,
        "unsed_kwargs_2": "abc",
        # should not appear
        "chat_template": "{% Hello world! %}",
1972
        "tokenize": True,
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
        # used by tokenizer
        "continue_final_message": True,
        "tools": tools,
        # both used by Qwen2-VL and Qwen3
        "add_generation_prompt": True,
        # only used by Qwen2-VL
        "add_vision_id": True,
        # only used by Qwen3
        "enable_thinking": True,
    }

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
1991
1992
1993
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
1994
        enforce_eager=model_info.enforce_eager,
1995
1996
        dtype=model_info.dtype,
    )
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010

    # Build the tokenizer
    tokenizer = get_tokenizer(
        model,
        trust_remote_code=model_config.trust_remote_code,
    )

    # Test detecting the tokenizer's chat_template
    chat_template = resolve_hf_chat_template(
        tokenizer,
        chat_template=None,
        tools=tools,
        model_config=model_config,
    )
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
    with pytest.raises(
        ValueError, match="Found unexpected chat template kwargs from request"
    ):
        # should raise error if `chat_template_kwargs` contains
        # `chat_template` or `tokenize`
        resolve_chat_template_kwargs(
            tokenizer,
            chat_template=chat_template,
            chat_template_kwargs=chat_template_kwargs,
        )
2021
2022
2023
2024
    resolved_chat_template_kwargs = resolve_chat_template_kwargs(
        tokenizer,
        chat_template=chat_template,
        chat_template_kwargs=chat_template_kwargs,
2025
        raise_on_unexpected=False,
2026
2027
2028
    )
    assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs

2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
    # Additional test: Verify HF base parameters work with **kwargs tokenizers
    # This validates the fix for tokenizers like Kimi K2 that use **kwargs
    # to receive standard HuggingFace parameters instead of declaring them explicitly
    from vllm.entrypoints.chat_utils import _get_hf_base_chat_template_params

    hf_base_params = _get_hf_base_chat_template_params()
    # Verify common HF parameters are in the base class
    assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset(
        hf_base_params
    ), f"Expected HF base params not found in {hf_base_params}"

    # Test with a mock tokenizer that uses **kwargs (like Kimi K2)
    class MockTokenizerWithKwargs:
        def apply_chat_template(self, conversation, **kwargs):
            return "mocked_output"

    mock_tokenizer = MockTokenizerWithKwargs()
    mock_kwargs = {
        "add_generation_prompt": True,
        "tools": tools,
        "continue_final_message": False,
        "unknown_param": "should_be_filtered",
    }
    resolved_mock = resolve_chat_template_kwargs(
        mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False
    )
    # HF base params should pass through even with **kwargs tokenizer
    assert "add_generation_prompt" in resolved_mock
    assert "tools" in resolved_mock
    assert "continue_final_message" in resolved_mock
    # Unknown params should be filtered out
    assert "unknown_param" not in resolved_mock

2062

2063
2064
# NOTE: Qwen2-Audio default chat template is specially defined inside
# processor class instead of using `tokenizer_config.json`
2065
2066
@pytest.mark.parametrize(
    ("model", "expected_format"),
2067
2068
2069
2070
2071
2072
2073
2074
    [
        (PHI3V_MODEL_ID, "string"),
        (QWEN2VL_MODEL_ID, "openai"),
        (QWEN25VL_MODEL_ID, "openai"),
        (ULTRAVOX_MODEL_ID, "string"),
        (QWEN2AUDIO_MODEL_ID, "openai"),
        (LLAMA_GUARD_MODEL_ID, "openai"),
    ],
2075
2076
)
def test_resolve_content_format_hf_defined(model, expected_format):
2077
2078
2079
2080
2081
2082
2083
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
2084
        revision=model_info.revision,
2085
2086
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
2087
2088
2089
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
2090
        enforce_eager=model_info.enforce_eager,
2091
2092
        dtype=model_info.dtype,
    )
2093

2094
    tokenizer = get_tokenizer(
2095
        model,
2096
        trust_remote_code=model_config.trust_remote_code,
2097
2098
    )

2099
    # Test detecting the tokenizer's chat_template
2100
    chat_template = resolve_hf_chat_template(
2101
2102
2103
        tokenizer,
        chat_template=None,
        tools=None,
2104
        model_config=model_config,
2105
    )
2106
2107
2108
2109
2110
2111
2112
2113
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
2114
2115
2116
2117
        None,  # Test detecting the tokenizer's chat_template
        None,
        "auto",
        tokenizer,
2118
        model_config=model_config,
2119
2120
2121
2122
2123
2124
2125
    )

    assert resolved_format == expected_format


@pytest.mark.parametrize(
    ("model", "expected_format"),
2126
2127
2128
2129
2130
2131
2132
2133
2134
    [
        ("Salesforce/blip2-opt-2.7b", "string"),
        ("facebook/chameleon-7b", "string"),
        ("deepseek-ai/deepseek-vl2-tiny", "string"),
        ("adept/fuyu-8b", "string"),
        ("google/paligemma-3b-mix-224", "string"),
        ("Qwen/Qwen-VL", "string"),
        ("Qwen/Qwen-VL-Chat", "string"),
    ],
2135
2136
2137
2138
2139
2140
2141
2142
2143
)
def test_resolve_content_format_fallbacks(model, expected_format):
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
2144
        revision=model_info.revision,
2145
2146
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
2147
2148
2149
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
2150
        enforce_eager=model_info.enforce_eager,
2151
2152
        dtype=model_info.dtype,
    )
2153

2154
    tokenizer = get_tokenizer(
2155
2156
2157
2158
2159
2160
2161
2162
2163
        model_config.tokenizer,
        trust_remote_code=model_config.trust_remote_code,
    )

    # Test detecting the tokenizer's chat_template
    chat_template = resolve_hf_chat_template(
        tokenizer,
        chat_template=None,
        tools=None,
2164
        model_config=model_config,
2165
2166
2167
2168
2169
2170
2171
2172
2173
    )
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
2174
        None,  # Test detecting the tokenizer's chat_template
2175
        None,
2176
2177
        "auto",
        tokenizer,
2178
        model_config=model_config,
2179
2180
2181
2182
2183
2184
2185
    )

    assert resolved_format == expected_format


@pytest.mark.parametrize(
    ("template_path", "expected_format"),
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
    [
        ("template_alpaca.jinja", "string"),
        ("template_baichuan.jinja", "string"),
        ("template_chatglm.jinja", "string"),
        ("template_chatglm2.jinja", "string"),
        ("template_chatml.jinja", "string"),
        ("template_dse_qwen2_vl.jinja", "openai"),
        ("template_falcon_180b.jinja", "string"),
        ("template_falcon.jinja", "string"),
        ("template_inkbot.jinja", "string"),
        ("template_teleflm.jinja", "string"),
        ("template_vlm2vec_phi3v.jinja", "openai"),
        ("template_vlm2vec_qwen2vl.jinja", "openai"),
        ("tool_chat_template_granite_20b_fc.jinja", "string"),
        ("tool_chat_template_hermes.jinja", "string"),
        ("tool_chat_template_internlm2_tool.jinja", "string"),
        ("tool_chat_template_llama3.1_json.jinja", "openai"),
        ("tool_chat_template_llama3.2_json.jinja", "openai"),
        ("tool_chat_template_mistral_parallel.jinja", "string"),
        ("tool_chat_template_mistral.jinja", "string"),
    ],
2207
2208
)
def test_resolve_content_format_examples(template_path, expected_format):
2209
2210
2211
2212
2213
2214
    model_config = ModelConfig(
        PHI3V_MODEL_ID,  # Dummy
        tokenizer=PHI3V_MODEL_ID,  # Dummy
        trust_remote_code=True,
    )

2215
    dummy_tokenizer = get_tokenizer(
2216
2217
        PHI3V_MODEL_ID,  # Dummy
        trust_remote_code=model_config.trust_remote_code,
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
    )
    dummy_tokenizer.chat_template = None

    chat_template = load_chat_template(EXAMPLES_DIR / template_path)
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
        chat_template,
2231
        None,
2232
2233
        "auto",
        dummy_tokenizer,
2234
        model_config=model_config,
2235
2236
2237
    )

    assert resolved_format == expected_format
Julien Denize's avatar
Julien Denize committed
2238
2239


2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
def test_parse_chat_messages_include_thinking_chunk(
    mistral_model_config, mistral_tokenizer
):
    messages = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "thinking",
                    "closed": True,
                    "thinking": "Only return the answer when you are confident.",
                },
            ],
        },
        {"role": "user", "content": "What is 2+2?"},
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
                {
                    "type": "text",
                    "text": "The answer is 4.",
                },
            ],
        },
    ]
Julien Denize's avatar
Julien Denize committed
2268

2269
    conversation_with_thinking, _, _ = parse_chat_messages(
Julien Denize's avatar
Julien Denize committed
2270
2271
2272
2273
2274
2275
        messages,
        mistral_model_config,
        mistral_tokenizer,
        content_format="openai",
    )

2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
    expected_conversation = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "text",
                    "text": "Only return the answer when you are confident.",
                },
            ],
        },
        {
            "role": "user",
            "content": [{"type": "text", "text": "What is 2+2?"}],
        },
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "text", "text": "2+2 = 4"},
                {"type": "text", "text": "The answer is 4."},
            ],
        },
    ]
Julien Denize's avatar
Julien Denize committed
2300
2301
2302
2303
2304

    assert conversation_with_thinking == expected_conversation


def test_apply_mistral_chat_template_thinking_chunk():
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
    messages = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "thinking",
                    "closed": True,
                    "thinking": "Only return the answer when you are confident.",
                },
            ],
        },
        {"role": "user", "content": "What is 2+2?"},
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
                {
                    "type": "text",
                    "text": "The answer is 4.",
                },
            ],
        },
        {"role": "user", "content": "Thanks, what is 3+3?"},
    ]
Julien Denize's avatar
Julien Denize committed
2331
    mistral_tokenizer = MistralTokenizer.from_pretrained(
2332
        "mistralai/Magistral-Small-2509"
2333
    )
Julien Denize's avatar
Julien Denize committed
2334

2335
2336
2337
    tokens_ids = apply_mistral_chat_template(
        mistral_tokenizer, messages, chat_template=None, tools=None
    )
Julien Denize's avatar
Julien Denize committed
2338
2339

    string_tokens = mistral_tokenizer.mistral.decode(
2340
2341
        tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
    )
Julien Denize's avatar
Julien Denize committed
2342
2343
2344
2345
2346
2347

    expected_tokens = (
        r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
        r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
        r"[INST]What is 2+2?[/INST]"
        r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
2348
2349
        r"[INST]Thanks, what is 3+3?[/INST]"
    )
Julien Denize's avatar
Julien Denize committed
2350
2351

    assert string_tokens == expected_tokens
2352
2353
2354
2355
2356
2357
2358
2359


def test_parse_chat_messages_single_empty_audio_with_uuid(
    qwen2_audio_model_config,
    qwen2_audio_tokenizer,
):
    audio_uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "input_audio",
                        "input_audio": {},
                        "uuid": audio_uuid,
                    },
                    {"type": "text", "text": "What does the audio say?"},
                ],
            }
        ],
2373
2374
2375
2376
2377
        qwen2_audio_model_config,
        qwen2_audio_tokenizer,
        content_format="string",
    )

2378
2379
2380
    assert conversation == [
        {
            "role": "user",
2381
2382
            "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
            "audio say?",
2383
2384
        }
    ]
2385
    _assert_mm_data_inputs(mm_data, {"audio": 1})
2386
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
2387
2388
2389
2390
2391
2392
2393
2394
2395


@pytest.mark.asyncio
async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
    qwen2_audio_model_config,
    qwen2_audio_tokenizer,
):
    audio_uuid = "abcd"
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "input_audio",
                        "input_audio": {},
                        "uuid": audio_uuid,
                    },
                    {"type": "text", "text": "What does the audio say?"},
                ],
            }
        ],
2409
2410
2411
2412
2413
        qwen2_audio_model_config,
        qwen2_audio_tokenizer,
        content_format="string",
    )

2414
2415
2416
    assert conversation == [
        {
            "role": "user",
2417
2418
            "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
            "audio say?",
2419
2420
        }
    ]
2421
    _assert_mm_data_inputs(await mm_future, {"audio": 1})
2422
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])