test_chat_utils.py 70.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import warnings
5
from collections.abc import Mapping
6
from typing import Literal
7
8

import pytest
9
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
10

11
from vllm.assets.audio import AudioAsset
12
from vllm.assets.image import ImageAsset
13
from vllm.assets.video import VideoAsset
14
from vllm.config import ModelConfig
15
16
from vllm.entrypoints.chat_utils import (
    _try_extract_ast,
17
    apply_mistral_chat_template,
18
19
20
21
22
23
24
    load_chat_template,
    parse_chat_messages,
    parse_chat_messages_futures,
    resolve_chat_template_content_format,
    resolve_chat_template_kwargs,
    resolve_hf_chat_template,
)
25
from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
26
27
28
29
30
from vllm.multimodal.utils import (
    encode_audio_base64,
    encode_image_base64,
    encode_video_base64,
)
31
from vllm.tokenizers import MistralTokenizer, get_tokenizer
32
from vllm.utils.serial_utils import tensor2base64
33

34
from ..models.registry import HF_EXAMPLE_MODELS
35
36
37
38
from ..utils import VLLM_PATH

EXAMPLES_DIR = VLLM_PATH / "examples"

39
PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
40
ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
41
QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
42
QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
43
QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
44
QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
45
QWEN3_MODEL_ID = "Qwen/Qwen3-8B"
46
LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
47
HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
48
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
49
50


51
@pytest.fixture(scope="function")
52
def phi3v_model_config():
53
54
55
56
57
58
59
60
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "image": 2,
        },
    )
61
62


63
64
@pytest.fixture(scope="function")
def phi3v_model_config_mm_interleaved():
65
66
67
68
69
70
71
72
73
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        interleave_mm_strings=True,
        limit_mm_per_prompt={
            "image": 2,
        },
    )
74
75


76
77
78
79
80
81
82
83
84
85
86
87
88
@pytest.fixture(scope="function")
def phi3v_model_config_image_embeds():
    return ModelConfig(
        PHI3V_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "image": 2,
        },
        enable_mm_embeds=True,
    )


89
90
91
92
93
94
95
96
97
98
99
100
@pytest.fixture(scope="function")
def qwen2_audio_model_config():
    return ModelConfig(
        QWEN2AUDIO_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "audio": 1,
        },
    )


101
102
103
104
105
106
107
108
109
110
111
112
113
@pytest.fixture(scope="function")
def audio_embeds_model_config():
    return ModelConfig(
        QWEN2AUDIO_MODEL_ID,
        runner="generate",
        trust_remote_code=True,
        limit_mm_per_prompt={
            "audio": 2,
        },
        enable_mm_embeds=True,
    )


114
115
@pytest.fixture(scope="function")
def qwen25omni_model_config_mm_interleaved():
116
117
118
119
120
121
122
123
124
125
    return ModelConfig(
        QWEN25OMNI_MODEL_ID,
        runner="generate",
        interleave_mm_strings=True,
        limit_mm_per_prompt={
            "image": 2,
            "audio": 1,
            "video": 1,
        },
    )
126
127


128
129
@pytest.fixture(scope="function")
def mistral_model_config():
130
131
132
133
134
135
136
    return ModelConfig(
        MISTRAL_MODEL_ID,
        runner="generate",
        limit_mm_per_prompt={
            "image": 2,
        },
    )
137
138


139
140
@pytest.fixture(scope="module")
def image_url():
141
    image = ImageAsset("cherry_blossom")
142
143
144
145
    base64 = encode_image_base64(image.pil_image)
    return f"data:image/jpeg;base64,{base64}"


146
147
@pytest.fixture(scope="module")
def video_url():
148
    video = VideoAsset("baby_reading", 1)
149
150
151
152
153
154
    base64 = encode_video_base64(video.np_ndarrays)
    return f"data:video/jpeg;base64,{base64}"


@pytest.fixture(scope="module")
def audio_url():
155
    audio = AudioAsset("mary_had_lamb")
156
157
158
159
    base64 = encode_audio_base64(*audio.audio_and_sample_rate)
    return f"data:audio/ogg;base64,{base64}"


160
def _assert_mm_data_is_image_input(
161
    mm_data: MultiModalDataDict | None,
162
    image_count: int,
163
    skipped_image_indices: list | None = None,
164
165
166
167
168
169
170
) -> None:
    assert mm_data is not None
    assert set(mm_data.keys()) == {"image"}

    image_data = mm_data.get("image")
    assert image_data is not None

171
    assert isinstance(image_data, list) and len(image_data) == image_count
172
173
174
    if skipped_image_indices is not None:
        for i in skipped_image_indices:
            assert image_data[i] is None
175
176


177
def _assert_mm_uuids(
178
    mm_uuids: MultiModalUUIDDict | None,
179
    media_count: int,
180
    expected_uuids: list[str | None],
181
182
183
184
185
186
187
188
189
    modality: str = "image",
) -> None:
    if len(expected_uuids) > 0:
        assert mm_uuids is not None
        assert modality in mm_uuids

        image_uuids = mm_uuids.get(modality)
        assert image_uuids is not None

190
        assert isinstance(image_uuids, list) and len(image_uuids) == media_count
191
192
193
194
195
196

        assert image_uuids == expected_uuids
    else:
        assert mm_uuids is None


197
198
199
200
201
ModalityType = Literal["image", "video", "audio"]
MultiModalDataCounts = Mapping[ModalityType, int]


def _assert_mm_data_inputs(
202
    mm_data: MultiModalDataDict | None,
203
    data_count: MultiModalDataCounts,
204
    skipped_media_indices: dict[str, list] | None = None,  # modality -> list[int]
205
206
207
208
209
210
211
212
213
) -> None:
    assert mm_data is not None
    assert set(data_count.keys()) == (set(mm_data.keys()))

    for modality, n in data_count.items():
        modality_data = mm_data.get(modality)
        assert modality_data is not None
        assert isinstance(modality_data, list) and len(modality_data) == n

214
        if skipped_media_indices is not None:
215
            skipped_media_indices_for_modality = skipped_media_indices.get(modality)
216
217
218
219
            assert skipped_media_indices_for_modality is not None
            for i in skipped_media_indices_for_modality:
                assert modality_data[i] is None

220

221
222
223
224
def test_parse_chat_messages_single_image(
    phi3v_model_config,
    image_url,
):
225
    conversation, mm_data, mm_uuids = parse_chat_messages(
226
227
228
229
230
231
232
233
234
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
235
236
237
        phi3v_model_config,
        content_format="string",
    )
238

239
240
241
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
242
    _assert_mm_data_is_image_input(mm_data, 1)
243
244
245
246
247
248
249
250
251
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])


def test_parse_chat_messages_single_image_with_uuid(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
252
253
254
255
256
257
258
259
260
261
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid,
262
                    },
263
264
265
266
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
267
268
269
270
        phi3v_model_config,
        content_format="string",
    )

271
272
273
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
274
275
276
277
    _assert_mm_data_is_image_input(mm_data, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


278
279
280
281
282
283
def test_parse_chat_messages_single_empty_image_with_uuid(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
284
285
286
287
288
289
290
291
292
293
294
295
296
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
297
298
299
300
        phi3v_model_config,
        content_format="string",
    )

301
302
303
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
304
305
306
307
    _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


308
309
310
311
312
313
def test_parse_chat_messages_single_image_with_bad_uuid_format(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
314
315
316
317
318
319
320
321
322
323
324
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                            "uuid": image_uuid,
                        },
                        "bad_uuid_key": image_uuid,
325
                    },
326
327
328
329
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
330
331
332
333
        phi3v_model_config,
        content_format="string",
    )

334
335
336
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
337
338
339
340
341
342
343
344
345
346
347
348
    _assert_mm_data_is_image_input(mm_data, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])


def test_parse_chat_messages_multiple_images_with_uuids(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
349
350
351
352
353
354
355
356
357
358
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid1,
359
                    },
360
361
362
363
364
365
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid2,
366
                    },
367
368
369
370
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
371
372
373
374
        phi3v_model_config,
        content_format="string",
    )

375
376
377
378
379
380
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
381
382
383
384
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


385
386
387
388
389
390
391
392
def test_parse_chat_messages_multiple_empty_images_with_uuids(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid1,
                    },
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
411
412
413
414
        phi3v_model_config,
        content_format="string",
    )

415
416
417
418
419
420
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
421
422
423
424
425
426
427
428
429
430
431
432
    _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[0, 1])
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


def test_parse_chat_messages_mixed_empty_images_with_uuids(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_data, mm_uuids = parse_chat_messages(
433
434
435
436
437
438
439
440
441
442
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": image_url,
                        },
                        "uuid": image_uuid1,
443
                    },
444
445
446
447
448
449
450
451
452
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
453
454
455
456
        phi3v_model_config,
        content_format="string",
    )

457
458
459
460
461
462
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in the image?",
        }
    ]
463
464
465
466
    _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[1])
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


467
468
469
470
471
472
473
@pytest.mark.asyncio
async def test_parse_chat_messages_single_image_with_uuid_async(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
474
475
476
477
478
479
480
481
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
482
                    },
483
484
485
486
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
487
488
489
490
        phi3v_model_config,
        content_format="string",
    )

491
492
493
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
494
495
496
497
    _assert_mm_data_is_image_input(await mm_future, 1)
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


498
499
500
501
502
503
504
@pytest.mark.asyncio
async def test_parse_chat_messages_empty_image_with_uuid_async(
    phi3v_model_config,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
505
506
507
508
509
510
511
512
513
514
515
516
517
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
518
519
520
521
        phi3v_model_config,
        content_format="string",
    )

522
523
524
525
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
    _assert_mm_data_is_image_input(await mm_future, 1, skipped_image_indices=[0])
526
527
528
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])


529
530
531
532
533
534
535
536
537
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_uuids_async(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
538
539
540
541
542
543
544
545
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid1,
546
                    },
547
548
549
550
551
552
553
554
555
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
556
557
558
559
        phi3v_model_config,
        content_format="string",
    )

560
561
562
563
564
565
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
566
567
568
569
    _assert_mm_data_is_image_input(await mm_future, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


570
571
572
573
574
575
576
577
578
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
    phi3v_model_config,
    image_url,
):
    image_uuid1 = "my_uuid_1"
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": image_uuid1,
                    },
                    {
                        "type": "image_pil",
                        "image_pil": None,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
597
598
599
600
        phi3v_model_config,
        content_format="string",
    )

601
602
603
604
605
606
607
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
    _assert_mm_data_is_image_input(await mm_future, 2, skipped_image_indices=[0, 1])
608
609
610
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])


611
612
613
614
615
616
617
618
@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
    phi3v_model_config,
    image_url,
):
    image_uuid2 = "my_uuid_2"

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
619
620
621
622
623
624
625
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
626
                    },
627
628
629
630
631
632
633
634
635
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                        "uuid": image_uuid2,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
636
637
638
639
        phi3v_model_config,
        content_format="string",
    )

640
641
642
643
644
645
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
646
647
    _assert_mm_data_is_image_input(await mm_future, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, image_uuid2])
648
649


650
651
652
653
def test_parse_chat_messages_empty_system(
    mistral_model_config,
):
    # Test string format
654
    conversation, _, _ = parse_chat_messages(
655
        [
656
            {"role": "system", "content": ""},
657
658
            {
                "role": "user",
659
                "content": [{"type": "text", "text": "Who are you?"}],
660
661
            },
        ],
662
663
664
        mistral_model_config,
        content_format="string",
    )
665
    assert conversation == [
666
667
        {"role": "system", "content": ""},
        {"role": "user", "content": "Who are you?"},
668
    ]
669
670

    # Test openai format
671
    conversation, _, _ = parse_chat_messages(
672
        [
673
            {"role": "system", "content": ""},
674
675
            {
                "role": "user",
676
                "content": [{"type": "text", "text": "Who are you?"}],
677
678
679
680
681
682
            },
        ],
        mistral_model_config,
        content_format="openai",
    )
    assert conversation == [
683
684
        {"role": "system", "content": [{"type": "text", "text": ""}]},
        {"role": "user", "content": [{"type": "text", "text": "Who are you?"}]},
685
    ]
686
687


688
@pytest.mark.asyncio
689
690
691
692
async def test_parse_chat_messages_single_image_async(
    phi3v_model_config,
    image_url,
):
693
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
694
695
696
697
698
699
700
701
702
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in the image?"},
                ],
            }
        ],
703
704
705
        phi3v_model_config,
        content_format="string",
    )
706

707
708
709
    assert conversation == [
        {"role": "user", "content": "<|image_1|>\nWhat's in the image?"}
    ]
710
    _assert_mm_data_is_image_input(await mm_future, 1)
711
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
712
713
714
715
716
717


def test_parse_chat_messages_multiple_images(
    phi3v_model_config,
    image_url,
):
718
    conversation, mm_data, mm_uuids = parse_chat_messages(
719
720
721
722
723
724
725
726
727
728
729
730
731
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
732
733
734
        phi3v_model_config,
        content_format="string",
    )
735

736
737
738
739
740
741
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
742
    _assert_mm_data_is_image_input(mm_data, 2)
743
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
744
745


746
747
748
749
750
def test_parse_chat_messages_empty_pil_image_with_uuid(
    phi3v_model_config,
):
    uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
751
752
753
754
755
756
757
758
759
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_pil", "image_pil": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
760
761
762
763
        phi3v_model_config,
        content_format="string",
    )

764
765
766
767
768
769
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
770
771
772
773
774
    _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


def test_parse_chat_messages_empty_image_embeds_with_uuid(
775
    phi3v_model_config_image_embeds,
776
777
778
):
    uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
779
780
781
782
783
784
785
786
787
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_embeds", "image_embeds": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
788
        phi3v_model_config_image_embeds,
789
790
791
        content_format="string",
    )

792
793
794
795
796
797
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
798
799
800
801
802
803
    assert mm_data is not None
    assert "image" in mm_data
    assert mm_data["image"] is None
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
def test_parse_chat_messages_empty_audio_embeds_with_uuid(
    audio_embeds_model_config,
):
    """Test audio_embeds with UUID (no actual embeds data)."""
    uuid = "test-audio-uuid-123"

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {"type": "audio_embeds", "audio_embeds": None, "uuid": uuid},
                ],
            }
        ],
        audio_embeds_model_config,
        content_format="string",
    )

    # Should have audio in mm_data as None (UUID provided)
    assert mm_data is not None
    assert "audio" in mm_data
    assert mm_data["audio"] is None
    # UUID should be recorded
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[uuid])


def test_parse_chat_messages_audio_embeds_with_string(
    audio_embeds_model_config,
):
    """Test audio_embeds with base64 string embedding data."""

    import torch

    # Create a sample audio embedding tensor
    audio_embedding = torch.randn(1, 128, 768)

    # Encode it as base64
845
    base64_audio_embedding = tensor2base64(audio_embedding)
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886

    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {
                        "type": "audio_embeds",
                        "audio_embeds": base64_audio_embedding,
                    },
                ],
            }
        ],
        audio_embeds_model_config,
        content_format="string",
    )

    # Should have audio embedding in mm_data (single tensor, not a list)
    assert mm_data is not None
    assert "audio" in mm_data
    assert isinstance(mm_data["audio"], torch.Tensor)
    assert mm_data["audio"].shape == audio_embedding.shape
    # No UUID provided
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


@pytest.mark.asyncio
async def test_parse_chat_messages_audio_embeds_async(
    audio_embeds_model_config,
):
    """Test audio_embeds with async futures."""

    import torch

    # Create a sample audio embedding tensor
    audio_embedding = torch.randn(1, 128, 768)

    # Encode it as base64
887
    base64_audio_embedding = tensor2base64(audio_embedding)
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917

    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
        [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "Describe this audio"},
                    {
                        "type": "audio_embeds",
                        "audio_embeds": base64_audio_embedding,
                    },
                ],
            }
        ],
        audio_embeds_model_config,
        content_format="string",
    )

    # Should have audio embedding in mm_data (single tensor, not a list)
    mm_data = await mm_future
    assert mm_data is not None
    assert "audio" in mm_data
    assert isinstance(mm_data["audio"], torch.Tensor)
    assert mm_data["audio"].shape == audio_embedding.shape
    # No UUID provided
    assert mm_uuids is not None
    assert "audio" in mm_uuids
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


918
919
@pytest.mark.asyncio
async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
920
    phi3v_model_config_image_embeds,
921
922
923
):
    uuid = "abcd"
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
924
925
926
927
928
929
930
931
932
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_embeds", "image_embeds": None, "uuid": uuid},
                    {"type": "text", "text": "What's in this image?"},
                ],
            }
        ],
933
        phi3v_model_config_image_embeds,
934
935
936
        content_format="string",
    )

937
938
939
940
941
942
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\nWhat's in this image?",
        }
    ]
943
944
945
946
947
948
949
    mm_data = await mm_future
    assert mm_data is not None
    assert "image" in mm_data
    assert mm_data["image"] is None
    _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])


950
@pytest.mark.asyncio
951
952
953
954
async def test_parse_chat_messages_multiple_images_async(
    phi3v_model_config,
    image_url,
):
955
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
956
957
958
959
960
961
962
963
964
965
966
967
968
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "image_pil",
                        "image_pil": ImageAsset("cherry_blossom").pil_image,
                    },
                    {"type": "text", "text": "What's in these images?"},
                ],
            }
        ],
969
970
971
        phi3v_model_config,
        content_format="string",
    )
972

973
974
975
976
977
978
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
979
    _assert_mm_data_is_image_input(await mm_future, 2)
980
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
981
982
983
984
985
986


def test_parse_chat_messages_placeholder_already_in_prompt(
    phi3v_model_config,
    image_url,
):
987
    conversation, mm_data, mm_uuids = parse_chat_messages(
988
989
990
991
992
993
994
995
996
997
998
999
1000
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "text",
                        "text": "What's in <|image_1|> and how does it compare to <|image_2|>?",  # noqa: E501
                    },
                ],
            }
        ],
1001
1002
1003
        phi3v_model_config,
        content_format="string",
    )
1004
1005
1006
1007
1008
1009
    assert conversation == [
        {
            "role": "user",
            "content": "What's in <|image_1|> and how does it compare to <|image_2|>?",
        }
    ]
1010
    _assert_mm_data_is_image_input(mm_data, 2)
1011
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1012
1013


1014
1015
1016
1017
def test_parse_chat_messages_placeholder_one_already_in_prompt(
    phi3v_model_config,
    image_url,
):
1018
    conversation, mm_data, mm_uuids = parse_chat_messages(
1019
1020
1021
1022
1023
1024
1025
1026
        [
            {
                "role": "user",
                "content": [
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {
                        "type": "text",
1027
1028
                        "text": "What's in <|image_1|> and how does it compare to "
                        "the other one?",
1029
1030
1031
1032
                    },
                ],
            }
        ],
1033
1034
1035
        phi3v_model_config,
        content_format="string",
    )
1036

1037
1038
1039
    assert conversation == [
        {
            "role": "user",
1040
1041
            "content": "<|image_2|>\nWhat's in <|image_1|> and how does it compare to "
            "the other one?",
1042
1043
        }
    ]
1044
    _assert_mm_data_is_image_input(mm_data, 2)
1045
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1046
1047


1048
1049
1050
1051
def test_parse_chat_messages_multiple_images_across_messages(
    phi3v_model_config,
    image_url,
):
1052
    conversation, mm_data, mm_uuids = parse_chat_messages(
1053
1054
        [
            {
1055
                "role": "user",
1056
                "content": [
1057
1058
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What's in this image?"},
1059
1060
                ],
            },
1061
            {"role": "assistant", "content": "Some stuff."},
1062
            {
1063
                "role": "user",
1064
                "content": [
1065
1066
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "What about this one?"},
1067
1068
1069
                ],
            },
        ],
1070
1071
1072
        phi3v_model_config,
        content_format="string",
    )
1073
1074

    assert conversation == [
1075
1076
1077
        {"role": "user", "content": "<|image_1|>\nWhat's in this image?"},
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "<|image_2|>\nWhat about this one?"},
1078
    ]
1079
    _assert_mm_data_is_image_input(mm_data, 2)
1080
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1081
1082


1083
def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
1084
    phi3v_model_config,
1085
    image_url,
1086
):
1087
1088
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
1089
1090
        [
            {
1091
                "role": "user",
1092
1093
1094
                "content": [
                    {
                        "type": "image_url",
1095
                        "image_url": {"url": image_url},
1096
1097
                        "uuid": image_uuid,
                    },
1098
                    {"type": "text", "text": "What's in this image?"},
1099
                ],
1100
            },
1101
            {"role": "assistant", "content": "Some stuff."},
1102
            {
1103
                "role": "user",
1104
1105
1106
                "content": [
                    {
                        "type": "image_url",
1107
                        "image_url": {"url": image_url},
1108
1109
                        "uuid": image_uuid,
                    },
1110
                    {"type": "text", "text": "What about this one?"},
1111
1112
1113
1114
1115
1116
1117
1118
                ],
            },
        ],
        phi3v_model_config,
        content_format="string",
    )

    assert conversation == [
1119
1120
1121
        {"role": "user", "content": "<|image_1|>\nWhat's in this image?"},
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "<|image_2|>\nWhat about this one?"},
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
    ]
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])


def test_parse_chat_messages_context_text_format(
    phi3v_model_config,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
                "role": "user",
1134
                "content": [{"type": "text", "text": "What's in this text?"}],
1135
            },
1136
1137
            {"role": "assistant", "content": "Some stuff."},
            {"role": "user", "content": "What about this one?"},
1138
        ],
1139
1140
1141
        phi3v_model_config,
        content_format="openai",
    )
1142
1143
1144
1145

    assert conversation == [
        {
            "role": "user",
1146
            "content": [{"type": "text", "text": "What's in this text?"}],
1147
1148
1149
        },
        {
            "role": "assistant",
1150
            "content": [{"type": "text", "text": "Some stuff."}],
1151
1152
1153
        },
        {
            "role": "user",
1154
            "content": [{"type": "text", "text": "What about this one?"}],
1155
1156
        },
    ]
1157
1158
    assert mm_data is None
    assert mm_uuids is None
1159
1160


1161
1162
1163
1164
def test_parse_chat_messages_rejects_too_many_images_in_one_message(
    phi3v_model_config,
    image_url,
):
1165
1166
1167
    with warnings.catch_warnings():
        warnings.filterwarnings(
            "ignore",
1168
1169
            message="coroutine 'async_get_and_parse_image' was never awaited",
        )
1170
        with pytest.raises(ValueError, match="At most"):
1171
            parse_chat_messages(
1172
1173
1174
1175
1176
1177
1178
                [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1179
                            },
1180
1181
1182
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1183
                            },
1184
1185
1186
                            {
                                "type": "image_url",
                                "image_url": {"url": image_url},
1187
                            },
1188
1189
1190
1191
                            {"type": "text", "text": "What's in these images?"},
                        ],
                    }
                ],
1192
1193
1194
                phi3v_model_config,
                content_format="string",
            )
1195
1196


1197
1198
1199
1200
def test_parse_chat_messages_rejects_too_many_images_across_messages(
    phi3v_model_config,
    image_url,
):
1201
1202
1203
    with warnings.catch_warnings():
        warnings.filterwarnings(
            "ignore",
1204
1205
            message="coroutine 'async_get_and_parse_image' was never awaited",
        )
1206
        with pytest.raises(ValueError, match="At most"):
1207
            parse_chat_messages(
1208
1209
                [
                    {
1210
                        "role": "user",
1211
1212
1213
                        "content": [
                            {
                                "type": "image_url",
1214
                                "image_url": {"url": image_url},
1215
                            },
1216
                            {"type": "text", "text": "What's in this image?"},
1217
1218
                        ],
                    },
1219
                    {"role": "assistant", "content": "Some stuff."},
1220
                    {
1221
                        "role": "user",
1222
1223
1224
                        "content": [
                            {
                                "type": "image_url",
1225
                                "image_url": {"url": image_url},
1226
1227
1228
                            },
                            {
                                "type": "image_url",
1229
                                "image_url": {"url": image_url},
1230
                            },
1231
                            {"type": "text", "text": "What about these two?"},
1232
1233
1234
                        ],
                    },
                ],
1235
1236
1237
                phi3v_model_config,
                content_format="string",
            )
1238
1239
1240
1241
1242
1243


def test_parse_chat_messages_multiple_images_uncommon_input(
    phi3v_model_config,
    image_url,
):
1244
    conversation, mm_data, mm_uuids = parse_chat_messages(
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
        [
            {
                "role": "user",
                "content": [
                    "What's in these images?",
                    {"image_url": image_url},
                    {"image_url": image_url},
                ],
            }
        ],
1255
1256
1257
        phi3v_model_config,
        content_format="string",
    )
1258

1259
1260
1261
1262
1263
1264
    assert conversation == [
        {
            "role": "user",
            "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?",
        }
    ]
1265
    _assert_mm_data_is_image_input(mm_data, 2)
1266
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1267
1268


1269
1270
1271
1272
def test_parse_chat_messages_multiple_images_interleave(
    phi3v_model_config_mm_interleaved,
    image_url,
):
1273
    conversation, mm_data, mm_uuids = parse_chat_messages(
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
                    },
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "and this one"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1289
1290
1291
1292
        phi3v_model_config_mm_interleaved,
        content_format="string",
    )

1293
1294
1295
1296
1297
1298
1299
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1300
    _assert_mm_data_is_image_input(mm_data, 2)
1301
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1302
1303
1304
1305
1306
1307
1308


@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_interleave_async(
    phi3v_model_config_mm_interleaved,
    image_url,
):
1309
    conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
                    },
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "and this one"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1325
1326
1327
1328
        phi3v_model_config_mm_interleaved,
        content_format="string",
    )

1329
1330
1331
1332
1333
1334
1335
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1336
    _assert_mm_data_is_image_input(await mm_data, 2)
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


@pytest.mark.asyncio
async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
    phi3v_model_config_mm_interleaved,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1347
1348
1349
1350
1351
1352
1353
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "I need you to compare this image",
1354
                    },
1355
1356
1357
1358
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
1359
                    },
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
                    {"type": "text", "text": "and this one"},
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url},
                        "uuid": image_uuid,
                    },
                    {"type": "text", "text": "Do they have differences?"},
                ],
            }
        ],
1370
1371
1372
1373
        phi3v_model_config_mm_interleaved,
        content_format="string",
    )

1374
1375
1376
1377
1378
1379
1380
    assert conversation == [
        {
            "role": "user",
            "content": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
            "Do they have differences?",
        }
    ]
1381
1382
    _assert_mm_data_is_image_input(await mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1383
1384
1385
1386
1387
1388


def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
    phi3v_model_config_mm_interleaved,
    image_url,
):
1389
    conversation, mm_data, mm_uuids = parse_chat_messages(
1390
1391
        [
            {
1392
                "role": "user",
1393
                "content": [
1394
1395
1396
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Be accurate."},
1397
1398
                ],
            },
1399
            {"role": "assistant", "content": "Some stuff."},
1400
            {
1401
                "role": "user",
1402
                "content": [
1403
1404
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
1405
1406
1407
                ],
            },
        ],
1408
1409
1410
1411
        phi3v_model_config_mm_interleaved,
        content_format="string",
    )

1412
1413
1414
1415
1416
    assert conversation == [
        {
            "role": "user",
            "content": "What's on this image?\n<|image_1|>\nBe accurate.",
        },
1417
1418
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "What's on this image?\n<|image_2|>"},
1419
    ]
1420
    _assert_mm_data_is_image_input(mm_data, 2)
1421
1422
1423
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])


1424
def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave(
1425
1426
1427
1428
1429
1430
1431
    phi3v_model_config_mm_interleaved,
    image_url,
):
    image_uuid = str(hash(image_url))
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1432
                "role": "user",
1433
                "content": [
1434
                    {"type": "text", "text": "What's on this image?"},
1435
1436
                    {
                        "type": "image_url",
1437
                        "image_url": {"url": image_url},
1438
1439
                        "uuid": image_uuid,
                    },
1440
                    {"type": "text", "text": "Be accurate."},
1441
1442
                ],
            },
1443
            {"role": "assistant", "content": "Some stuff."},
1444
            {
1445
                "role": "user",
1446
                "content": [
1447
                    {"type": "text", "text": "What's on this image?"},
1448
1449
                    {
                        "type": "image_url",
1450
                        "image_url": {"url": image_url},
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
                        "uuid": image_uuid,
                    },
                ],
            },
        ],
        phi3v_model_config_mm_interleaved,
        content_format="string",
    )

    assert conversation == [
        {
            "role": "user",
            "content": "What's on this image?\n<|image_1|>\nBe accurate.",
        },
1465
1466
        {"role": "assistant", "content": "Some stuff."},
        {"role": "user", "content": "What's on this image?\n<|image_2|>"},
1467
1468
1469
    ]
    _assert_mm_data_is_image_input(mm_data, 2)
    _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1470
1471
1472


def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1473
1474
1475
1476
1477
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
1478
    conversation, mm_data, mm_uuids = parse_chat_messages(
1479
1480
        [
            {
1481
                "role": "user",
1482
                "content": [
1483
1484
1485
1486
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "Now listen to this audio"},
                    {"type": "audio_url", "audio_url": {"url": audio_url}},
1487
1488
                ],
            },
1489
            {"role": "assistant", "content": "Some stuff."},
1490
            {
1491
                "role": "user",
1492
                "content": [
1493
1494
1495
1496
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "And what's in the video?"},
                    {"type": "video_url", "video_url": {"url": video_url}},
1497
1498
1499
1500
1501
1502
1503
1504
1505
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        content_format="string",
    )

    assert conversation == [
        {
1506
            "role": "user",
1507
1508
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1509
        },
1510
        {"role": "assistant", "content": "Some stuff."},
1511
        {
1512
            "role": "user",
1513
1514
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1515
1516
        },
    ]
1517
1518

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1519
    _assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=[None, None])
1520
1521
1522
1523
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])


1524
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave(
1525
1526
1527
1528
1529
1530
1531
1532
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1533
                "role": "user",
1534
                "content": [
1535
                    {"type": "text", "text": "What's on this image?"},
1536
1537
                    {
                        "type": "image_url",
1538
                        "image_url": {"url": image_url},
1539
1540
                        "uuid": "image_123",
                    },
1541
                    {"type": "text", "text": "Now listen to this audio"},
1542
1543
                    {
                        "type": "audio_url",
1544
                        "audio_url": {"url": audio_url},
1545
1546
1547
1548
                        "uuid": "audio_123",
                    },
                ],
            },
1549
            {"role": "assistant", "content": "Some stuff."},
1550
            {
1551
                "role": "user",
1552
                "content": [
1553
                    {"type": "text", "text": "What's on this image?"},
1554
1555
                    {
                        "type": "image_url",
1556
                        "image_url": {"url": image_url},
1557
1558
                        "uuid": "image_123",
                    },
1559
                    {"type": "text", "text": "And what's in the video?"},
1560
1561
                    {
                        "type": "video_url",
1562
                        "video_url": {"url": video_url},
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
                        "uuid": "video_123",
                    },
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        content_format="string",
    )

    assert conversation == [
        {
1574
            "role": "user",
1575
1576
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1577
        },
1578
        {"role": "assistant", "content": "Some stuff."},
1579
        {
1580
            "role": "user",
1581
1582
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1583
1584
1585
1586
        },
    ]

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1587
1588
1589
1590
1591
    _assert_mm_uuids(
        mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
    )
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
1592
1593


1594
1595
1596
1597
1598
1599
1600
1601
1602
def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave(  # noqa: E501
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1603
                "role": "user",
1604
                "content": [
1605
                    {"type": "text", "text": "What's on this image?"},
1606
1607
1608
1609
1610
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": "image_123",
                    },
1611
                    {"type": "text", "text": "Now listen to this audio"},
1612
1613
1614
1615
1616
1617
1618
                    {
                        "type": "audio_url",
                        "audio_url": None,
                        "uuid": "audio_123",
                    },
                ],
            },
1619
            {"role": "assistant", "content": "Some stuff."},
1620
            {
1621
                "role": "user",
1622
                "content": [
1623
                    {"type": "text", "text": "What's on this image?"},
1624
1625
1626
1627
1628
                    {
                        "type": "image_url",
                        "image_url": None,
                        "uuid": "image_123",
                    },
1629
                    {"type": "text", "text": "And what's in the video?"},
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
                    {
                        "type": "video_url",
                        "video_url": None,
                        "uuid": "video_123",
                    },
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        content_format="string",
    )

    assert conversation == [
        {
1644
            "role": "user",
1645
1646
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1647
        },
1648
        {"role": "assistant", "content": "Some stuff."},
1649
        {
1650
            "role": "user",
1651
1652
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1653
1654
1655
        },
    ]

1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
    _assert_mm_data_inputs(
        mm_data,
        {"image": 2, "video": 1, "audio": 1},
        skipped_media_indices={"image": [0, 1], "video": [0], "audio": [0]},
    )
    _assert_mm_uuids(
        mm_uuids, 2, modality="image", expected_uuids=["image_123", "image_123"]
    )
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=["audio_123"])
1666
1667


1668
1669
1670
1671
1672
1673
1674
1675
1676
def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave(  # noqa: E501
    qwen25omni_model_config_mm_interleaved,
    image_url,
    video_url,
    audio_url,
):
    conversation, mm_data, mm_uuids = parse_chat_messages(
        [
            {
1677
                "role": "user",
1678
                "content": [
1679
                    {"type": "text", "text": "What's on this image?"},
1680
1681
                    {
                        "type": "image_url",
1682
                        "image_url": {"url": image_url},
1683
1684
                        "uuid": "image_123",
                    },
1685
1686
                    {"type": "text", "text": "Now listen to this audio"},
                    {"type": "audio_url", "audio_url": {"url": audio_url}},
1687
1688
                ],
            },
1689
            {"role": "assistant", "content": "Some stuff."},
1690
            {
1691
                "role": "user",
1692
                "content": [
1693
1694
1695
                    {"type": "text", "text": "What's on this image?"},
                    {"type": "image_url", "image_url": {"url": image_url}},
                    {"type": "text", "text": "And what's in the video?"},
1696
1697
                    {
                        "type": "video_url",
1698
                        "video_url": {"url": video_url},
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
                        "uuid": "video_123",
                    },
                ],
            },
        ],
        qwen25omni_model_config_mm_interleaved,
        content_format="string",
    )

    assert conversation == [
        {
1710
            "role": "user",
1711
1712
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nNow listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>",
1713
        },
1714
        {"role": "assistant", "content": "Some stuff."},
1715
        {
1716
            "role": "user",
1717
1718
            "content": "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>"
            "\nAnd what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1719
1720
1721
1722
        },
    ]

    _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1723
1724
    _assert_mm_uuids(mm_uuids, 2, modality="image", expected_uuids=["image_123", None])
    _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=["video_123"])
1725
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
1726
1727
1728
1729
1730
1731
1732


def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
    phi3v_model_config_mm_interleaved,
    image_url,
):
    with pytest.raises(
1733
1734
1735
        ValueError,
        match=r"Found more '<|image_1|>' placeholders in input prompt "
        "than actual multimodal data items.",
1736
    ):
1737
        parse_chat_messages(
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
            [
                {
                    "role": "user",
                    "content": [
                        {"type": "image_url", "image_url": {"url": image_url}},
                        {"type": "image_url", "image_url": {"url": image_url}},
                        {
                            "type": "text",
                            "text": "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n"  # noqa: E501
                            "Do they have differences?",
                        },
                    ],
                }
            ],
1752
1753
1754
1755
1756
            phi3v_model_config_mm_interleaved,
            content_format="string",
        )


1757
1758
1759
1760
1761
@pytest.mark.parametrize(
    "model",
    [
        QWEN2VL_MODEL_ID,  # tokenizer.chat_template is of type str
        HERMES_MODEL_ID,  # tokenizer.chat_template is of type dict
1762
1763
    ],
)
1764
1765
1766
@pytest.mark.parametrize("use_tools", [True, False])
def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
    """checks that chat_template is a dict type for HF models."""
1767
1768
1769
1770
1771
1772
1773
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
1774
        revision=model_info.revision,
1775
1776
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
1777
1778
1779
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
1780
        enforce_eager=model_info.enforce_eager,
1781
1782
        dtype=model_info.dtype,
    )
1783

1784
1785
    # Build the tokenizer
    tokenizer = get_tokenizer(
1786
        model,
1787
        trust_remote_code=model_config.trust_remote_code,
1788
1789
    )

1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
    tools = (
        [
            {
                "type": "function",
                "function": {
                    "name": "dummy_function_name",
                    "description": "This is a dummy function",
                    "parameters": sample_json_schema,
                },
            }
        ]
        if use_tools
        else None
    )
1804
1805

    # Test detecting the tokenizer's chat_template
1806
    chat_template = resolve_hf_chat_template(
1807
1808
1809
        tokenizer,
        chat_template=None,
        tools=tools,
1810
        model_config=model_config,
1811
1812
1813
1814
    )
    assert isinstance(chat_template, str)


1815
1816
1817
1818
1819
1820
@pytest.mark.parametrize(
    "model, expected_kwargs",
    [
        (
            QWEN2VL_MODEL_ID,
            {
1821
1822
1823
1824
                "add_vision_id",
                "add_generation_prompt",
                "continue_final_message",
                "tools",
1825
1826
1827
1828
1829
            },
        ),
        (
            QWEN3_MODEL_ID,
            {
1830
1831
1832
1833
                "enable_thinking",
                "add_generation_prompt",
                "continue_final_message",
                "tools",
1834
1835
1836
1837
            },
        ),
    ],
)
1838
def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwargs):
1839
1840
1841
1842
    """checks that chat_template is a dict type for HF models."""
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
    tools = [
        {
            "type": "function",
            "function": {
                "name": "dummy_function_name",
                "description": "This is a dummy function",
                "parameters": sample_json_schema,
            },
        }
    ]
1853
1854
1855
1856
1857
1858
1859

    chat_template_kwargs = {
        # both unused
        "unsed_kwargs_1": 123,
        "unsed_kwargs_2": "abc",
        # should not appear
        "chat_template": "{% Hello world! %}",
1860
        "tokenize": True,
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
        # used by tokenizer
        "continue_final_message": True,
        "tools": tools,
        # both used by Qwen2-VL and Qwen3
        "add_generation_prompt": True,
        # only used by Qwen2-VL
        "add_vision_id": True,
        # only used by Qwen3
        "enable_thinking": True,
    }

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
        revision=model_info.revision,
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
1879
1880
1881
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
1882
        enforce_eager=model_info.enforce_eager,
1883
1884
        dtype=model_info.dtype,
    )
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898

    # Build the tokenizer
    tokenizer = get_tokenizer(
        model,
        trust_remote_code=model_config.trust_remote_code,
    )

    # Test detecting the tokenizer's chat_template
    chat_template = resolve_hf_chat_template(
        tokenizer,
        chat_template=None,
        tools=tools,
        model_config=model_config,
    )
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
    with pytest.raises(
        ValueError, match="Found unexpected chat template kwargs from request"
    ):
        # should raise error if `chat_template_kwargs` contains
        # `chat_template` or `tokenize`
        resolve_chat_template_kwargs(
            tokenizer,
            chat_template=chat_template,
            chat_template_kwargs=chat_template_kwargs,
        )
1909
1910
1911
1912
    resolved_chat_template_kwargs = resolve_chat_template_kwargs(
        tokenizer,
        chat_template=chat_template,
        chat_template_kwargs=chat_template_kwargs,
1913
        raise_on_unexpected=False,
1914
1915
1916
    )
    assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs

1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
    # Additional test: Verify HF base parameters work with **kwargs tokenizers
    # This validates the fix for tokenizers like Kimi K2 that use **kwargs
    # to receive standard HuggingFace parameters instead of declaring them explicitly
    from vllm.entrypoints.chat_utils import _get_hf_base_chat_template_params

    hf_base_params = _get_hf_base_chat_template_params()
    # Verify common HF parameters are in the base class
    assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset(
        hf_base_params
    ), f"Expected HF base params not found in {hf_base_params}"

    # Test with a mock tokenizer that uses **kwargs (like Kimi K2)
    class MockTokenizerWithKwargs:
        def apply_chat_template(self, conversation, **kwargs):
            return "mocked_output"

    mock_tokenizer = MockTokenizerWithKwargs()
    mock_kwargs = {
        "add_generation_prompt": True,
        "tools": tools,
        "continue_final_message": False,
        "unknown_param": "should_be_filtered",
    }
    resolved_mock = resolve_chat_template_kwargs(
        mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False
    )
    # HF base params should pass through even with **kwargs tokenizer
    assert "add_generation_prompt" in resolved_mock
    assert "tools" in resolved_mock
    assert "continue_final_message" in resolved_mock
    # Unknown params should be filtered out
    assert "unknown_param" not in resolved_mock

1950

1951
1952
# NOTE: Qwen2-Audio default chat template is specially defined inside
# processor class instead of using `tokenizer_config.json`
1953
1954
@pytest.mark.parametrize(
    ("model", "expected_format"),
1955
1956
1957
1958
1959
1960
1961
1962
    [
        (PHI3V_MODEL_ID, "string"),
        (QWEN2VL_MODEL_ID, "openai"),
        (QWEN25VL_MODEL_ID, "openai"),
        (ULTRAVOX_MODEL_ID, "string"),
        (QWEN2AUDIO_MODEL_ID, "openai"),
        (LLAMA_GUARD_MODEL_ID, "openai"),
    ],
1963
1964
)
def test_resolve_content_format_hf_defined(model, expected_format):
1965
1966
1967
1968
1969
1970
1971
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
1972
        revision=model_info.revision,
1973
1974
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
1975
1976
1977
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
1978
        enforce_eager=model_info.enforce_eager,
1979
1980
        dtype=model_info.dtype,
    )
1981

1982
    tokenizer = get_tokenizer(
1983
        model,
1984
        trust_remote_code=model_config.trust_remote_code,
1985
1986
    )

1987
    # Test detecting the tokenizer's chat_template
1988
    chat_template = resolve_hf_chat_template(
1989
1990
1991
        tokenizer,
        chat_template=None,
        tools=None,
1992
        model_config=model_config,
1993
    )
1994
1995
1996
1997
1998
1999
2000
2001
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
2002
2003
2004
2005
        None,  # Test detecting the tokenizer's chat_template
        None,
        "auto",
        tokenizer,
2006
        model_config=model_config,
2007
2008
2009
2010
2011
2012
2013
    )

    assert resolved_format == expected_format


@pytest.mark.parametrize(
    ("model", "expected_format"),
2014
2015
2016
2017
2018
2019
2020
2021
2022
    [
        ("Salesforce/blip2-opt-2.7b", "string"),
        ("facebook/chameleon-7b", "string"),
        ("deepseek-ai/deepseek-vl2-tiny", "string"),
        ("adept/fuyu-8b", "string"),
        ("google/paligemma-3b-mix-224", "string"),
        ("Qwen/Qwen-VL", "string"),
        ("Qwen/Qwen-VL-Chat", "string"),
    ],
2023
2024
2025
2026
2027
2028
2029
2030
2031
)
def test_resolve_content_format_fallbacks(model, expected_format):
    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
    model_info.check_available_online(on_fail="skip")

    model_config = ModelConfig(
        model,
        tokenizer=model_info.tokenizer or model,
        tokenizer_mode=model_info.tokenizer_mode,
2032
        revision=model_info.revision,
2033
2034
        trust_remote_code=model_info.trust_remote_code,
        hf_overrides=model_info.hf_overrides,
2035
2036
2037
        skip_tokenizer_init=model_info.require_embed_inputs,
        enable_prompt_embeds=model_info.require_embed_inputs,
        enable_mm_embeds=model_info.require_embed_inputs,
2038
        enforce_eager=model_info.enforce_eager,
2039
2040
        dtype=model_info.dtype,
    )
2041

2042
    tokenizer = get_tokenizer(
2043
2044
2045
2046
2047
2048
2049
2050
2051
        model_config.tokenizer,
        trust_remote_code=model_config.trust_remote_code,
    )

    # Test detecting the tokenizer's chat_template
    chat_template = resolve_hf_chat_template(
        tokenizer,
        chat_template=None,
        tools=None,
2052
        model_config=model_config,
2053
2054
2055
2056
2057
2058
2059
2060
2061
    )
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
2062
        None,  # Test detecting the tokenizer's chat_template
2063
        None,
2064
2065
        "auto",
        tokenizer,
2066
        model_config=model_config,
2067
2068
2069
2070
2071
2072
2073
    )

    assert resolved_format == expected_format


@pytest.mark.parametrize(
    ("template_path", "expected_format"),
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
    [
        ("template_alpaca.jinja", "string"),
        ("template_baichuan.jinja", "string"),
        ("template_chatglm.jinja", "string"),
        ("template_chatglm2.jinja", "string"),
        ("template_chatml.jinja", "string"),
        ("template_dse_qwen2_vl.jinja", "openai"),
        ("template_falcon_180b.jinja", "string"),
        ("template_falcon.jinja", "string"),
        ("template_inkbot.jinja", "string"),
        ("template_teleflm.jinja", "string"),
        ("template_vlm2vec_phi3v.jinja", "openai"),
        ("template_vlm2vec_qwen2vl.jinja", "openai"),
        ("tool_chat_template_granite_20b_fc.jinja", "string"),
        ("tool_chat_template_hermes.jinja", "string"),
        ("tool_chat_template_internlm2_tool.jinja", "string"),
        ("tool_chat_template_llama3.1_json.jinja", "openai"),
        ("tool_chat_template_llama3.2_json.jinja", "openai"),
        ("tool_chat_template_mistral_parallel.jinja", "string"),
        ("tool_chat_template_mistral.jinja", "string"),
    ],
2095
2096
)
def test_resolve_content_format_examples(template_path, expected_format):
2097
2098
2099
2100
2101
2102
    model_config = ModelConfig(
        PHI3V_MODEL_ID,  # Dummy
        tokenizer=PHI3V_MODEL_ID,  # Dummy
        trust_remote_code=True,
    )

2103
    dummy_tokenizer = get_tokenizer(
2104
2105
        PHI3V_MODEL_ID,  # Dummy
        trust_remote_code=model_config.trust_remote_code,
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
    )
    dummy_tokenizer.chat_template = None

    chat_template = load_chat_template(EXAMPLES_DIR / template_path)
    assert isinstance(chat_template, str)

    print("[TEXT]")
    print(chat_template)
    print("[AST]")
    print(_try_extract_ast(chat_template))

    resolved_format = resolve_chat_template_content_format(
        chat_template,
2119
        None,
2120
2121
        "auto",
        dummy_tokenizer,
2122
        model_config=model_config,
2123
2124
2125
    )

    assert resolved_format == expected_format
Julien Denize's avatar
Julien Denize committed
2126
2127


2128
def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
    messages = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "thinking",
                    "closed": True,
                    "thinking": "Only return the answer when you are confident.",
                },
            ],
        },
        {"role": "user", "content": "What is 2+2?"},
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
                {
                    "type": "text",
                    "text": "The answer is 4.",
                },
            ],
        },
    ]
Julien Denize's avatar
Julien Denize committed
2154

2155
    conversation_with_thinking, _, _ = parse_chat_messages(
Julien Denize's avatar
Julien Denize committed
2156
2157
2158
2159
2160
        messages,
        mistral_model_config,
        content_format="openai",
    )

2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
    expected_conversation = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "text",
                    "text": "Only return the answer when you are confident.",
                },
            ],
        },
        {
            "role": "user",
            "content": [{"type": "text", "text": "What is 2+2?"}],
        },
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "text", "text": "2+2 = 4"},
                {"type": "text", "text": "The answer is 4."},
            ],
        },
    ]
Julien Denize's avatar
Julien Denize committed
2185
2186
2187
2188
2189

    assert conversation_with_thinking == expected_conversation


def test_apply_mistral_chat_template_thinking_chunk():
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
    messages = [
        {
            "role": "system",
            "content": [
                {"type": "text", "text": "You are a helpful assistant."},
                {
                    "type": "thinking",
                    "closed": True,
                    "thinking": "Only return the answer when you are confident.",
                },
            ],
        },
        {"role": "user", "content": "What is 2+2?"},
        {
            "role": "assistant",
            "content": [
                {"type": "text", "text": "Let me think about it."},
                {"type": "thinking", "closed": True, "thinking": "2+2 = 4"},
                {
                    "type": "text",
                    "text": "The answer is 4.",
                },
            ],
        },
        {"role": "user", "content": "Thanks, what is 3+3?"},
    ]
Julien Denize's avatar
Julien Denize committed
2216
    mistral_tokenizer = MistralTokenizer.from_pretrained(
2217
        "mistralai/Magistral-Small-2509"
2218
    )
Julien Denize's avatar
Julien Denize committed
2219

2220
2221
2222
    tokens_ids = apply_mistral_chat_template(
        mistral_tokenizer, messages, chat_template=None, tools=None
    )
Julien Denize's avatar
Julien Denize committed
2223
2224

    string_tokens = mistral_tokenizer.mistral.decode(
2225
2226
        tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP
    )
Julien Denize's avatar
Julien Denize committed
2227
2228
2229
2230
2231
2232

    expected_tokens = (
        r"<s>[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the"
        r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]"
        r"[INST]What is 2+2?[/INST]"
        r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4.</s>"
2233
2234
        r"[INST]Thanks, what is 3+3?[/INST]"
    )
Julien Denize's avatar
Julien Denize committed
2235
2236

    assert string_tokens == expected_tokens
2237
2238
2239
2240
2241
2242
2243


def test_parse_chat_messages_single_empty_audio_with_uuid(
    qwen2_audio_model_config,
):
    audio_uuid = "abcd"
    conversation, mm_data, mm_uuids = parse_chat_messages(
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "input_audio",
                        "input_audio": {},
                        "uuid": audio_uuid,
                    },
                    {"type": "text", "text": "What does the audio say?"},
                ],
            }
        ],
2257
2258
2259
2260
        qwen2_audio_model_config,
        content_format="string",
    )

2261
2262
2263
    assert conversation == [
        {
            "role": "user",
2264
2265
            "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
            "audio say?",
2266
2267
        }
    ]
2268
    _assert_mm_data_inputs(mm_data, {"audio": 1})
2269
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])
2270
2271
2272
2273
2274
2275
2276
2277


@pytest.mark.asyncio
async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
    qwen2_audio_model_config,
):
    audio_uuid = "abcd"
    conversation, mm_future, mm_uuids = parse_chat_messages_futures(
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "input_audio",
                        "input_audio": {},
                        "uuid": audio_uuid,
                    },
                    {"type": "text", "text": "What does the audio say?"},
                ],
            }
        ],
2291
2292
2293
2294
        qwen2_audio_model_config,
        content_format="string",
    )

2295
2296
2297
    assert conversation == [
        {
            "role": "user",
2298
2299
            "content": "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the "
            "audio say?",
2300
2301
        }
    ]
2302
    _assert_mm_data_inputs(await mm_future, {"audio": 1})
2303
    _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid])