test_processing.py 22.4 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
from contextlib import nullcontext
4
from types import MethodType
5
from typing import cast
6
from unittest.mock import MagicMock
7

8
import numpy as np
9
import pytest
10
from transformers import ProcessorMixin
11
12
13

from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
14
15
16
# yapf conflicts with isort for this block
# yapf: disable
from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
17
18
19
                                        PromptInsertion, PromptReplacement,
                                        apply_text_matches,
                                        apply_token_matches,
20
                                        find_mm_placeholders,
21
                                        find_text_matches, find_token_matches,
22
                                        iter_token_matches)
23
# yapf: enable
24
from vllm.multimodal.profiling import MultiModalProfiler
25
26
from vllm.transformers_utils.tokenizer import (AnyTokenizer,
                                               cached_tokenizer_from_config)
27
28
from vllm.utils import full_groupby

29
30
from .utils import random_image

31
32
33
34
35

# yapf: disable
@pytest.mark.parametrize(
    ("token_ids", "match_ids", "expected"),
    [
36
        ([], [], []),
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
        ([], [32000], []),
        (
            [32000, 32000, 32000],
            [32000],
            [
                { "start_idx": 0, "end_idx": 1 },
                { "start_idx": 1, "end_idx": 2 },
                { "start_idx": 2, "end_idx": 3 },
            ],
        ),
        (
            [32000, 32000, 32000],
            [32000, 32000],
            [{ "start_idx": 0, "end_idx": 2 }],
        ),
        (
            [32000, 32000, 32000],
            [32000, 32000, 32000],
            [{ "start_idx": 0, "end_idx": 3 }],
        ),
        (
            [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
            [28747, 32000],
            [
                { "start_idx": 1, "end_idx": 3 },
                { "start_idx": 6, "end_idx": 8 },
            ],
        ),
        (
            [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
            [28747, 32000, 32000, 32000],
            [
                { "start_idx": 1, "end_idx": 5 },
            ],
        ),
        (
            [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
            [28747, 0, 32000],
            [],
        ),
    ],
)
# yapf: enable
def test_iter_token_matches(token_ids, match_ids, expected):
    result = list(iter_token_matches(token_ids, match_ids))

    # Manually constructed results
    assert [item._asdict() for item in result] == expected

    # Invariants
    match_lens = [end - start for start, end in result]
    print("match_lens:", match_lens)  # Only displayed on error
    assert all(match_len == len(match_ids) for match_len in match_lens)


# yapf: disable
@pytest.mark.parametrize(
    ("prompt", "target_by_key", "expected_by_key"),
    [
        (
            [],
            {
                "pattern_1": [],
                "pattern_2": [32000],
            },
            {
103
                "pattern_1": [],
104
                "pattern_2": [],
105
            },
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
        ),
        (
            [32000, 32000, 32000, 32000],
            {
                "pattern_1": [32000],
                "pattern_2": [32000, 32000],
                "pattern_3": [32000, 32000, 32000],
            },
            {
                "pattern_1": [
                    { "start_idx": 0, "end_idx": 1 },
                    { "start_idx": 1, "end_idx": 2 },
                    { "start_idx": 2, "end_idx": 3 },
                    { "start_idx": 3, "end_idx": 4 },
                ],
                "pattern_2": [
                    { "start_idx": 0, "end_idx": 2 },
                    { "start_idx": 2, "end_idx": 4 },
                ],
                "pattern_3": [
                    { "start_idx": 0, "end_idx": 3 },
                ],
            },
        ),
        (
            [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
            {
                "pattern_1": [28747, 32000],
                "pattern_2": [28747, 32000, 32000, 32000],
                "pattern_3": [28747, 0, 32000],
            },
            {
                "pattern_1": [
                    { "start_idx": 1, "end_idx": 3 },
                    { "start_idx": 6, "end_idx": 8 },
                ],
                "pattern_2": [
                    { "start_idx": 1, "end_idx": 5 },
                ],
                "pattern_3": [],
            },
        ),
    ],
)
150
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
151
# yapf: enable
152
153
154
155
156
157
def test_find_token_matches(
    prompt,
    target_by_key,
    expected_by_key,
    update_type,
):
158
159
160
    # Should not be used since there is nothing to convert to token IDs
    mock_tokenizer = cast(AnyTokenizer, object())

161
162
    prompt_updates = [
        update_type(key, target, []).bind(mock_tokenizer)
163
164
        for key, target in target_by_key.items()
    ]
165
    result = find_token_matches(prompt, prompt_updates)
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262

    # Only displayed on error
    print("result:", result)

    # Manually constructed results
    result_groups = dict(full_groupby(result, key=lambda x: x.modality))
    assert {
        key: [
            dict(start_idx=item.start_idx, end_idx=item.end_idx)
            for item in result_groups.get(key, [])
        ]
        for key in expected_by_key
    } == expected_by_key


# yapf: disable
@pytest.mark.parametrize(
    ("prompt", "target_by_key", "expected_by_key"),
    [
        # Detokenized test cases of `test_find_token_matches`
        # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
        (
            "",
            {
                "pattern_1": "",
                "pattern_2": "<image>",
            },
            {
                "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
                "pattern_2": [],
            }
        ),
        (
            "<image><image><image><image>",
            {
                "pattern_1": "<image>",
                "pattern_2": "<image><image>",
                "pattern_3": "<image><image><image>",
            },
            {
                "pattern_1": [
                    { "start_idx": 0, "end_idx": 7 },
                    { "start_idx": 7, "end_idx": 14 },
                    { "start_idx": 14, "end_idx": 21 },
                    { "start_idx": 21, "end_idx": 28 },
                ],
                "pattern_2": [
                    { "start_idx": 0, "end_idx": 14 },
                    { "start_idx": 14, "end_idx": 28 },
                ],
                "pattern_3": [
                    { "start_idx": 0, "end_idx": 21 },
                ],
            },
        ),
        (
            "Image:<image><image><image>Image:<image><image>!",
            {
                "pattern_1": "Image:<image>",
                "pattern_2": "Image:<image><image><image>",
                "pattern_3": "Image:<unk><image>",
            },
            {
                "pattern_1": [
                    { "start_idx": 0, "end_idx": 13 },
                    { "start_idx": 27, "end_idx": 40 },
                ],
                "pattern_2": [
                    { "start_idx": 0, "end_idx": 27 },
                ],
                "pattern_3": [],
            },
        ),
        # Test regex escape
        (
            "<|image|><image><|image|><image>",
            {
                "pattern_1": "<|image|>",
                "pattern_2": "<|image|><image>",
                "pattern_3": "<|image|><image><|image|>",
            },
            {
                "pattern_1": [
                    { "start_idx": 0, "end_idx": 9 },
                    { "start_idx": 16, "end_idx": 25 },
                ],
                "pattern_2": [
                    { "start_idx": 0, "end_idx": 16 },
                    { "start_idx": 16, "end_idx": 32 },
                ],
                "pattern_3": [
                    { "start_idx": 0, "end_idx": 25 },
                ],
            },
        ),
    ],
)
263
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
264
# yapf: enable
265
266
267
268
269
270
def test_find_text_matches(
    prompt,
    target_by_key,
    expected_by_key,
    update_type,
):
271
272
273
    # Should not be used since there is nothing to convert to text
    mock_tokenizer = cast(AnyTokenizer, object())

274
275
    prompt_updates = [
        update_type(key, target, []).bind(mock_tokenizer)
276
277
        for key, target in target_by_key.items()
    ]
278
    result = find_text_matches(prompt, prompt_updates)
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295

    # Only displayed on error
    print("result:", result)

    # Manually constructed results
    result_groups = dict(full_groupby(result, key=lambda x: x.modality))
    assert {
        key: [
            dict(start_idx=item.start_idx, end_idx=item.end_idx)
            for item in result_groups.get(key, [])
        ]
        for key in expected_by_key
    } == expected_by_key


# yapf: disable
@pytest.mark.parametrize(
296
    ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"),  # noqa: E501
297
298
299
300
301
302
303
304
305
306
307
    [
        (
            "Image:<image>Image:<image><image>!",
            {
                # We use `<image>` before `Image:` to test matches that
                # occur out of order
                "pattern_1": "<image>",
                "pattern_2": "Image:",
                "pattern_3": "!",
            },
            {
308
309
310
311
312
313
                # Test whether target is confused with replacement
                "pattern_1": "<image><image>",
                # Test empty replacement
                "pattern_2": "",
                # Test dynamic replacement (beyond the form of `unit * count`)
                "pattern_3": "?!?",
314
            },
315
316
317
318
319
320
321
322
323
324
325
326
            {
                PromptInsertion: {
                    0: "Image:<image>Image:<image><image>!",
                    1: "Image:<image><image><image>Image:<image><image>!?!?",
                    2: "Image:<image><image><image><image><image>Image:<image><image>!?!??!?",  # noqa: E501
                },
                PromptReplacement: {
                    0: "Image:<image>Image:<image><image>!",
                    1: "<image><image>Image:<image><image>?!?",
                    2: "<image><image><image><image><image>?!?",
                },
            },
327
328
329
330
        ),
    ]
)
# yapf: enable
331
def test_find_update_text(
332
333
334
    prompt,
    target_by_key,
    repl_by_key,
335
    expected_by_update_type_mm_count,
336
337
338
339
):
    # Should not be used since there is nothing to convert to text
    mock_tokenizer = cast(AnyTokenizer, object())

340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
    for (
            update_type,
            expected_by_mm_count,
    ) in expected_by_update_type_mm_count.items():
        mm_prompt_updates = {
            key:
            [update_type(key, target, repl_by_key[key]).bind(mock_tokenizer)]
            for key, target in target_by_key.items()
        }
        mm_matches = {
            key: find_text_matches(prompt, updates)
            for key, updates in mm_prompt_updates.items()
        }

        for mm_count, expected in expected_by_mm_count.items():
            result = apply_text_matches(
                prompt,
                mm_matches,
                {key: mm_count
                 for key in repl_by_key},
            )

            # Only displayed on error
            print("update_type:", update_type)
            print("mm_count:", mm_count)
            print("mm_matches:", mm_matches)
            print("result:", result)

            # Manually constructed results
            assert result == expected
370
371
372
373


# yapf: disable
@pytest.mark.parametrize(
374
    ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"),  # noqa: E501
375
376
377
378
379
380
381
382
383
384
385
386
387
    [
        # Tokenized test cases of `test_find_replace_text`
        # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
        (
            [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
            {
                # We use `<image>` before `Image:` to test matches that
                # occur out of order
                "pattern_1": [32000],
                "pattern_2": [9833, 28747],
                "pattern_3": [918],
            },
            {
388
389
390
391
392
393
                # Test whether target is confused with replacement
                "pattern_1": [32000, 32000],
                # Test empty replacement
                "pattern_2": [],
                # Test dynamic replacement (beyond the form of `unit * count`)
                "pattern_3": [1550, 918, 1550],
394
            },
395
396
397
398
399
400
401
402
403
404
405
406
            {
                PromptInsertion: {
                    0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
                    1: [1, 9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550],  # noqa: E501
                    2: [1, 9833, 28747, 32000, 32000, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550, 1550, 918, 1550],  # noqa: E501
                },
                PromptReplacement: {
                    0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
                    1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],  # noqa: E501
                    2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
                },
            },
407
408
409
410
        ),
    ]
)
# yapf: enable
411
def test_find_update_tokens(
412
413
414
    prompt,
    target_by_key,
    repl_by_key,
415
    expected_by_update_type_mm_count,
416
417
418
419
):
    # Should not be used since there is nothing to convert to tokens
    mock_tokenizer = cast(AnyTokenizer, object())

420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
    for (
            update_type,
            expected_by_mm_count,
    ) in expected_by_update_type_mm_count.items():
        mm_prompt_updates = {
            key:
            [update_type(key, target, repl_by_key[key]).bind(mock_tokenizer)]
            for key, target in target_by_key.items()
        }
        mm_matches = {
            key: find_token_matches(prompt, updates)
            for key, updates in mm_prompt_updates.items()
        }

        for mm_count, expected in expected_by_mm_count.items():
            result = apply_token_matches(
                prompt,
                mm_matches,
                {key: mm_count
                 for key in repl_by_key},
            )

            # Only displayed on error
            print("update_type:", update_type)
            print("mm_count:", mm_count)
            print("mm_matches:", mm_matches)
            print("result:", result)

            # Manually constructed results
            assert result == expected
450
451
452
453
454
455
456


# yapf: disable
@pytest.mark.parametrize(
    "repl_by_key",
    [
        {
457
458
459
            "pattern_1": [32000, 32000],
            "pattern_2": [],
            "pattern_3": [1550, 918, 1550],
460
461
            # Test different modalities having the same tokens (32000)
            "pattern_4": [32000],
462
463
464
465
466
467
468
469
        },
    ],
)
@pytest.mark.parametrize(
    ("prompt", "expected"),
    [
        (
            [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
470
471
            {
                "pattern_1": [
472
                    PlaceholderFeaturesInfo(
473
474
475
                        modality="pattern_1",
                        item_idx=0,
                        start_idx=6,
476
                        tokens=[32000, 32000],
477
478
                    ),
                ],
479
                "pattern_4": [
480
                    PlaceholderFeaturesInfo(
481
482
483
                        modality="pattern_4",
                        item_idx=0,
                        start_idx=3,
484
                        tokens=[32000],
485
486
                    ),
                ],
487
488
            }

489
490
        ),
        (
491
            [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
492
493
            {
                "pattern_1": [
494
                    PlaceholderFeaturesInfo(
495
496
497
                        modality="pattern_1",
                        item_idx=0,
                        start_idx=1,
498
                        tokens=[32000, 32000],
499
                    ),
500
                    PlaceholderFeaturesInfo(
501
502
503
                        modality="pattern_1",
                        item_idx=1,
                        start_idx=5,
504
                        tokens=[32000, 32000],
505
506
507
                    ),
                ],
                "pattern_3": [
508
                    PlaceholderFeaturesInfo(
509
510
511
                        modality="pattern_3",
                        item_idx=0,
                        start_idx=7,
512
                        tokens=[1550, 918, 1550],
513
514
                    ),
                ],
515
                # No match for pattern_4 as it has lower priority than pattern_1
516
            }
517
518
        ),
        (
519
            [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
520
521
            {
                "pattern_1": [
522
                    PlaceholderFeaturesInfo(
523
524
525
                        modality="pattern_1",
                        item_idx=0,
                        start_idx=1,
526
                        tokens=[32000, 32000],
527
                    ),
528
                    PlaceholderFeaturesInfo(
529
530
531
                        modality="pattern_1",
                        item_idx=1,
                        start_idx=3,
532
                        tokens=[32000, 32000],
533
534
                    ),
                ],
535
                "pattern_4": [
536
                    PlaceholderFeaturesInfo(
537
538
539
                        modality="pattern_4",
                        item_idx=0,
                        start_idx=5,
540
                        tokens=[32000],
541
542
                    ),
                ],
543
                "pattern_3": [
544
                    PlaceholderFeaturesInfo(
545
546
547
                        modality="pattern_3",
                        item_idx=0,
                        start_idx=6,
548
                        tokens=[1550, 918, 1550],
549
550
551
                    ),
                ],
            }
552
553
554
        ),
    ]
)
555
@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
556
# yapf: enable
557
def test_find_mm_placeholders(
558
559
560
    repl_by_key,
    prompt,
    expected,
561
    update_type,
562
563
564
565
):
    # Should not be used since there is nothing to convert to tokens
    mock_tokenizer = cast(AnyTokenizer, object())

566
567
    mm_prompt_updates = {
        key: [update_type(key, [], repl).bind(mock_tokenizer)]
568
        for key, repl in repl_by_key.items()
569
    }
570

571
    result = find_mm_placeholders(
572
        mm_prompt_updates,
573
574
575
576
577
        prompt,
        # Effectively match all occurrences in the prompt
        {key: 3
         for key in repl_by_key},
    )
578
579
580

    # Only displayed on error
    print("result:", result)
581
582

    # Manually constructed results
583
    assert result == expected
584
585


586
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
@pytest.mark.parametrize(
    ("limit", "num_supported", "is_valid"),
    [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
     (2, 1, False), (2, 2, True)],
)
def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
    limit_mm_per_prompt = {"image": limit}

    model_config = ModelConfig(
        model=model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="half",
        revision=None,
        limit_mm_per_prompt=limit_mm_per_prompt,
    )

607
    processor = MULTIMODAL_REGISTRY.create_processor(
608
        model_config,
609
        tokenizer=cached_tokenizer_from_config(model_config),
610
    )
611
    profiler = MultiModalProfiler(processor)
612
613

    mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
614
    processor.info.get_supported_mm_limits = mock_supported_mm_limits
615
616
617
618
619
620
621

    if is_valid:
        exc_ctx = nullcontext()
    else:
        exc_ctx = pytest.raises(ValueError, match="this model only supports")

    with exc_ctx:
622
        profiler.get_dummy_data(model_config.max_model_len)
623
624


625
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
@pytest.mark.parametrize(
    ("num_images", "limit", "is_valid"),
    [(0, 0, True), (0, 1, True), (1, 0, False), (1, 1, True), (1, 2, True),
     (2, 1, False), (2, 2, True)],
)
def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
    limit_mm_per_prompt = {"image": limit}

    model_config = ModelConfig(
        model=model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="half",
        revision=None,
        limit_mm_per_prompt=limit_mm_per_prompt,
    )

646
    processor = MULTIMODAL_REGISTRY.create_processor(
647
        model_config,
648
        tokenizer=cached_tokenizer_from_config(model_config),
649
650
651
    )

    rng = np.random.RandomState(0)
652
    image = random_image(rng, min_wh=128, max_wh=256)
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
    if num_images == 0:
        mm_data = {}
    elif num_images == 1:
        mm_data = {"image": image}
    else:
        mm_data = {"image": [image] * num_images}

    if is_valid:
        exc_ctx = nullcontext()
    else:
        exc_ctx = pytest.raises(ValueError, match=f"passed {num_images} image")

    with exc_ctx:
        processor.apply(
            "<image>" * num_images,
            mm_data=mm_data,
            hf_processor_mm_kwargs={},
        )
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693


class _ProcessorProxy:

    def __init__(self, processor: ProcessorMixin) -> None:
        super().__init__()

        self.__processor = processor

    def __getattr__(self, key: str):
        return getattr(self.__processor, key)

    def __call__(
        self,
        text=None,
        images=None,
        videos=None,
        exists=None,
        return_tensors=None,
    ):
        return dict(exists=exists)


694
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])  # Dummy
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
# yapf: disable
@pytest.mark.parametrize(
    ("call_kwargs", "expected_kwargs"),
    [
        # Should ignore invalid kwargs
        ({"does_not_exist": 100}, {"exists": None}),
        ({"exists": 1}, {"exists": 1}),
        ({"does_not_exist": 100, "exists": 1}, {"exists": 1}),
    ],
)
# yapf: enable
def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs):
    model_config = ModelConfig(
        model=model_id,
        task="auto",
        tokenizer=model_id,
        tokenizer_mode="auto",
        trust_remote_code=False,
        seed=0,
        dtype="half",
        revision=None,
    )

    processor = MULTIMODAL_REGISTRY.create_processor(
        model_config,
720
        tokenizer=cached_tokenizer_from_config(model_config),
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
    )
    orig_get_hf_processor = processor.info.get_hf_processor

    def get_hf_processor(self, **kwargs):
        assert kwargs == call_kwargs
        return _ProcessorProxy(orig_get_hf_processor())

    processor.info.get_hf_processor = MethodType(get_hf_processor,
                                                 processor.info)

    out_kwargs = processor._call_hf_processor(
        prompt="",
        mm_data={},
        mm_kwargs=call_kwargs,
    )

    assert out_kwargs == expected_kwargs