vision_language.py 63.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""
Cyrus Leung's avatar
Cyrus Leung committed
4
5
This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.
6
7
8
9

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
10

11
import os
12
import random
13
from contextlib import contextmanager
14
from dataclasses import asdict
15
from typing import NamedTuple
16

17
from huggingface_hub import snapshot_download
18
from transformers import AutoProcessor, AutoTokenizer
19

20
from vllm import LLM, EngineArgs, SamplingParams
21
from vllm.assets.image import ImageAsset
22
from vllm.assets.video import VideoAsset
23
from vllm.lora.request import LoRARequest
24
from vllm.multimodal.image import convert_image_mode
25
from vllm.utils.argparse_utils import FlexibleArgumentParser
26

27
28
29
30

class ModelRequestData(NamedTuple):
    engine_args: EngineArgs
    prompts: list[str]
31
32
    stop_token_ids: list[int] | None = None
    lora_requests: list[LoRARequest] | None = None
33
    sampling_params: list[SamplingParams] | None = None
34
35


36
37
38
39
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
# lower-end GPUs.
# Unless specified, these settings have been tested to work on a single L4.

40

41
# Aria
42
def run_aria(questions: list[str], modality: str) -> ModelRequestData:
43
44
45
    assert modality == "image"
    model_name = "rhymes-ai/Aria"

46
    # NOTE: Need L40 (or equivalent) to avoid OOM
47
48
49
50
51
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        dtype="bfloat16",
52
        limit_mm_per_prompt={modality: 1},
53
    )
54

55
56
57
58
59
60
61
    prompts = [
        (
            f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
            "<|im_end|>\n<|im_start|>assistant\n"
        )
        for question in questions
    ]
62
63

    stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
64
65
66
67
68
69

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
70
71


Jennifer Zhao's avatar
Jennifer Zhao committed
72
73
74
# Aya Vision
def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
75
    model_name = "CohereLabs/aya-vision-8b"
Jennifer Zhao's avatar
Jennifer Zhao committed
76
77
78
79
80
81

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        mm_processor_kwargs={"crop_to_patches": True},
82
        limit_mm_per_prompt={modality: 1},
Jennifer Zhao's avatar
Jennifer Zhao committed
83
84
85
86
87
88
89
90
91
92
93
    )
    prompts = [
        f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
        for question in questions
    ]
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
# Bee-8B
def run_bee(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "Open-Bee/Bee-8B-RL"

    prompts = [
        (
            f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<image>\n{question}<|im_end|>"
            f"<|im_start|>assistant\n<think>\n"
        )
        for question in questions
    ]

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=16384,
        limit_mm_per_prompt={modality: 1},
        trust_remote_code=True,
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
def run_bagel(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "ByteDance-Seed/BAGEL-7B-MoT"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
        max_num_seqs=2,
        limit_mm_per_prompt={modality: 1},
    )

    prompts = [
        (
            f"<|im_start|>user\n<|image_pad|>\n{question}<|im_end|>\n"
            f"<|im_start|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


147
# BLIP-2
148
def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
149
150
151
152
    assert modality == "image"

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
153
    prompts = [f"Question: {question} Answer:" for question in questions]
154
    engine_args = EngineArgs(
155
        model="Salesforce/blip2-opt-2.7b",
156
        limit_mm_per_prompt={modality: 1},
157
158
159
160
161
162
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
163
164
165


# Chameleon
166
def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:
167
168
    assert modality == "image"

169
    prompts = [f"{question}<image>" for question in questions]
170
171
172
173
    engine_args = EngineArgs(
        model="facebook/chameleon-7b",
        max_model_len=4096,
        max_num_seqs=2,
174
        limit_mm_per_prompt={modality: 1},
175
176
177
178
179
180
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
181
182


183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
def run_command_a_vision(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "CohereLabs/command-a-vision-07-2025"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=32768,
        tensor_parallel_size=4,
        limit_mm_per_prompt={modality: 1},
    )

    prompts = [
        f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><|IMG_PATCH|>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


206
# Deepseek-VL2
207
def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:
208
209
    assert modality == "image"

210
    model_name = "deepseek-ai/deepseek-vl2-tiny"
211

212
213
214
215
216
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
217
        limit_mm_per_prompt={modality: 1},
218
    )
219

220
    prompts = [
221
        f"<|User|>: <image>\n{question}\n\n<|Assistant|>:" for question in questions
222
    ]
223
224
225
226
227

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
228
229


230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
def run_deepseek_ocr(questions: list[str], modality: str) -> ModelRequestData:
    from vllm.model_executor.models.deepseek_ocr import NGramPerReqLogitsProcessor

    assert modality == "image"

    model_name = "deepseek-ai/DeepSeek-OCR"

    engine_args = EngineArgs(
        model=model_name,
        limit_mm_per_prompt={modality: 1},
        logits_processors=[NGramPerReqLogitsProcessor],
    )

    # deepseek-ocr use plain prompt template
    prompts = [f"<image>\n{question}" for question in questions]

    # The following sampling params config is taken from
    # the official Deepseek-OCR inference example.
    # (IMPORTANT) Use the custom logits processor and avoid skipping
    # special tokens for this model for the optimal OCR performance.
    sampling_params = [
        SamplingParams(
            temperature=0.0,
            max_tokens=8192,
            # ngram logit processor args
            extra_args=dict(
                ngram_size=30,
                window_size=90,
                # whitelist: <td>, </td>
                whitelist_token_ids={128821, 128822},
            ),
            skip_special_tokens=False,
        )
        for _ in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        sampling_params=sampling_params,
    )


# Dots-OCR
def run_dots_ocr(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    prompts = [f"<|img|><|imgpad|><|endofimg|>{question}" for question in questions]
    engine_args = EngineArgs(
        model="rednote-hilab/dots.ocr",
        limit_mm_per_prompt={modality: 1},
        trust_remote_code=True,
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
# Ernie4.5-VL
def run_ernie45_vl(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "baidu/ERNIE-4.5-VL-28B-A3B-PT"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        limit_mm_per_prompt={modality: 1},
        trust_remote_code=True,
    )

    if modality == "image":
        placeholder = "Picture 1:<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>"
    elif modality == "video":
        placeholder = "Video 1:<|VIDEO_START|><|video@placeholder|><|VIDEO_END|>"

    prompts = [
        (
            f"<|begin_of_sentence|>User: {question}{placeholder}\n"
            "Assistant: <think></think>"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


321
# Fuyu
322
def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:
323
324
    assert modality == "image"

325
    prompts = [f"{question}\n" for question in questions]
326
327
328
329
    engine_args = EngineArgs(
        model="adept/fuyu-8b",
        max_model_len=2048,
        max_num_seqs=2,
330
        limit_mm_per_prompt={modality: 1},
331
332
333
334
335
336
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
337
338


339
# Gemma 3
340
def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:
341
342
343
    assert modality == "image"
    model_name = "google/gemma-3-4b-it"

344
    engine_args = EngineArgs(
345
346
347
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
348
        mm_processor_kwargs={"do_pan_and_scan": True},
349
        limit_mm_per_prompt={modality: 1},
350
    )
351

352
353
354
355
356
357
358
359
    prompts = [
        (
            "<bos><start_of_turn>user\n"
            f"<start_of_image>{question}<end_of_turn>\n"
            "<start_of_turn>model\n"
        )
        for question in questions
    ]
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
360
361
362
363
364
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )

365

Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
# Gemma3N
def run_gemma3n(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "google/gemma-3n-E2B-it"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        limit_mm_per_prompt={modality: 1},
        enforce_eager=True,
    )

    prompts = [
        (
            "<start_of_turn>user\n"
            f"<image_soft_token>{question}<end_of_turn>\n"
            "<start_of_turn>model\n"
        )
        for question in questions
    ]
387
388
389
390
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
391
392


393
# GLM-4v
394
def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:
395
    assert modality == "image"
396
    model_name = "zai-org/glm-4v-9b"
397

398
399
400
401
402
403
404
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        trust_remote_code=True,
        enforce_eager=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
405
        limit_mm_per_prompt={modality: 1},
406
    )
407

408
    prompts = [
409
410
411
412
        (
            "<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>"
            f"{question}<|assistant|>"
        )
413
        for question in questions
414
    ]
415

416
    stop_token_ids = [151329, 151336, 151338]
417
418
419
420
421
422

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
423
424


425
426
# GLM-4.1V
def run_glm4_1v(questions: list[str], modality: str) -> ModelRequestData:
427
    model_name = "zai-org/GLM-4.1V-9B-Thinking"
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        mm_processor_kwargs={
            "size": {"shortest_edge": 12544, "longest_edge": 47040000},
            "fps": 1,
        },
        limit_mm_per_prompt={modality: 1},
        enforce_eager=True,
    )

    if modality == "image":
        placeholder = "<|begin_of_image|><|image|><|end_of_image|>"
    elif modality == "video":
        placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

    prompts = [
        (
            "[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
            f"{placeholder}"
            f"{question}<|assistant|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
# GLM-4.5V
def run_glm4_5v(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "zai-org/GLM-4.5V"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        mm_processor_kwargs={
            "size": {"shortest_edge": 12544, "longest_edge": 47040000},
            "fps": 1,
        },
        limit_mm_per_prompt={modality: 1},
        enforce_eager=True,
        tensor_parallel_size=4,
    )

    if modality == "image":
        placeholder = "<|begin_of_image|><|image|><|end_of_image|>"
    elif modality == "video":
        placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

    prompts = [
        (
            "[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
            f"{placeholder}"
            f"{question}<|assistant|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


# GLM-4.5V-FP8
def run_glm4_5v_fp8(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "zai-org/GLM-4.5V-FP8"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        mm_processor_kwargs={
            "size": {"shortest_edge": 12544, "longest_edge": 47040000},
            "fps": 1,
        },
        limit_mm_per_prompt={modality: 1},
        enforce_eager=True,
        tensor_parallel_size=4,
    )

    if modality == "image":
        placeholder = "<|begin_of_image|><|image|><|end_of_image|>"
    elif modality == "video":
        placeholder = "<|begin_of_video|><|video|><|end_of_video|>"

    prompts = [
        (
            "[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
            f"{placeholder}"
            f"{question}<|assistant|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


535
# H2OVL-Mississippi
536
def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
537
538
    assert modality == "image"

539
    model_name = "h2oai/h2ovl-mississippi-800m"
540

541
    engine_args = EngineArgs(
542
543
544
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
545
        limit_mm_per_prompt={modality: 1},
546
547
    )

548
549
550
551
552
553
554
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
555
556

    # Stop tokens for H2OVL-Mississippi
557
    # https://huggingface.co/h2oai/h2ovl-mississippi-800m
558
    stop_token_ids = [tokenizer.eos_token_id]
559
560
561
562
563
564

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
565
566


567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
# HunyuanOCR
def run_hunyuan_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "tencent/HunyuanOCR"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        limit_mm_per_prompt={modality: 1},
    )

    placeholder = "<|hy_place▁holder▁no▁100|><|hy_place▁holder▁no▁102|><|hy_place▁holder▁no▁101|>"  # noqa: E501
    prompts = [
        f"<|hy_begin▁of▁sentence|>{placeholder}{question}<|hy_User|>"
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=None,
    )


592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
# naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B
def run_hyperclovax_seed_vision(
    questions: list[str], modality: str
) -> ModelRequestData:
    model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192 if modality == "image" else 16384,
        limit_mm_per_prompt={modality: 1},
    )

    messages = list()
    for question in questions:
        if modality == "image":
            """
610
611
            ocr: List the words in the image in raster order.
                Even if the word order feels unnatural for reading,
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
                the model will handle it as long as it follows raster order.
                e.g. "Naver, CLOVA, bigshane"
            lens_keywords: List the entity names in the image.
                e.g. "iPhone"
            lens_local_keywords: List the entity names with quads in the image.
                e.g. "[0.07, 0.21, 0.92, 0.90] iPhone"
            """
            messages.append(
                [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "image",
                                "ocr": "",
                                "lens_keywords": "",
                                "lens_local_keywords": "",
                            },
                            {
                                "type": "text",
                                "text": question,
                            },
                        ],
                    }
                ]
            )
        elif modality == "video":
            messages.append(
                [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "video",
                            },
                            {
                                "type": "text",
                                "text": question,
                            },
                        ],
                    }
                ]
            )
        else:
            raise ValueError(f"Unsupported modality: {modality}")

    prompts = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True,
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=None,
    )


671
# Idefics3-8B-Llama3
672
def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
673
674
675
    assert modality == "image"
    model_name = "HuggingFaceM4/Idefics3-8B-Llama3"

676
    engine_args = EngineArgs(
677
678
679
680
681
682
683
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        # if you are running out of memory, you can reduce the "longest_edge".
        # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
        mm_processor_kwargs={
684
            "size": {"longest_edge": 3 * 364},
685
        },
686
        limit_mm_per_prompt={modality: 1},
687
    )
688
689
690
691
    prompts = [
        (f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]
692
693
694
695
696

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
697
698


Lyu Han's avatar
Lyu Han committed
699
700
# Intern-S1
def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
701
    model_name = "internlm/Intern-S1-mini"
Lyu Han's avatar
Lyu Han committed
702
703
704
705
706
707
708
709
710
711

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
        max_num_seqs=2,
        limit_mm_per_prompt={modality: 1},
        enforce_eager=True,
    )

712
713
714
715
716
    if modality == "image":
        placeholder = "<IMG_CONTEXT>"
    elif modality == "video":
        placeholder = "<video>"

Lyu Han's avatar
Lyu Han committed
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"{placeholder}\n{question}"}]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


732
# InternVL
733
def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
734
    model_name = "OpenGVLab/InternVL3-2B"
735

736
    engine_args = EngineArgs(
737
738
        model=model_name,
        trust_remote_code=True,
739
        max_model_len=8192,
740
        limit_mm_per_prompt={modality: 1},
741
742
    )

743
744
745
746
747
    if modality == "image":
        placeholder = "<image>"
    elif modality == "video":
        placeholder = "<video>"

748
749
750
751
752
753
754
755
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"{placeholder}\n{question}"}]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
756
757
758
759
760
761
762

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
763
    stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
764
765
766
767
768
769

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
770
771


772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
# Kanana-V
def run_kanana_v(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "kakaocorp/kanana-1.5-v-3b-instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        trust_remote_code=True,
        limit_mm_per_prompt={modality: 1},
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
# Keye-VL
def run_keye_vl(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "Kwai-Keye/Keye-VL-8B-Preview"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        trust_remote_code=True,
        limit_mm_per_prompt={modality: 1},
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

    prompts = [
        (
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
# Keye-VL-1.5
def run_keye_vl1_5(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "Kwai-Keye/Keye-VL-1.5-8B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        trust_remote_code=True,
        limit_mm_per_prompt={modality: 1},
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

    prompts = [
        (
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


861
862
863
864
865
866
867
# Kimi-VL
def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    prompts = [
        "<|im_user|>user<|im_middle|><|media_start|>image<|media_content|>"
        f"<|media_pad|><|media_end|>{question}<|im_end|>"
868
869
        "<|im_assistant|>assistant<|im_middle|>"
        for question in questions
870
871
872
873
874
    ]

    engine_args = EngineArgs(
        model="moonshotai/Kimi-VL-A3B-Instruct",
        trust_remote_code=True,
Cyrus Leung's avatar
Cyrus Leung committed
875
        max_model_len=4096,
876
        limit_mm_per_prompt={modality: 1},
877
878
879
880
881
882
883
884
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
# LightOnOCR
def run_lightonocr(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    prompts = [
        "<|im_start|>system<|im_end|>\n<|im_start|>user\n<|image_pad|><|im_end|>\n<|im_start|>assistant\n"
        for _ in questions
    ]

    engine_args = EngineArgs(
        model="lightonai/LightOnOCR-1B",
        limit_mm_per_prompt={modality: 1},
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
def run_lfm2_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "LiquidAI/LFM2-VL-450M"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        limit_mm_per_prompt={modality: 1},
    )

    processor = AutoProcessor.from_pretrained(model_name)
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": question}],
            }
        ]
        for question in questions
    ]
    prompts = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
def run_llama4(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=4,
        tensor_parallel_size=8,
        gpu_memory_utilization=0.4,
        limit_mm_per_prompt={modality: 1},
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name)
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": f"{question}"}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )
    stop_token_ids = None
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )


971
# LLaVA-1.5
972
def run_llava(questions: list[str], modality: str) -> ModelRequestData:
973
    assert modality == "image"
974

975
    prompts = [f"USER: <image>\n{question}\nASSISTANT:" for question in questions]
976

977
978
979
    engine_args = EngineArgs(
        model="llava-hf/llava-1.5-7b-hf",
        max_model_len=4096,
980
        limit_mm_per_prompt={modality: 1},
981
982
983
984
985
986
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
987
988
989


# LLaVA-1.6/LLaVA-NeXT
990
def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:
991
    assert modality == "image"
992

993
    prompts = [f"[INST] <image>\n{question} [/INST]" for question in questions]
994
995
996
    engine_args = EngineArgs(
        model="llava-hf/llava-v1.6-mistral-7b-hf",
        max_model_len=8192,
997
        limit_mm_per_prompt={modality: 1},
998
999
1000
1001
1002
1003
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1004
1005
1006
1007


# LlaVA-NeXT-Video
# Currently only support for video input
1008
def run_llava_next_video(questions: list[str], modality: str) -> ModelRequestData:
1009
1010
    assert modality == "video"

1011
    prompts = [f"USER: <video>\n{question} ASSISTANT:" for question in questions]
1012
1013
1014
    engine_args = EngineArgs(
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        max_model_len=8192,
1015
        max_num_seqs=2,
1016
        limit_mm_per_prompt={modality: 1},
1017
1018
1019
1020
1021
1022
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1023
1024


1025
# LLaVA-OneVision
1026
def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData:
1027
    if modality == "video":
1028
        prompts = [
1029
            f"<|im_start|>user <video>\n{question}<|im_end|><|im_start|>assistant\n"
1030
            for question in questions
1031
        ]
1032
1033

    elif modality == "image":
1034
        prompts = [
1035
            f"<|im_start|>user <image>\n{question}<|im_end|><|im_start|>assistant\n"
1036
            for question in questions
1037
        ]
1038

1039
1040
1041
    engine_args = EngineArgs(
        model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
        max_model_len=16384,
1042
        limit_mm_per_prompt={modality: 1},
1043
1044
1045
1046
1047
1048
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1049
1050


1051
# Mantis
1052
def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
1053
    assert modality == "image"
1054

1055
1056
    llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"  # noqa: E501
    prompts = [llama3_template.format(f"{question}\n<image>") for question in questions]
1057

1058
    engine_args = EngineArgs(
1059
        model="TIGER-Lab/Mantis-8B-siglip-llama3",
1060
        max_model_len=4096,
1061
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
1062
        limit_mm_per_prompt={modality: 1},
1063
    )
1064
    stop_token_ids = [128009]
1065
1066
1067
1068
1069
1070

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
1071
1072
1073


# MiniCPM-V
1074
def run_minicpmv_base(questions: list[str], modality: str, model_name):
1075
1076
    assert modality in ["image", "video"]
    # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
1077
1078
1079
1080
1081
1082
1083

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
1084
1085
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

1086
    # 2.6
1087
1088
1089
1090
1091
1092
1093
1094
1095
    # model_name = "openbmb/MiniCPM-V-2_6"
    # o2.6

    # modality supports
    # 2.0: image
    # 2.5: image
    # 2.6: image, video
    # o2.6: image, video, audio
    # model_name = "openbmb/MiniCPM-o-2_6"
1096
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1097
    engine_args = EngineArgs(
1098
        model=model_name,
1099
1100
        max_model_len=4096,
        max_num_seqs=2,
1101
        trust_remote_code=True,
1102
        limit_mm_per_prompt={modality: 1},
1103
    )
1104
1105
1106
1107
1108
1109
1110
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

1111
    # 2.6 / o2.6
1112
    stop_tokens = ["<|im_end|>", "<|endoftext|>"]
1113
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
1114

1115
1116
1117
1118
1119
    modality_placeholder = {
        "image": "(<image>./</image>)",
        "video": "(<video>./</video>)",
    }

1120
1121
    prompts = [
        tokenizer.apply_chat_template(
1122
1123
1124
1125
1126
1127
            [
                {
                    "role": "user",
                    "content": f"{modality_placeholder[modality]}\n{question}",
                }
            ],
1128
            tokenize=False,
1129
1130
1131
            add_generation_prompt=True,
        )
        for question in questions
1132
    ]
1133
1134
1135
1136
1137
1138

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
1139
1140


1141
def run_minicpmo(questions: list[str], modality: str) -> ModelRequestData:
1142
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-o-2_6")
1143
1144


1145
def run_minicpmv(questions: list[str], modality: str) -> ModelRequestData:
1146
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-V-2_6")
1147
1148


1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
def run_minimax_vl_01(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "MiniMaxAI/MiniMax-VL-01"

    engine_args = EngineArgs(
        model=model_name,
        max_num_seqs=2,
        limit_mm_per_prompt={modality: 1},
        trust_remote_code=True,
        tensor_parallel_size=8,
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name)
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": question}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
# Mistral-3 HF-format
def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"

    # NOTE: Need L40 (or equivalent) to avoid OOM
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        tensor_parallel_size=2,
1194
        limit_mm_per_prompt={modality: 1},
1195
        ignore_patterns=["consolidated.safetensors"],
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
    )

    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1206
1207
# Molmo
def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
1208
1209
    assert modality == "image"

1210
    model_name = "allenai/Molmo-7B-D-0924"
1211
1212
1213

    engine_args = EngineArgs(
        model=model_name,
1214
1215
        trust_remote_code=True,
        dtype="bfloat16",
1216
        limit_mm_per_prompt={modality: 1},
1217
1218
    )

1219
    prompts = [
1220
        f"<|im_start|>user <image>\n{question}<|im_end|><|im_start|>assistant\n"
1221
1222
        for question in questions
    ]
1223

1224
1225
1226
1227
1228
1229
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
# Molmo2
def run_molmo2(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "allenai/Molmo2-8B"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        dtype="bfloat16",
        limit_mm_per_prompt={modality: 1},
        max_num_batched_tokens=36864,
    )

    if modality == "image":
        placeholder = "<|image|>"
    elif modality == "video":
        placeholder = "<|video|>"
    else:
        raise ValueError(f"Unsupported modality for molmo2: {modality}")

    prompts = [
        f"{placeholder}<|im_start|>user\n{question}<|im_end|>\n<|im_start|>assistant\n"
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1260
1261
1262
# Nemontron_VL
def run_nemotron_vl(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"
1263

1264
    engine_args = EngineArgs(
1265
        model=model_name,
1266
        trust_remote_code=True,
1267
        max_model_len=8192,
1268
        limit_mm_per_prompt={modality: 1},
1269
    )
1270

1271
1272
1273
1274
1275
1276
    assert modality == "image"
    placeholder = "<image>"

    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"{placeholder}\n{question}"}]
1277
        for question in questions
1278
    ]
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
    stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
1290
1291
1292
1293

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
1294
        stop_token_ids=stop_token_ids,
1295
    )
1296
1297


1298
# NVLM-D
1299
def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:
1300
1301
1302
1303
1304
    assert modality == "image"

    model_name = "nvidia/NVLM-D-72B"

    # Adjust this as necessary to fit in GPU
1305
    engine_args = EngineArgs(
1306
1307
1308
1309
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        tensor_parallel_size=4,
1310
        limit_mm_per_prompt={modality: 1},
1311
1312
    )

1313
1314
1315
1316
1317
1318
1319
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
1320
1321
1322
1323
1324

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1325
1326


1327
1328
# Ovis
def run_ovis(questions: list[str], modality: str) -> ModelRequestData:
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
    assert modality == "image"

    model_name = "AIDC-AI/Ovis2-1B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        trust_remote_code=True,
        dtype="half",
1339
        limit_mm_per_prompt={modality: 1},
1340
1341
    )

1342
1343
1344
1345
1346
1347
1348
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
1349
1350
1351
1352
1353
1354
1355

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
# Ovis2_5
def run_ovis2_5(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "AIDC-AI/Ovis2.5-2B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        trust_remote_code=True,
        dtype="half",
        limit_mm_per_prompt={modality: 1},
    )
    if modality == "image":
        placeholder = "<image>"
    elif modality == "video":
        placeholder = "<video>"

1373
1374
    prompts = [
        f"<|im_start|>user\n\n{placeholder}\n{question}<|im_end|>\n<|im_start|>assistant\n"
1375
1376
1377
1378
1379
1380
1381
1382
1383
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
# PaddleOCR-VL
def run_paddleocr_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "PaddlePaddle/PaddleOCR-VL"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        limit_mm_per_prompt={modality: 1},
        trust_remote_code=True,
    )

    placeholder = "<|IMAGE_START|><|IMAGE_PLACEHOLDER|><|IMAGE_END|>"
    prompts = [
        (f"<|begin_of_sentence|>User: {question}{placeholder}\nAssistant: ")
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1410
# PaliGemma
1411
def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:
1412
    assert modality == "image"
1413

1414
    # PaliGemma has special prompt format for VQA
1415
1416
1417
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma-3b-mix-224",
1418
        limit_mm_per_prompt={modality: 1},
1419
    )
1420
1421
1422
1423
1424

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1425
1426


1427
# PaliGemma 2
1428
def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:
1429
    assert modality == "image"
1430

1431
    # PaliGemma 2 has special prompt format for VQA
1432
1433
1434
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma2-3b-ft-docci-448",
1435
        limit_mm_per_prompt={modality: 1},
1436
    )
1437
1438
1439
1440
1441

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1442
1443


1444
# Phi-3-Vision
1445
def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
1446
1447
    assert modality == "image"

1448
1449
1450
1451
    prompts = [
        f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
        for question in questions
    ]
1452

1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
1465
    engine_args = EngineArgs(
1466
1467
        model="microsoft/Phi-3.5-vision-instruct",
        trust_remote_code=True,
1468
        max_model_len=4096,
1469
        max_num_seqs=2,
1470
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
1471
        mm_processor_kwargs={"num_crops": 16},
1472
        limit_mm_per_prompt={modality: 1},
1473
    )
1474
1475
1476
1477
1478

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1479
1480


1481
# Phi-4-multimodal-instruct
1482
def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
    """
    Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
    show how to process image inputs.
    """
    assert modality == "image"
    model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
    # Since the vision-lora and speech-lora co-exist with the base model,
    # we have to manually specify the path of the lora weights.
    vision_lora_path = os.path.join(model_path, "vision-lora")
    prompts = [
1493
        f"<|user|><|image_1|>{question}<|end|><|assistant|>" for question in questions
1494
    ]
1495
    engine_args = EngineArgs(
1496
1497
        model=model_path,
        trust_remote_code=True,
1498
        max_model_len=5120,
1499
        max_num_seqs=2,
1500
        max_num_batched_tokens=12800,
1501
1502
        enable_lora=True,
        max_lora_rank=320,
1503
1504
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
        mm_processor_kwargs={"dynamic_hd": 16},
1505
        limit_mm_per_prompt={modality: 1},
1506
1507
    )

1508
1509
1510
1511
1512
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
    )
1513
1514


1515
# Pixtral HF-format
1516
def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
1517
1518
1519
1520
    assert modality == "image"

    model_name = "mistral-community/pixtral-12b"

1521
    # NOTE: Need L40 (or equivalent) to avoid OOM
1522
    engine_args = EngineArgs(
1523
        model=model_name,
1524
        max_model_len=6144,
1525
        max_num_seqs=2,
1526
        limit_mm_per_prompt={modality: 1},
1527
1528
    )

1529
    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
1530
1531
1532
1533
1534

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1535
1536


1537
# Qwen-VL
1538
def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:
1539
1540
    assert modality == "image"

1541
    engine_args = EngineArgs(
1542
        model="Qwen/Qwen-VL",
1543
        trust_remote_code=True,
1544
1545
        max_model_len=1024,
        max_num_seqs=2,
1546
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
1547
        limit_mm_per_prompt={modality: 1},
1548
1549
    )

1550
    prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]
1551
1552
1553
1554
1555

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1556
1557


1558
# Qwen2-VL
1559
def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:
1560
    model_name = "Qwen/Qwen2-VL-7B-Instruct"
1561

1562
    engine_args = EngineArgs(
1563
        model=model_name,
1564
1565
1566
        max_model_len=4096,
        max_num_seqs=5,
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
1567
        mm_processor_kwargs={
1568
1569
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
1570
        },
1571
        limit_mm_per_prompt={modality: 1},
1572
    )
1573

1574
1575
1576
1577
1578
    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

1579
    prompts = [
1580
1581
1582
1583
1584
1585
1586
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
1587
    ]
1588
1589
1590
1591
1592

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
1593
1594


Roger Wang's avatar
Roger Wang committed
1595
# Qwen2.5-VL
1596
def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
Roger Wang's avatar
Roger Wang committed
1597
1598
    model_name = "Qwen/Qwen2.5-VL-3B-Instruct"

1599
    engine_args = EngineArgs(
Roger Wang's avatar
Roger Wang committed
1600
1601
1602
1603
1604
1605
1606
1607
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": 1,
        },
1608
        limit_mm_per_prompt={modality: 1},
Roger Wang's avatar
Roger Wang committed
1609
1610
1611
1612
1613
1614
1615
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

1616
    prompts = [
1617
1618
1619
1620
1621
1622
1623
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
1624
    ]
1625
1626
1627
1628
1629

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
Roger Wang's avatar
Roger Wang committed
1630
1631


1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
# Qwen2.5-Omni
def run_qwen2_5_omni(questions: list[str], modality: str):
    model_name = "Qwen/Qwen2.5-Omni-7B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
1643
            "fps": 1,
1644
        },
1645
        limit_mm_per_prompt={modality: 1},
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
    )

    if modality == "image":
        placeholder = "<|IMAGE|>"
    elif modality == "video":
        placeholder = "<|VIDEO|>"

    default_system = (
        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
        "Group, capable of perceiving auditory and visual inputs, as well as "
1656
1657
        "generating text and speech."
    )
1658

1659
1660
1661
1662
1663
1664
1665
1666
1667
    prompts = [
        (
            f"<|im_start|>system\n{default_system}<|im_end|>\n"
            f"<|im_start|>user\n<|vision_bos|>{placeholder}<|vision_eos|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]
1668
1669
1670
1671
1672
1673
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
# Qwen3-VL-Dense
def run_qwen3_vl(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "Qwen/Qwen3-VL-4B-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": 1,
        },
        limit_mm_per_prompt={modality: 1},
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

    prompts = [
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


# Qwen3-VL-MOE
def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "Qwen/Qwen3-VL-30B-A3B-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": 1,
        },
        limit_mm_per_prompt={modality: 1},
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

    prompts = [
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
# R-4B
def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "YannQi/R-4B"

    prompts = [
        f"<|im_start|>user <image>\n{question}<|im_end|><|im_start|>assistant\n"
        for question in questions
    ]

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=16384,
        limit_mm_per_prompt={modality: 1},
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1770
1771
# SkyworkR1V
def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
1772
    assert modality == "image"
1773
1774

    model_name = "Skywork/Skywork-R1V-38B"
1775
1776
1777
1778
1779
1780
1781

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        limit_mm_per_prompt={modality: 1},
    )
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794

    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    # Stop tokens for SkyworkR1V
    # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
    stop_tokens = ["<|end▁of▁sentence|>", "<|endoftext|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
1795
1796
1797
1798

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
1799
        stop_token_ids=stop_token_ids,
1800
1801
1802
    )


1803
1804
1805
1806
# SmolVLM2-2.2B-Instruct
def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
1807
1808
1809

    engine_args = EngineArgs(
        model=model_name,
1810
1811
1812
1813
1814
1815
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        mm_processor_kwargs={
            "max_image_size": {"longest_edge": 384},
        },
1816
1817
        limit_mm_per_prompt={modality: 1},
    )
1818
1819
1820
1821
    prompts = [
        (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]
1822

1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


# Step3
def run_step3(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "stepfun-ai/step3-fp8"

    # NOTE: Below are verified configurations for step3-fp8
    # on 8xH100 GPUs.
    engine_args = EngineArgs(
        model=model_name,
        max_num_batched_tokens=4096,
        gpu_memory_utilization=0.85,
        tensor_parallel_size=8,
        limit_mm_per_prompt={modality: 1},
        reasoning_parser="step3",
    )
1845
1846

    prompts = [
1847
1848
        "<|begin▁of▁sentence|> You are a helpful assistant. <|BOT|>user\n "
        f"<im_patch>{question} <|EOT|><|BOT|>assistant\n<think>\n"
1849
1850
1851
1852
1853
1854
1855
1856
1857
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1858
1859
# omni-research/Tarsier-7b
def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
1860
    assert modality == "image"
1861
    model_name = "omni-research/Tarsier-7b"
1862
1863
1864
1865
1866

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
1867
        limit_mm_per_prompt={modality: 1},
1868
    )
1869
    prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
1870

1871
1872
1873
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
1874
    )
1875

1876
1877
1878
1879
1880
1881
1882

def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
    model_name = "omni-research/Tarsier2-Recap-7b"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
1883
1884
1885
1886
        hf_overrides={
            "architectures": ["Tarsier2ForConditionalGeneration"],
            "model_type": "tarsier2",
        },
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
        limit_mm_per_prompt={modality: 1},
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

    prompts = [
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]
1904
1905
1906
1907
1908
1909
1910

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1911
model_example_map = {
1912
    "aria": run_aria,
Jennifer Zhao's avatar
Jennifer Zhao committed
1913
    "aya_vision": run_aya_vision,
1914
    "bagel": run_bagel,
1915
    "bee": run_bee,
1916
1917
    "blip-2": run_blip2,
    "chameleon": run_chameleon,
1918
    "command_a_vision": run_command_a_vision,
1919
    "deepseek_vl_v2": run_deepseek_vl2,
1920
1921
    "deepseek_ocr": run_deepseek_ocr,
    "dots_ocr": run_dots_ocr,
1922
    "ernie45_vl": run_ernie45_vl,
1923
    "fuyu": run_fuyu,
1924
    "gemma3": run_gemma3,
Nicolò Lucchesi's avatar
Nicolò Lucchesi committed
1925
    "gemma3n": run_gemma3n,
1926
    "glm4v": run_glm4v,
1927
    "glm4_1v": run_glm4_1v,
1928
1929
    "glm4_5v": run_glm4_5v,
    "glm4_5v_fp8": run_glm4_5v_fp8,
1930
    "h2ovl_chat": run_h2ovl,
1931
    "hunyuan_vl": run_hunyuan_vl,
1932
    "hyperclovax_seed_vision": run_hyperclovax_seed_vision,
1933
    "idefics3": run_idefics3,
Lyu Han's avatar
Lyu Han committed
1934
    "interns1": run_interns1,
1935
    "internvl_chat": run_internvl,
1936
    "kanana_v": run_kanana_v,
1937
    "keye_vl": run_keye_vl,
1938
    "keye_vl1_5": run_keye_vl1_5,
1939
    "kimi_vl": run_kimi_vl,
1940
    "lightonocr": run_lightonocr,
1941
    "lfm2_vl": run_lfm2_vl,
1942
    "llama4": run_llama4,
1943
1944
    "llava": run_llava,
    "llava-next": run_llava_next,
1945
    "llava-next-video": run_llava_next_video,
1946
    "llava-onevision": run_llava_onevision,
1947
    "mantis": run_mantis,
1948
    "minicpmo": run_minicpmo,
1949
    "minicpmv": run_minicpmv,
1950
    "minimax_vl_01": run_minimax_vl_01,
1951
    "mistral3": run_mistral3,
1952
    "molmo": run_molmo,
1953
    "molmo2": run_molmo2,
1954
    "nemotron_vl": run_nemotron_vl,
1955
    "NVLM_D": run_nvlm_d,
1956
    "ovis": run_ovis,
1957
    "ovis2_5": run_ovis2_5,
1958
    "paddleocr_vl": run_paddleocr_vl,
1959
1960
1961
    "paligemma": run_paligemma,
    "paligemma2": run_paligemma2,
    "phi3_v": run_phi3v,
1962
    "phi4_mm": run_phi4mm,
1963
    "pixtral_hf": run_pixtral_hf,
1964
    "qwen_vl": run_qwen_vl,
1965
    "qwen2_vl": run_qwen2_vl,
Roger Wang's avatar
Roger Wang committed
1966
    "qwen2_5_vl": run_qwen2_5_vl,
1967
    "qwen2_5_omni": run_qwen2_5_omni,
1968
1969
    "qwen3_vl": run_qwen3_vl,
    "qwen3_vl_moe": run_qwen3_vl_moe,
1970
    "rvl": run_r_vl,
1971
    "skywork_chat": run_skyworkr1v,
1972
    "smolvlm": run_smolvlm,
1973
    "step3": run_step3,
汪志鹏's avatar
汪志鹏 committed
1974
    "tarsier": run_tarsier,
1975
    "tarsier2": run_tarsier2,
1976
1977
1978
}


1979
1980
1981
1982
MODELS_NEED_VIDEO_METADATA = [
    "glm4_1v",
    "glm4_5v",
    "glm4_5v_fp8",
1983
    "molmo2",
1984
1985
    "qwen3_vl",
    "qwen3_vl_moe",
1986
1987
1988
]


1989
1990
1991
1992
1993
1994
1995
1996
1997
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
1998
        image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
1999
2000
2001
2002
2003
2004
        img_questions = [
            "What is the content of this image?",
            "Describe the content of this image in detail.",
            "What's in the image?",
            "Where is this image taken?",
        ]
2005
2006
2007

        return {
            "data": image,
2008
            "questions": img_questions,
2009
2010
2011
2012
        }

    if args.modality == "video":
        # Input video and question
2013
        needs_metadata = args.model_type in MODELS_NEED_VIDEO_METADATA
2014
        video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
2015
        metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata
2016
        vid_questions = ["Why is this video funny?"]
2017
2018

        return {
2019
            "data": ([(video, metadata)] if needs_metadata else video),
2020
            "questions": vid_questions,
2021
2022
2023
2024
2025
2026
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


2027
2028
2029
2030
def apply_image_repeat(
    image_repeat_prob, num_prompts, data, prompts: list[str], modality
):
    """Repeats images with provided probability of "image_repeat_prob".
2031
2032
    Used to simulate hit/miss for the MM preprocessor cache.
    """
2033
    assert image_repeat_prob <= 1.0 and image_repeat_prob >= 0
2034
2035
2036
2037
    no_yes = [0, 1]
    probs = [1.0 - image_repeat_prob, image_repeat_prob]

    inputs = []
2038
    inputs_with_empty_media = []
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
    cur_image = data
    for i in range(num_prompts):
        if image_repeat_prob is not None:
            res = random.choices(no_yes, probs)[0]
            if res == 0:
                # No repeat => Modify one pixel
                cur_image = cur_image.copy()
                new_val = (i // 256 // 256, i // 256, i % 256)
                cur_image.putpixel((0, 0), new_val)

2049
2050
        uuid = "uuid_{}".format(i)

2051
2052
2053
2054
        inputs.append(
            {
                "prompt": prompts[i % len(prompts)],
                "multi_modal_data": {modality: cur_image},
2055
2056
2057
2058
2059
2060
2061
2062
2063
                "multi_modal_uuids": {modality: uuid},
            }
        )

        inputs_with_empty_media.append(
            {
                "prompt": prompts[i % len(prompts)],
                "multi_modal_data": {modality: None},
                "multi_modal_uuids": {modality: uuid},
2064
            }
2065
        )
2066

2067
    return inputs, inputs_with_empty_media
2068
2069


2070
2071
2072
2073
@contextmanager
def time_counter(enable: bool):
    if enable:
        import time
2074

2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
        start_time = time.time()
        yield
        elapsed_time = time.time() - start_time
        print("-" * 50)
        print("-- generate time = {}".format(elapsed_time))
        print("-" * 50)
    else:
        yield


2085
2086
def parse_args():
    parser = FlexibleArgumentParser(
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
        description="Demo on using vLLM for offline inference with "
        "vision language models for text generation"
    )
    parser.add_argument(
        "--model-type",
        "-m",
        type=str,
        default="llava",
        choices=model_example_map.keys(),
        help='Huggingface "model_type".',
    )
    parser.add_argument(
        "--num-prompts", type=int, default=4, help="Number of prompts to run."
    )
    parser.add_argument(
        "--modality",
        type=str,
        default="image",
        choices=["image", "video"],
        help="Modality of the input.",
    )
    parser.add_argument(
        "--num-frames",
        type=int,
        default=16,
        help="Number of frames to extract from the video.",
    )
    parser.add_argument(
        "--seed",
        type=int,
2117
        default=0,
2118
2119
        help="Set the seed when initializing `vllm.LLM`.",
    )
2120
2121

    parser.add_argument(
2122
        "--image-repeat-prob",
2123
2124
        type=float,
        default=None,
2125
2126
        help="Simulates the hit-ratio for multi-modal preprocessor cache (if enabled)",
    )
2127
2128

    parser.add_argument(
2129
        "--disable-mm-processor-cache",
2130
        action="store_true",
2131
        help="If True, disables caching of multi-modal processor.",
2132
    )
2133
2134

    parser.add_argument(
2135
2136
2137
2138
        "--time-generate",
        action="store_true",
        help="If True, then print the total generate() call time",
    )
2139
2140

    parser.add_argument(
2141
2142
2143
2144
2145
        "--use-different-prompt-per-request",
        action="store_true",
        help="If True, then use different prompt (with the same multi-modal "
        "data) for each request.",
    )
2146
2147
2148
2149
2150
2151
2152

    parser.add_argument(
        "--verify-mm-cache-hit-with-uuids",
        action="store_true",
        help="If True, will send all requests in a second batch with empty mm "
        "data to verify cache hits with UUIDs.",
    )
2153
2154
2155
2156
2157
2158
2159
    parser.add_argument(
        "--tensor-parallel-size",
        "-tp",
        type=int,
        default=None,
        help="Tensor parallel size to override the model's default setting. ",
    )
2160
2161
2162
    return parser.parse_args()


2163
2164
2165
2166
2167
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

2168
2169
2170
2171
2172
2173
    if args.tensor_parallel_size is not None and args.tensor_parallel_size < 1:
        raise ValueError(
            f"tensor_parallel_size must be a positive integer, "
            f"got {args.tensor_parallel_size}"
        )

2174
2175
2176
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
2177
    questions = mm_input["questions"]
2178

2179
2180
    req_data = model_example_map[model](questions, modality)

2181
2182
2183
    # Disable other modalities to save memory
    default_limits = {"image": 0, "video": 0, "audio": 0}
    req_data.engine_args.limit_mm_per_prompt = default_limits | dict(
2184
2185
        req_data.engine_args.limit_mm_per_prompt or {}
    )
2186
2187
2188

    engine_args = asdict(req_data.engine_args) | {
        "seed": args.seed,
2189
        "mm_processor_cache_gb": 0 if args.disable_mm_processor_cache else 4,
2190
    }
2191
2192
    if args.tensor_parallel_size is not None:
        engine_args["tensor_parallel_size"] = args.tensor_parallel_size
2193
2194
    llm = LLM(**engine_args)

2195
    # Don't want to check the flag multiple times, so just hijack `prompts`.
2196
2197
2198
2199
2200
    prompts = (
        req_data.prompts
        if args.use_different_prompt_per_request
        else [req_data.prompts[0]]
    )
2201
2202
2203

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
2204
2205
2206
2207
2208
2209
    sampling_params = (
        SamplingParams(
            temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
        )
        if req_data.sampling_params is None
        else req_data.sampling_params
2210
    )
2211
2212
2213
2214

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
2215
        uuid = "uuid_0"
2216
        inputs = {
2217
            "prompt": prompts[0],
2218
            "multi_modal_data": {modality: data},
2219
2220
2221
2222
2223
2224
            "multi_modal_uuids": {modality: uuid},
        }
        inputs_with_empty_media = {
            "prompt": prompts[0],
            "multi_modal_data": {modality: None},
            "multi_modal_uuids": {modality: uuid},
2225
2226
2227
        }
    else:
        # Batch inference
2228
2229
        if args.image_repeat_prob is not None:
            # Repeat images with specified probability of "image_repeat_prob"
2230
2231
2232
2233
2234
2235
            inputs, inputs_with_empty_media = apply_image_repeat(
                args.image_repeat_prob,
                args.num_prompts,
                data,
                prompts,
                modality,
2236
            )
2237
2238
        else:
            # Use the same image for all prompts
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
            inputs = []
            inputs_with_empty_media = []
            for i in range(args.num_prompts):
                uuid = "uuid_{}".format(i)
                inputs.append(
                    {
                        "prompt": prompts[i % len(prompts)],
                        "multi_modal_data": {modality: data},
                        "multi_modal_uuids": {modality: uuid},
                    }
                )
                inputs_with_empty_media.append(
                    {
                        "prompt": prompts[i % len(prompts)],
                        "multi_modal_data": {modality: None},
                        "multi_modal_uuids": {modality: uuid},
                    }
                )
2257

2258
    # Add LoRA request if applicable
2259
2260
2261
    lora_request = (
        req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
    )
2262

2263
2264
2265
2266
2267
2268
    with time_counter(args.time_generate):
        outputs = llm.generate(
            inputs,
            sampling_params=sampling_params,
            lora_request=lora_request,
        )
2269

2270
    print("-" * 50)
2271
2272
2273
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)
2274
        print("-" * 50)
2275

2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
    if args.verify_mm_cache_hit_with_uuids:
        try:
            # Verify cache hits with UUIDs
            print(
                "Sending a second batch of requests with empty media"
                " and matching UUIDs."
            )
            outputs = llm.generate(
                inputs_with_empty_media,
                sampling_params=sampling_params,
                lora_request=lora_request,
            )
            print("-" * 50)
            for o in outputs:
                generated_text = o.outputs[0].text
                print(generated_text)
                print("-" * 50)
        except Exception as e:
            print(f"Failed to verify cache hits with UUIDs. Error: {e}")

2296
2297

if __name__ == "__main__":
2298
    args = parse_args()
2299
    main(args)