vision_language.py 36.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
"""
Cyrus Leung's avatar
Cyrus Leung committed
3
4
This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.
5
6
7
8

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
9

10
import os
11
import random
12
from contextlib import contextmanager
13
14
from dataclasses import asdict
from typing import NamedTuple, Optional
15

16
from huggingface_hub import snapshot_download
17
18
from transformers import AutoTokenizer

19
from vllm import LLM, EngineArgs, SamplingParams
20
from vllm.assets.image import ImageAsset
21
from vllm.assets.video import VideoAsset
22
from vllm.lora.request import LoRARequest
23
from vllm.multimodal.image import convert_image_mode
24
25
from vllm.utils import FlexibleArgumentParser

26
27
28
29
30
31
32
33

class ModelRequestData(NamedTuple):
    engine_args: EngineArgs
    prompts: list[str]
    stop_token_ids: Optional[list[int]] = None
    lora_requests: Optional[list[LoRARequest]] = None


34
35
36
37
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
# lower-end GPUs.
# Unless specified, these settings have been tested to work on a single L4.

38

39
# Aria
40
def run_aria(questions: list[str], modality: str) -> ModelRequestData:
41
42
43
    assert modality == "image"
    model_name = "rhymes-ai/Aria"

44
    # NOTE: Need L40 (or equivalent) to avoid OOM
45
46
47
48
49
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        dtype="bfloat16",
50
        limit_mm_per_prompt={modality: 1},
51
    )
52

53
54
55
56
57
58
59
    prompts = [
        (
            f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
            "<|im_end|>\n<|im_start|>assistant\n"
        )
        for question in questions
    ]
60
61

    stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
62
63
64
65
66
67

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
68
69


Jennifer Zhao's avatar
Jennifer Zhao committed
70
71
72
73
74
75
76
77
78
79
# Aya Vision
def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "CohereForAI/aya-vision-8b"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        mm_processor_kwargs={"crop_to_patches": True},
80
        limit_mm_per_prompt={modality: 1},
Jennifer Zhao's avatar
Jennifer Zhao committed
81
82
83
84
85
86
87
88
89
90
91
    )
    prompts = [
        f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
        for question in questions
    ]
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


92
# BLIP-2
93
def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
94
95
96
97
    assert modality == "image"

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
98
    prompts = [f"Question: {question} Answer:" for question in questions]
99
    engine_args = EngineArgs(
100
        model="Salesforce/blip2-opt-6.7b",
101
        limit_mm_per_prompt={modality: 1},
102
103
104
105
106
107
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
108
109
110


# Chameleon
111
def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:
112
113
    assert modality == "image"

114
    prompts = [f"{question}<image>" for question in questions]
115
116
117
118
    engine_args = EngineArgs(
        model="facebook/chameleon-7b",
        max_model_len=4096,
        max_num_seqs=2,
119
        limit_mm_per_prompt={modality: 1},
120
121
122
123
124
125
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
126
127


128
# Deepseek-VL2
129
def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:
130
131
    assert modality == "image"

132
    model_name = "deepseek-ai/deepseek-vl2-tiny"
133

134
135
136
137
138
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
139
        limit_mm_per_prompt={modality: 1},
140
    )
141

142
    prompts = [
143
        f"<|User|>: <image>\n{question}\n\n<|Assistant|>:" for question in questions
144
    ]
145
146
147
148
149

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
150
151


152
# Florence2
153
def run_florence2(questions: list[str], modality: str) -> ModelRequestData:
154
155
    assert modality == "image"

156
157
    engine_args = EngineArgs(
        model="microsoft/Florence-2-large",
158
        tokenizer="Isotr0py/Florence-2-tokenizer",
159
160
        max_model_len=4096,
        max_num_seqs=2,
161
162
        trust_remote_code=True,
        dtype="bfloat16",
163
        limit_mm_per_prompt={modality: 1},
164
    )
165

166
167
168
169
170
171
    prompts = ["<MORE_DETAILED_CAPTION>" for _ in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
172
173


174
# Fuyu
175
def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:
176
177
    assert modality == "image"

178
    prompts = [f"{question}\n" for question in questions]
179
180
181
182
    engine_args = EngineArgs(
        model="adept/fuyu-8b",
        max_model_len=2048,
        max_num_seqs=2,
183
        limit_mm_per_prompt={modality: 1},
184
185
186
187
188
189
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
190
191


192
# Gemma 3
193
def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:
194
195
196
    assert modality == "image"
    model_name = "google/gemma-3-4b-it"

197
    engine_args = EngineArgs(
198
199
200
201
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        mm_processor_kwargs={"do_pan_and_scan": True},
202
        limit_mm_per_prompt={modality: 1},
203
    )
204

205
206
207
208
209
210
211
212
    prompts = [
        (
            "<bos><start_of_turn>user\n"
            f"<start_of_image>{question}<end_of_turn>\n"
            "<start_of_turn>model\n"
        )
        for question in questions
    ]
213
214
215
216
217

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
218
219


220
# GLM-4v
221
def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:
222
223
224
    assert modality == "image"
    model_name = "THUDM/glm-4v-9b"

225
226
227
228
229
230
231
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        trust_remote_code=True,
        enforce_eager=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
232
        limit_mm_per_prompt={modality: 1},
233
    )
234

235
236
    prompts = [
        f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
237
238
        {question}<|assistant|>"
        for question in questions
239
    ]
240

241
    stop_token_ids = [151329, 151336, 151338]
242
243
244
245
246
247

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
248
249
250


# H2OVL-Mississippi
251
def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
252
253
    assert modality == "image"

254
    model_name = "h2oai/h2ovl-mississippi-800m"
255

256
    engine_args = EngineArgs(
257
258
259
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
260
        limit_mm_per_prompt={modality: 1},
261
262
    )

263
264
265
266
267
268
269
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
270
271

    # Stop tokens for H2OVL-Mississippi
272
    # https://huggingface.co/h2oai/h2ovl-mississippi-800m
273
    stop_token_ids = [tokenizer.eos_token_id]
274
275
276
277
278
279

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
280
281
282


# Idefics3-8B-Llama3
283
def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
284
285
286
    assert modality == "image"
    model_name = "HuggingFaceM4/Idefics3-8B-Llama3"

287
    engine_args = EngineArgs(
288
289
290
291
292
293
294
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        # if you are running out of memory, you can reduce the "longest_edge".
        # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
        mm_processor_kwargs={
295
            "size": {"longest_edge": 3 * 364},
296
        },
297
        limit_mm_per_prompt={modality: 1},
298
    )
299
300
301
302
    prompts = [
        (f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]
303
304
305
306
307

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
308
309


310
311
312
313
314
315
316
317
318
319
320
# SmolVLM2-2.2B-Instruct
def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        mm_processor_kwargs={
321
            "max_image_size": {"longest_edge": 384},
322
        },
323
        limit_mm_per_prompt={modality: 1},
324
325
326
327
328
329
330
331
332
333
334
335
    )
    prompts = [
        (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


336
# InternVL
337
def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
338
    model_name = "OpenGVLab/InternVL3-2B"
339

340
    engine_args = EngineArgs(
341
342
        model=model_name,
        trust_remote_code=True,
343
        max_model_len=8192,
344
        limit_mm_per_prompt={modality: 1},
345
346
    )

347
348
349
350
351
    if modality == "image":
        placeholder = "<image>"
    elif modality == "video":
        placeholder = "<video>"

352
353
354
355
356
357
358
359
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"{placeholder}\n{question}"}]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
360
361
362
363
364
365
366

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
367
    stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
368
369
370
371
372
373

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
374
375


376
377
378
379
380
381
382
# Kimi-VL
def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    prompts = [
        "<|im_user|>user<|im_middle|><|media_start|>image<|media_content|>"
        f"<|media_pad|><|media_end|>{question}<|im_end|>"
383
384
        "<|im_assistant|>assistant<|im_middle|>"
        for question in questions
385
386
387
388
389
    ]

    engine_args = EngineArgs(
        model="moonshotai/Kimi-VL-A3B-Instruct",
        trust_remote_code=True,
Cyrus Leung's avatar
Cyrus Leung committed
390
        max_model_len=4096,
391
        limit_mm_per_prompt={modality: 1},
392
393
394
395
396
397
398
399
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


400
# LLaVA-1.5
401
def run_llava(questions: list[str], modality: str) -> ModelRequestData:
402
    assert modality == "image"
403

404
    prompts = [f"USER: <image>\n{question}\nASSISTANT:" for question in questions]
405

406
407
408
    engine_args = EngineArgs(
        model="llava-hf/llava-1.5-7b-hf",
        max_model_len=4096,
409
        limit_mm_per_prompt={modality: 1},
410
411
412
413
414
415
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
416
417
418


# LLaVA-1.6/LLaVA-NeXT
419
def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:
420
    assert modality == "image"
421

422
    prompts = [f"[INST] <image>\n{question} [/INST]" for question in questions]
423
424
425
    engine_args = EngineArgs(
        model="llava-hf/llava-v1.6-mistral-7b-hf",
        max_model_len=8192,
426
        limit_mm_per_prompt={modality: 1},
427
428
429
430
431
432
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
433
434
435
436


# LlaVA-NeXT-Video
# Currently only support for video input
437
def run_llava_next_video(questions: list[str], modality: str) -> ModelRequestData:
438
439
    assert modality == "video"

440
    prompts = [f"USER: <video>\n{question} ASSISTANT:" for question in questions]
441
442
443
    engine_args = EngineArgs(
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        max_model_len=8192,
444
        max_num_seqs=2,
445
        limit_mm_per_prompt={modality: 1},
446
447
448
449
450
451
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
452
453


454
# LLaVA-OneVision
455
def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData:
456
    if modality == "video":
457
458
        prompts = [
            f"<|im_start|>user <video>\n{question}<|im_end|> \
459
460
        <|im_start|>assistant\n"
            for question in questions
461
        ]
462
463

    elif modality == "image":
464
465
        prompts = [
            f"<|im_start|>user <image>\n{question}<|im_end|> \
466
467
        <|im_start|>assistant\n"
            for question in questions
468
        ]
469

470
471
472
    engine_args = EngineArgs(
        model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
        max_model_len=16384,
473
        limit_mm_per_prompt={modality: 1},
474
475
476
477
478
479
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
480
481


482
# Mantis
483
def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
484
    assert modality == "image"
485

486
487
    llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"  # noqa: E501
    prompts = [llama3_template.format(f"{question}\n<image>") for question in questions]
488

489
    engine_args = EngineArgs(
490
        model="TIGER-Lab/Mantis-8B-siglip-llama3",
491
        max_model_len=4096,
492
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
493
        limit_mm_per_prompt={modality: 1},
494
    )
495
    stop_token_ids = [128009]
496
497
498
499
500
501

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
502
503
504


# MiniCPM-V
505
def run_minicpmv_base(questions: list[str], modality: str, model_name):
506
507
    assert modality in ["image", "video"]
    # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
508
509
510
511
512
513
514

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
515
516
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

517
    # 2.6
518
519
520
521
522
523
524
525
526
    # model_name = "openbmb/MiniCPM-V-2_6"
    # o2.6

    # modality supports
    # 2.0: image
    # 2.5: image
    # 2.6: image, video
    # o2.6: image, video, audio
    # model_name = "openbmb/MiniCPM-o-2_6"
527
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
528
    engine_args = EngineArgs(
529
        model=model_name,
530
531
        max_model_len=4096,
        max_num_seqs=2,
532
        trust_remote_code=True,
533
        limit_mm_per_prompt={modality: 1},
534
    )
535
536
537
538
539
540
541
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

542
    # 2.6 / o2.6
543
    stop_tokens = ["<|im_end|>", "<|endoftext|>"]
544
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
545

546
547
548
549
550
    modality_placeholder = {
        "image": "(<image>./</image>)",
        "video": "(<video>./</video>)",
    }

551
552
    prompts = [
        tokenizer.apply_chat_template(
553
554
555
556
557
558
            [
                {
                    "role": "user",
                    "content": f"{modality_placeholder[modality]}\n{question}",
                }
            ],
559
            tokenize=False,
560
561
562
            add_generation_prompt=True,
        )
        for question in questions
563
    ]
564
565
566
567
568
569

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
570
571


572
def run_minicpmo(questions: list[str], modality: str) -> ModelRequestData:
573
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-o-2_6")
574
575


576
def run_minicpmv(questions: list[str], modality: str) -> ModelRequestData:
577
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-V-2_6")
578
579


580
581
582
583
584
585
586
587
588
589
590
591
# Mistral-3 HF-format
def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"

    # NOTE: Need L40 (or equivalent) to avoid OOM
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        tensor_parallel_size=2,
592
        limit_mm_per_prompt={modality: 1},
593
594
595
596
597
598
599
600
601
602
    )

    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


603
# LLama 3.2
604
def run_mllama(questions: list[str], modality: str) -> ModelRequestData:
605
606
    assert modality == "image"

607
    model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
608

609
610
611
612
613
    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (131072) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

    # The configuration below has been confirmed to launch on a single L40 GPU.
614
    engine_args = EngineArgs(
615
        model=model_name,
616
        max_model_len=8192,
617
        max_num_seqs=2,
618
        limit_mm_per_prompt={modality: 1},
619
620
    )

621
    tokenizer = AutoTokenizer.from_pretrained(model_name)
622
623
624
625
626
627
628
629
630
631
632
633
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": question}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )
634
635
636
637
638

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
639
640


641
def run_llama4(questions: list[str], modality: str) -> ModelRequestData:
642
643
644
645
646
647
648
649
650
651
    assert modality == "image"

    model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=4,
        tensor_parallel_size=8,
        gpu_memory_utilization=0.4,
652
        limit_mm_per_prompt={modality: 1},
653
654
655
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name)
656
657
658
659
660
661
662
663
664
665
666
667
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": f"{question}"}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )
668
669
670
671
672
673
674
675
    stop_token_ids = None
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )


676
# Molmo
677
def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
678
679
    assert modality == "image"

680
    model_name = "allenai/Molmo-7B-D-0924"
681

682
    engine_args = EngineArgs(
683
        model=model_name,
684
        trust_remote_code=True,
685
        dtype="bfloat16",
686
        limit_mm_per_prompt={modality: 1},
687
    )
688

689
690
    prompts = [
        f"<|im_start|>user <image>\n{question}<|im_end|> \
691
692
        <|im_start|>assistant\n"
        for question in questions
693
    ]
694
695
696
697
698

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
699
700


701
# NVLM-D
702
def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:
703
704
705
706
707
    assert modality == "image"

    model_name = "nvidia/NVLM-D-72B"

    # Adjust this as necessary to fit in GPU
708
    engine_args = EngineArgs(
709
710
711
712
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        tensor_parallel_size=4,
713
        limit_mm_per_prompt={modality: 1},
714
715
    )

716
717
718
719
720
721
722
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
723
724
725
726
727

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
728
729


730
731
# Ovis
def run_ovis(questions: list[str], modality: str) -> ModelRequestData:
732
733
734
735
736
737
738
739
740
741
    assert modality == "image"

    model_name = "AIDC-AI/Ovis2-1B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        trust_remote_code=True,
        dtype="half",
742
        limit_mm_per_prompt={modality: 1},
743
744
    )

745
746
747
748
749
750
751
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
752
753
754
755
756
757
758

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


759
# PaliGemma
760
def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:
761
    assert modality == "image"
762

763
    # PaliGemma has special prompt format for VQA
764
765
766
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma-3b-mix-224",
767
        limit_mm_per_prompt={modality: 1},
768
    )
769
770
771
772
773

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
774
775


776
# PaliGemma 2
777
def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:
778
    assert modality == "image"
779

780
    # PaliGemma 2 has special prompt format for VQA
781
782
783
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma2-3b-ft-docci-448",
784
        limit_mm_per_prompt={modality: 1},
785
    )
786
787
788
789
790

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
791
792


793
# Phi-3-Vision
794
def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
795
796
    assert modality == "image"

797
798
799
800
    prompts = [
        f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
        for question in questions
    ]
801

802
803
804
805
806
807
808
809
810
811
812
813
    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
814
    engine_args = EngineArgs(
815
816
        model="microsoft/Phi-3.5-vision-instruct",
        trust_remote_code=True,
817
        max_model_len=4096,
818
        max_num_seqs=2,
819
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
820
        mm_processor_kwargs={"num_crops": 16},
821
        limit_mm_per_prompt={modality: 1},
822
    )
823
824
825
826
827

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
828
829


830
# Phi-4-multimodal-instruct
831
def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
832
833
834
835
836
837
838
839
840
841
    """
    Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
    show how to process image inputs.
    """
    assert modality == "image"
    model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
    # Since the vision-lora and speech-lora co-exist with the base model,
    # we have to manually specify the path of the lora weights.
    vision_lora_path = os.path.join(model_path, "vision-lora")
    prompts = [
842
        f"<|user|><|image_1|>{question}<|end|><|assistant|>" for question in questions
843
    ]
844
    engine_args = EngineArgs(
845
846
        model=model_path,
        trust_remote_code=True,
847
        max_model_len=5120,
848
        max_num_seqs=2,
849
        max_num_batched_tokens=12800,
850
851
        enable_lora=True,
        max_lora_rank=320,
852
853
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
        mm_processor_kwargs={"dynamic_hd": 16},
854
        limit_mm_per_prompt={modality: 1},
855
856
    )

857
858
859
860
861
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
    )
862
863


864
# Pixtral HF-format
865
def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
866
867
868
869
    assert modality == "image"

    model_name = "mistral-community/pixtral-12b"

870
    # NOTE: Need L40 (or equivalent) to avoid OOM
871
    engine_args = EngineArgs(
872
        model=model_name,
873
        max_model_len=6144,
874
        max_num_seqs=2,
875
        limit_mm_per_prompt={modality: 1},
876
877
    )

878
    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
879
880
881
882
883

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
884
885


886
# Qwen
887
def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:
888
889
    assert modality == "image"

890
    engine_args = EngineArgs(
891
        model="Qwen/Qwen-VL",
892
        trust_remote_code=True,
893
894
        max_model_len=1024,
        max_num_seqs=2,
895
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
896
        limit_mm_per_prompt={modality: 1},
897
898
    )

899
    prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]
900
901
902
903
904

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
905
906


907
# Qwen2-VL
908
def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:
909
    model_name = "Qwen/Qwen2-VL-7B-Instruct"
910

911
    engine_args = EngineArgs(
912
        model=model_name,
913
914
915
        max_model_len=4096,
        max_num_seqs=5,
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
916
        mm_processor_kwargs={
917
918
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
919
        },
920
        limit_mm_per_prompt={modality: 1},
921
    )
922

923
924
925
926
927
    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

928
    prompts = [
929
930
931
932
933
934
935
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
936
    ]
937
938
939
940
941

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
942
943


Roger Wang's avatar
Roger Wang committed
944
# Qwen2.5-VL
945
def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
Roger Wang's avatar
Roger Wang committed
946
947
    model_name = "Qwen/Qwen2.5-VL-3B-Instruct"

948
    engine_args = EngineArgs(
Roger Wang's avatar
Roger Wang committed
949
950
951
952
953
954
955
956
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": 1,
        },
957
        limit_mm_per_prompt={modality: 1},
Roger Wang's avatar
Roger Wang committed
958
959
960
961
962
963
964
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

965
    prompts = [
966
967
968
969
970
971
972
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
973
    ]
974
975
976
977
978

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
Roger Wang's avatar
Roger Wang committed
979
980


981
982
983
984
985
986
987
988
989
990
991
992
993
# Qwen2.5-Omni
def run_qwen2_5_omni(questions: list[str], modality: str):
    model_name = "Qwen/Qwen2.5-Omni-7B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": [1],
        },
994
        limit_mm_per_prompt={modality: 1},
995
996
997
998
999
1000
1001
1002
1003
1004
    )

    if modality == "image":
        placeholder = "<|IMAGE|>"
    elif modality == "video":
        placeholder = "<|VIDEO|>"

    default_system = (
        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
        "Group, capable of perceiving auditory and visual inputs, as well as "
1005
1006
        "generating text and speech."
    )
1007

1008
1009
1010
1011
1012
1013
1014
1015
1016
    prompts = [
        (
            f"<|im_start|>system\n{default_system}<|im_end|>\n"
            f"<|im_start|>user\n<|vision_bos|>{placeholder}<|vision_eos|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]
1017
1018
1019
1020
1021
1022
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
# SkyworkR1V
def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "Skywork/Skywork-R1V-38B"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
1033
        limit_mm_per_prompt={modality: 1},
1034
1035
    )

1036
1037
1038
1039
1040
1041
1042
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055

    # Stop tokens for SkyworkR1V
    # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
    stop_tokens = ["<|end▁of▁sentence|>", "<|endoftext|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )


1056
model_example_map = {
1057
    "aria": run_aria,
Jennifer Zhao's avatar
Jennifer Zhao committed
1058
    "aya_vision": run_aya_vision,
1059
1060
    "blip-2": run_blip2,
    "chameleon": run_chameleon,
1061
    "deepseek_vl_v2": run_deepseek_vl2,
1062
    "florence2": run_florence2,
1063
    "fuyu": run_fuyu,
1064
    "gemma3": run_gemma3,
1065
1066
1067
1068
    "glm4v": run_glm4v,
    "h2ovl_chat": run_h2ovl,
    "idefics3": run_idefics3,
    "internvl_chat": run_internvl,
1069
    "kimi_vl": run_kimi_vl,
1070
1071
    "llava": run_llava,
    "llava-next": run_llava_next,
1072
    "llava-next-video": run_llava_next_video,
1073
    "llava-onevision": run_llava_onevision,
1074
    "mantis": run_mantis,
1075
    "minicpmo": run_minicpmo,
1076
    "minicpmv": run_minicpmv,
1077
    "mistral3": run_mistral3,
1078
    "mllama": run_mllama,
1079
    "llama4": run_llama4,
1080
    "molmo": run_molmo,
1081
    "NVLM_D": run_nvlm_d,
1082
    "ovis": run_ovis,
1083
1084
1085
    "paligemma": run_paligemma,
    "paligemma2": run_paligemma2,
    "phi3_v": run_phi3v,
1086
    "phi4_mm": run_phi4mm,
1087
    "pixtral_hf": run_pixtral_hf,
1088
    "qwen_vl": run_qwen_vl,
1089
    "qwen2_vl": run_qwen2_vl,
Roger Wang's avatar
Roger Wang committed
1090
    "qwen2_5_vl": run_qwen2_5_vl,
1091
    "qwen2_5_omni": run_qwen2_5_omni,
1092
    "skywork_chat": run_skyworkr1v,
1093
    "smolvlm": run_smolvlm,
1094
1095
1096
}


1097
1098
1099
1100
1101
1102
1103
1104
1105
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
1106
        image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
1107
1108
1109
1110
1111
1112
        img_questions = [
            "What is the content of this image?",
            "Describe the content of this image in detail.",
            "What's in the image?",
            "Where is this image taken?",
        ]
1113
1114
1115

        return {
            "data": image,
1116
            "questions": img_questions,
1117
1118
1119
1120
        }

    if args.modality == "video":
        # Input video and question
1121
        video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
1122
        vid_questions = ["Why is this video funny?"]
1123
1124
1125

        return {
            "data": video,
1126
            "questions": vid_questions,
1127
1128
1129
1130
1131
1132
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


1133
1134
1135
1136
def apply_image_repeat(
    image_repeat_prob, num_prompts, data, prompts: list[str], modality
):
    """Repeats images with provided probability of "image_repeat_prob".
1137
1138
    Used to simulate hit/miss for the MM preprocessor cache.
    """
1139
    assert image_repeat_prob <= 1.0 and image_repeat_prob >= 0
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
    no_yes = [0, 1]
    probs = [1.0 - image_repeat_prob, image_repeat_prob]

    inputs = []
    cur_image = data
    for i in range(num_prompts):
        if image_repeat_prob is not None:
            res = random.choices(no_yes, probs)[0]
            if res == 0:
                # No repeat => Modify one pixel
                cur_image = cur_image.copy()
                new_val = (i // 256 // 256, i // 256, i % 256)
                cur_image.putpixel((0, 0), new_val)

1154
1155
1156
1157
        inputs.append(
            {
                "prompt": prompts[i % len(prompts)],
                "multi_modal_data": {modality: cur_image},
1158
            }
1159
        )
1160
1161
1162
1163

    return inputs


1164
1165
1166
1167
@contextmanager
def time_counter(enable: bool):
    if enable:
        import time
1168

1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
        start_time = time.time()
        yield
        elapsed_time = time.time() - start_time
        print("-" * 50)
        print("-- generate time = {}".format(elapsed_time))
        print("-" * 50)
    else:
        yield


1179
1180
def parse_args():
    parser = FlexibleArgumentParser(
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
        description="Demo on using vLLM for offline inference with "
        "vision language models for text generation"
    )
    parser.add_argument(
        "--model-type",
        "-m",
        type=str,
        default="llava",
        choices=model_example_map.keys(),
        help='Huggingface "model_type".',
    )
    parser.add_argument(
        "--num-prompts", type=int, default=4, help="Number of prompts to run."
    )
    parser.add_argument(
        "--modality",
        type=str,
        default="image",
        choices=["image", "video"],
        help="Modality of the input.",
    )
    parser.add_argument(
        "--num-frames",
        type=int,
        default=16,
        help="Number of frames to extract from the video.",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Set the seed when initializing `vllm.LLM`.",
    )
1214
1215

    parser.add_argument(
1216
        "--image-repeat-prob",
1217
1218
        type=float,
        default=None,
1219
1220
        help="Simulates the hit-ratio for multi-modal preprocessor cache (if enabled)",
    )
1221
1222

    parser.add_argument(
1223
1224
1225
1226
        "--disable-mm-preprocessor-cache",
        action="store_true",
        help="If True, disables caching of multi-modal preprocessor/mapper.",
    )
1227
1228

    parser.add_argument(
1229
1230
1231
1232
        "--time-generate",
        action="store_true",
        help="If True, then print the total generate() call time",
    )
1233
1234

    parser.add_argument(
1235
1236
1237
1238
1239
        "--use-different-prompt-per-request",
        action="store_true",
        help="If True, then use different prompt (with the same multi-modal "
        "data) for each request.",
    )
1240
1241
1242
    return parser.parse_args()


1243
1244
1245
1246
1247
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

1248
1249
1250
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
1251
    questions = mm_input["questions"]
1252

1253
1254
    req_data = model_example_map[model](questions, modality)

1255
1256
1257
    # Disable other modalities to save memory
    default_limits = {"image": 0, "video": 0, "audio": 0}
    req_data.engine_args.limit_mm_per_prompt = default_limits | dict(
1258
1259
        req_data.engine_args.limit_mm_per_prompt or {}
    )
1260
1261
1262
1263
1264

    engine_args = asdict(req_data.engine_args) | {
        "seed": args.seed,
        "disable_mm_preprocessor_cache": args.disable_mm_preprocessor_cache,
    }
1265
1266
    llm = LLM(**engine_args)

1267
    # Don't want to check the flag multiple times, so just hijack `prompts`.
1268
1269
1270
1271
1272
    prompts = (
        req_data.prompts
        if args.use_different_prompt_per_request
        else [req_data.prompts[0]]
    )
1273
1274
1275

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
1276
1277
1278
    sampling_params = SamplingParams(
        temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
    )
1279
1280
1281
1282
1283

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        inputs = {
1284
            "prompt": prompts[0],
1285
            "multi_modal_data": {modality: data},
1286
1287
1288
        }
    else:
        # Batch inference
1289
1290
        if args.image_repeat_prob is not None:
            # Repeat images with specified probability of "image_repeat_prob"
1291
1292
1293
            inputs = apply_image_repeat(
                args.image_repeat_prob, args.num_prompts, data, prompts, modality
            )
1294
1295
        else:
            # Use the same image for all prompts
1296
1297
1298
1299
1300
1301
1302
            inputs = [
                {
                    "prompt": prompts[i % len(prompts)],
                    "multi_modal_data": {modality: data},
                }
                for i in range(args.num_prompts)
            ]
1303

1304
    # Add LoRA request if applicable
1305
1306
1307
    lora_request = (
        req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
    )
1308

1309
1310
1311
1312
1313
1314
    with time_counter(args.time_generate):
        outputs = llm.generate(
            inputs,
            sampling_params=sampling_params,
            lora_request=lora_request,
        )
1315

1316
    print("-" * 50)
1317
1318
1319
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)
1320
        print("-" * 50)
1321
1322
1323


if __name__ == "__main__":
1324
    args = parse_args()
1325
    main(args)