"vscode:/vscode.git/clone" did not exist on "47de8821d3cdd32fce7df6312318223aee591fd2"
vision_language.py 37.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
"""
Cyrus Leung's avatar
Cyrus Leung committed
3
4
This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.
5
6
7
8

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
9

10
import os
11
import random
12
from contextlib import contextmanager
13
14
from dataclasses import asdict
from typing import NamedTuple, Optional
15

16
from huggingface_hub import snapshot_download
17
18
from transformers import AutoTokenizer

19
from vllm import LLM, EngineArgs, SamplingParams
20
from vllm.assets.image import ImageAsset
21
from vllm.assets.video import VideoAsset
22
from vllm.lora.request import LoRARequest
23
from vllm.multimodal.image import convert_image_mode
24
25
from vllm.utils import FlexibleArgumentParser

26
27
28
29
30
31
32
33

class ModelRequestData(NamedTuple):
    engine_args: EngineArgs
    prompts: list[str]
    stop_token_ids: Optional[list[int]] = None
    lora_requests: Optional[list[LoRARequest]] = None


34
35
36
37
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
# lower-end GPUs.
# Unless specified, these settings have been tested to work on a single L4.

38

39
# Aria
40
def run_aria(questions: list[str], modality: str) -> ModelRequestData:
41
42
43
    assert modality == "image"
    model_name = "rhymes-ai/Aria"

44
    # NOTE: Need L40 (or equivalent) to avoid OOM
45
46
47
48
49
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        dtype="bfloat16",
50
        limit_mm_per_prompt={modality: 1},
51
    )
52

53
54
55
56
57
58
59
    prompts = [
        (
            f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
            "<|im_end|>\n<|im_start|>assistant\n"
        )
        for question in questions
    ]
60
61

    stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
62
63
64
65
66
67

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
68
69


Jennifer Zhao's avatar
Jennifer Zhao committed
70
71
72
73
74
75
76
77
78
79
# Aya Vision
def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "CohereForAI/aya-vision-8b"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        mm_processor_kwargs={"crop_to_patches": True},
80
        limit_mm_per_prompt={modality: 1},
Jennifer Zhao's avatar
Jennifer Zhao committed
81
82
83
84
85
86
87
88
89
90
91
    )
    prompts = [
        f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
        for question in questions
    ]
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


92
# BLIP-2
93
def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
94
95
96
97
    assert modality == "image"

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
98
    prompts = [f"Question: {question} Answer:" for question in questions]
99
    engine_args = EngineArgs(
100
        model="Salesforce/blip2-opt-6.7b",
101
        limit_mm_per_prompt={modality: 1},
102
103
104
105
106
107
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
108
109
110


# Chameleon
111
def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:
112
113
    assert modality == "image"

114
    prompts = [f"{question}<image>" for question in questions]
115
116
117
118
    engine_args = EngineArgs(
        model="facebook/chameleon-7b",
        max_model_len=4096,
        max_num_seqs=2,
119
        limit_mm_per_prompt={modality: 1},
120
121
122
123
124
125
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
126
127


128
# Deepseek-VL2
129
def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:
130
131
    assert modality == "image"

132
    model_name = "deepseek-ai/deepseek-vl2-tiny"
133

134
135
136
137
138
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
139
        limit_mm_per_prompt={modality: 1},
140
    )
141

142
    prompts = [
143
        f"<|User|>: <image>\n{question}\n\n<|Assistant|>:" for question in questions
144
    ]
145
146
147
148
149

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
150
151


152
# Florence2
153
def run_florence2(questions: list[str], modality: str) -> ModelRequestData:
154
155
    assert modality == "image"

156
157
    engine_args = EngineArgs(
        model="microsoft/Florence-2-large",
158
        tokenizer="Isotr0py/Florence-2-tokenizer",
159
160
        max_model_len=4096,
        max_num_seqs=2,
161
162
        trust_remote_code=True,
        dtype="bfloat16",
163
        limit_mm_per_prompt={modality: 1},
164
    )
165

166
167
168
169
170
171
    prompts = ["<MORE_DETAILED_CAPTION>" for _ in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
172
173


174
# Fuyu
175
def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:
176
177
    assert modality == "image"

178
    prompts = [f"{question}\n" for question in questions]
179
180
181
182
    engine_args = EngineArgs(
        model="adept/fuyu-8b",
        max_model_len=2048,
        max_num_seqs=2,
183
        limit_mm_per_prompt={modality: 1},
184
185
186
187
188
189
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
190
191


192
# Gemma 3
193
def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:
194
195
196
    assert modality == "image"
    model_name = "google/gemma-3-4b-it"

197
    engine_args = EngineArgs(
198
199
200
201
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        mm_processor_kwargs={"do_pan_and_scan": True},
202
        limit_mm_per_prompt={modality: 1},
203
    )
204

205
206
207
208
209
210
211
212
    prompts = [
        (
            "<bos><start_of_turn>user\n"
            f"<start_of_image>{question}<end_of_turn>\n"
            "<start_of_turn>model\n"
        )
        for question in questions
    ]
213
214
215
216
217

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
218
219


220
# GLM-4v
221
def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:
222
223
224
    assert modality == "image"
    model_name = "THUDM/glm-4v-9b"

225
226
227
228
229
230
231
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        trust_remote_code=True,
        enforce_eager=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
232
        limit_mm_per_prompt={modality: 1},
233
    )
234

235
236
    prompts = [
        f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
237
238
        {question}<|assistant|>"
        for question in questions
239
    ]
240

241
    stop_token_ids = [151329, 151336, 151338]
242
243
244
245
246
247

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
248
249
250


# H2OVL-Mississippi
251
def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
252
253
    assert modality == "image"

254
    model_name = "h2oai/h2ovl-mississippi-800m"
255

256
    engine_args = EngineArgs(
257
258
259
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
260
        limit_mm_per_prompt={modality: 1},
261
262
    )

263
264
265
266
267
268
269
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
270
271

    # Stop tokens for H2OVL-Mississippi
272
    # https://huggingface.co/h2oai/h2ovl-mississippi-800m
273
    stop_token_ids = [tokenizer.eos_token_id]
274
275
276
277
278
279

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
280
281
282


# Idefics3-8B-Llama3
283
def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
284
285
286
    assert modality == "image"
    model_name = "HuggingFaceM4/Idefics3-8B-Llama3"

287
    engine_args = EngineArgs(
288
289
290
291
292
293
294
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        # if you are running out of memory, you can reduce the "longest_edge".
        # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
        mm_processor_kwargs={
295
            "size": {"longest_edge": 3 * 364},
296
        },
297
        limit_mm_per_prompt={modality: 1},
298
    )
299
300
301
302
    prompts = [
        (f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]
303
304
305
306
307

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
308
309


310
311
312
313
314
315
316
317
318
319
320
# SmolVLM2-2.2B-Instruct
def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        mm_processor_kwargs={
321
            "max_image_size": {"longest_edge": 384},
322
        },
323
        limit_mm_per_prompt={modality: 1},
324
325
326
327
328
329
330
331
332
333
334
335
    )
    prompts = [
        (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


汪志鹏's avatar
汪志鹏 committed
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
# omni-research/Tarsier-7b
def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "omni-research/Tarsier-7b"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        limit_mm_per_prompt={modality: 1},
    )
    prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


355
# InternVL
356
def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
357
    model_name = "OpenGVLab/InternVL3-2B"
358

359
    engine_args = EngineArgs(
360
361
        model=model_name,
        trust_remote_code=True,
362
        max_model_len=8192,
363
        limit_mm_per_prompt={modality: 1},
364
365
    )

366
367
368
369
370
    if modality == "image":
        placeholder = "<image>"
    elif modality == "video":
        placeholder = "<video>"

371
372
373
374
375
376
377
378
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"{placeholder}\n{question}"}]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
379
380
381
382
383
384
385

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
386
    stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
387
388
389
390
391
392

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
393
394


395
396
397
398
399
400
401
# Kimi-VL
def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    prompts = [
        "<|im_user|>user<|im_middle|><|media_start|>image<|media_content|>"
        f"<|media_pad|><|media_end|>{question}<|im_end|>"
402
403
        "<|im_assistant|>assistant<|im_middle|>"
        for question in questions
404
405
406
407
408
    ]

    engine_args = EngineArgs(
        model="moonshotai/Kimi-VL-A3B-Instruct",
        trust_remote_code=True,
Cyrus Leung's avatar
Cyrus Leung committed
409
        max_model_len=4096,
410
        limit_mm_per_prompt={modality: 1},
411
412
413
414
415
416
417
418
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


419
# LLaVA-1.5
420
def run_llava(questions: list[str], modality: str) -> ModelRequestData:
421
    assert modality == "image"
422

423
    prompts = [f"USER: <image>\n{question}\nASSISTANT:" for question in questions]
424

425
426
427
    engine_args = EngineArgs(
        model="llava-hf/llava-1.5-7b-hf",
        max_model_len=4096,
428
        limit_mm_per_prompt={modality: 1},
429
430
431
432
433
434
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
435
436
437


# LLaVA-1.6/LLaVA-NeXT
438
def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:
439
    assert modality == "image"
440

441
    prompts = [f"[INST] <image>\n{question} [/INST]" for question in questions]
442
443
444
    engine_args = EngineArgs(
        model="llava-hf/llava-v1.6-mistral-7b-hf",
        max_model_len=8192,
445
        limit_mm_per_prompt={modality: 1},
446
447
448
449
450
451
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
452
453
454
455


# LlaVA-NeXT-Video
# Currently only support for video input
456
def run_llava_next_video(questions: list[str], modality: str) -> ModelRequestData:
457
458
    assert modality == "video"

459
    prompts = [f"USER: <video>\n{question} ASSISTANT:" for question in questions]
460
461
462
    engine_args = EngineArgs(
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        max_model_len=8192,
463
        max_num_seqs=2,
464
        limit_mm_per_prompt={modality: 1},
465
466
467
468
469
470
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
471
472


473
# LLaVA-OneVision
474
def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData:
475
    if modality == "video":
476
477
        prompts = [
            f"<|im_start|>user <video>\n{question}<|im_end|> \
478
479
        <|im_start|>assistant\n"
            for question in questions
480
        ]
481
482

    elif modality == "image":
483
484
        prompts = [
            f"<|im_start|>user <image>\n{question}<|im_end|> \
485
486
        <|im_start|>assistant\n"
            for question in questions
487
        ]
488

489
490
491
    engine_args = EngineArgs(
        model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
        max_model_len=16384,
492
        limit_mm_per_prompt={modality: 1},
493
494
495
496
497
498
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
499
500


501
# Mantis
502
def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
503
    assert modality == "image"
504

505
506
    llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"  # noqa: E501
    prompts = [llama3_template.format(f"{question}\n<image>") for question in questions]
507

508
    engine_args = EngineArgs(
509
        model="TIGER-Lab/Mantis-8B-siglip-llama3",
510
        max_model_len=4096,
511
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
512
        limit_mm_per_prompt={modality: 1},
513
    )
514
    stop_token_ids = [128009]
515
516
517
518
519
520

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
521
522
523


# MiniCPM-V
524
def run_minicpmv_base(questions: list[str], modality: str, model_name):
525
526
    assert modality in ["image", "video"]
    # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
527
528
529
530
531
532
533

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
534
535
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

536
    # 2.6
537
538
539
540
541
542
543
544
545
    # model_name = "openbmb/MiniCPM-V-2_6"
    # o2.6

    # modality supports
    # 2.0: image
    # 2.5: image
    # 2.6: image, video
    # o2.6: image, video, audio
    # model_name = "openbmb/MiniCPM-o-2_6"
546
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
547
    engine_args = EngineArgs(
548
        model=model_name,
549
550
        max_model_len=4096,
        max_num_seqs=2,
551
        trust_remote_code=True,
552
        limit_mm_per_prompt={modality: 1},
553
    )
554
555
556
557
558
559
560
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

561
    # 2.6 / o2.6
562
    stop_tokens = ["<|im_end|>", "<|endoftext|>"]
563
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
564

565
566
567
568
569
    modality_placeholder = {
        "image": "(<image>./</image>)",
        "video": "(<video>./</video>)",
    }

570
571
    prompts = [
        tokenizer.apply_chat_template(
572
573
574
575
576
577
            [
                {
                    "role": "user",
                    "content": f"{modality_placeholder[modality]}\n{question}",
                }
            ],
578
            tokenize=False,
579
580
581
            add_generation_prompt=True,
        )
        for question in questions
582
    ]
583
584
585
586
587
588

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
589
590


591
def run_minicpmo(questions: list[str], modality: str) -> ModelRequestData:
592
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-o-2_6")
593
594


595
def run_minicpmv(questions: list[str], modality: str) -> ModelRequestData:
596
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-V-2_6")
597
598


599
600
601
602
603
604
605
606
607
608
609
610
# Mistral-3 HF-format
def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"

    # NOTE: Need L40 (or equivalent) to avoid OOM
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        tensor_parallel_size=2,
611
        limit_mm_per_prompt={modality: 1},
612
613
614
615
616
617
618
619
620
621
    )

    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


622
# LLama 3.2
623
def run_mllama(questions: list[str], modality: str) -> ModelRequestData:
624
625
    assert modality == "image"

626
    model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
627

628
629
630
631
632
    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (131072) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

    # The configuration below has been confirmed to launch on a single L40 GPU.
633
    engine_args = EngineArgs(
634
        model=model_name,
635
        max_model_len=8192,
636
        max_num_seqs=2,
637
        limit_mm_per_prompt={modality: 1},
638
639
    )

640
    tokenizer = AutoTokenizer.from_pretrained(model_name)
641
642
643
644
645
646
647
648
649
650
651
652
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": question}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )
653
654
655
656
657

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
658
659


660
def run_llama4(questions: list[str], modality: str) -> ModelRequestData:
661
662
663
664
665
666
667
668
669
670
    assert modality == "image"

    model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=4,
        tensor_parallel_size=8,
        gpu_memory_utilization=0.4,
671
        limit_mm_per_prompt={modality: 1},
672
673
674
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name)
675
676
677
678
679
680
681
682
683
684
685
686
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": f"{question}"}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )
687
688
689
690
691
692
693
694
    stop_token_ids = None
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )


695
# Molmo
696
def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
697
698
    assert modality == "image"

699
    model_name = "allenai/Molmo-7B-D-0924"
700

701
    engine_args = EngineArgs(
702
        model=model_name,
703
        trust_remote_code=True,
704
        dtype="bfloat16",
705
        limit_mm_per_prompt={modality: 1},
706
    )
707

708
709
    prompts = [
        f"<|im_start|>user <image>\n{question}<|im_end|> \
710
711
        <|im_start|>assistant\n"
        for question in questions
712
    ]
713
714
715
716
717

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
718
719


720
# NVLM-D
721
def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:
722
723
724
725
726
    assert modality == "image"

    model_name = "nvidia/NVLM-D-72B"

    # Adjust this as necessary to fit in GPU
727
    engine_args = EngineArgs(
728
729
730
731
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        tensor_parallel_size=4,
732
        limit_mm_per_prompt={modality: 1},
733
734
    )

735
736
737
738
739
740
741
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
742
743
744
745
746

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
747
748


749
750
# Ovis
def run_ovis(questions: list[str], modality: str) -> ModelRequestData:
751
752
753
754
755
756
757
758
759
760
    assert modality == "image"

    model_name = "AIDC-AI/Ovis2-1B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        trust_remote_code=True,
        dtype="half",
761
        limit_mm_per_prompt={modality: 1},
762
763
    )

764
765
766
767
768
769
770
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
771
772
773
774
775
776
777

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


778
# PaliGemma
779
def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:
780
    assert modality == "image"
781

782
    # PaliGemma has special prompt format for VQA
783
784
785
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma-3b-mix-224",
786
        limit_mm_per_prompt={modality: 1},
787
    )
788
789
790
791
792

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
793
794


795
# PaliGemma 2
796
def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:
797
    assert modality == "image"
798

799
    # PaliGemma 2 has special prompt format for VQA
800
801
802
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma2-3b-ft-docci-448",
803
        limit_mm_per_prompt={modality: 1},
804
    )
805
806
807
808
809

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
810
811


812
# Phi-3-Vision
813
def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
814
815
    assert modality == "image"

816
817
818
819
    prompts = [
        f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
        for question in questions
    ]
820

821
822
823
824
825
826
827
828
829
830
831
832
    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
833
    engine_args = EngineArgs(
834
835
        model="microsoft/Phi-3.5-vision-instruct",
        trust_remote_code=True,
836
        max_model_len=4096,
837
        max_num_seqs=2,
838
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
839
        mm_processor_kwargs={"num_crops": 16},
840
        limit_mm_per_prompt={modality: 1},
841
    )
842
843
844
845
846

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
847
848


849
# Phi-4-multimodal-instruct
850
def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
851
852
853
854
855
856
857
858
859
860
    """
    Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
    show how to process image inputs.
    """
    assert modality == "image"
    model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
    # Since the vision-lora and speech-lora co-exist with the base model,
    # we have to manually specify the path of the lora weights.
    vision_lora_path = os.path.join(model_path, "vision-lora")
    prompts = [
861
        f"<|user|><|image_1|>{question}<|end|><|assistant|>" for question in questions
862
    ]
863
    engine_args = EngineArgs(
864
865
        model=model_path,
        trust_remote_code=True,
866
        max_model_len=5120,
867
        max_num_seqs=2,
868
        max_num_batched_tokens=12800,
869
870
        enable_lora=True,
        max_lora_rank=320,
871
872
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
        mm_processor_kwargs={"dynamic_hd": 16},
873
        limit_mm_per_prompt={modality: 1},
874
875
    )

876
877
878
879
880
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
    )
881
882


883
# Pixtral HF-format
884
def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
885
886
887
888
    assert modality == "image"

    model_name = "mistral-community/pixtral-12b"

889
    # NOTE: Need L40 (or equivalent) to avoid OOM
890
    engine_args = EngineArgs(
891
        model=model_name,
892
        max_model_len=6144,
893
        max_num_seqs=2,
894
        limit_mm_per_prompt={modality: 1},
895
896
    )

897
    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
898
899
900
901
902

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
903
904


905
# Qwen
906
def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:
907
908
    assert modality == "image"

909
    engine_args = EngineArgs(
910
        model="Qwen/Qwen-VL",
911
        trust_remote_code=True,
912
913
        max_model_len=1024,
        max_num_seqs=2,
914
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
915
        limit_mm_per_prompt={modality: 1},
916
917
    )

918
    prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]
919
920
921
922
923

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
924
925


926
# Qwen2-VL
927
def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:
928
    model_name = "Qwen/Qwen2-VL-7B-Instruct"
929

930
    engine_args = EngineArgs(
931
        model=model_name,
932
933
934
        max_model_len=4096,
        max_num_seqs=5,
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
935
        mm_processor_kwargs={
936
937
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
938
        },
939
        limit_mm_per_prompt={modality: 1},
940
    )
941

942
943
944
945
946
    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

947
    prompts = [
948
949
950
951
952
953
954
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
955
    ]
956
957
958
959
960

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
961
962


Roger Wang's avatar
Roger Wang committed
963
# Qwen2.5-VL
964
def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
Roger Wang's avatar
Roger Wang committed
965
966
    model_name = "Qwen/Qwen2.5-VL-3B-Instruct"

967
    engine_args = EngineArgs(
Roger Wang's avatar
Roger Wang committed
968
969
970
971
972
973
974
975
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": 1,
        },
976
        limit_mm_per_prompt={modality: 1},
Roger Wang's avatar
Roger Wang committed
977
978
979
980
981
982
983
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

984
    prompts = [
985
986
987
988
989
990
991
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
992
    ]
993
994
995
996
997

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
Roger Wang's avatar
Roger Wang committed
998
999


1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
# Qwen2.5-Omni
def run_qwen2_5_omni(questions: list[str], modality: str):
    model_name = "Qwen/Qwen2.5-Omni-7B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": [1],
        },
1013
        limit_mm_per_prompt={modality: 1},
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
    )

    if modality == "image":
        placeholder = "<|IMAGE|>"
    elif modality == "video":
        placeholder = "<|VIDEO|>"

    default_system = (
        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
        "Group, capable of perceiving auditory and visual inputs, as well as "
1024
1025
        "generating text and speech."
    )
1026

1027
1028
1029
1030
1031
1032
1033
1034
1035
    prompts = [
        (
            f"<|im_start|>system\n{default_system}<|im_end|>\n"
            f"<|im_start|>user\n<|vision_bos|>{placeholder}<|vision_eos|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]
1036
1037
1038
1039
1040
1041
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
# SkyworkR1V
def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "Skywork/Skywork-R1V-38B"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
1052
        limit_mm_per_prompt={modality: 1},
1053
1054
    )

1055
1056
1057
1058
1059
1060
1061
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074

    # Stop tokens for SkyworkR1V
    # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
    stop_tokens = ["<|end▁of▁sentence|>", "<|endoftext|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )


1075
model_example_map = {
1076
    "aria": run_aria,
Jennifer Zhao's avatar
Jennifer Zhao committed
1077
    "aya_vision": run_aya_vision,
1078
1079
    "blip-2": run_blip2,
    "chameleon": run_chameleon,
1080
    "deepseek_vl_v2": run_deepseek_vl2,
1081
    "florence2": run_florence2,
1082
    "fuyu": run_fuyu,
1083
    "gemma3": run_gemma3,
1084
1085
1086
1087
    "glm4v": run_glm4v,
    "h2ovl_chat": run_h2ovl,
    "idefics3": run_idefics3,
    "internvl_chat": run_internvl,
1088
    "kimi_vl": run_kimi_vl,
1089
1090
    "llava": run_llava,
    "llava-next": run_llava_next,
1091
    "llava-next-video": run_llava_next_video,
1092
    "llava-onevision": run_llava_onevision,
1093
    "mantis": run_mantis,
1094
    "minicpmo": run_minicpmo,
1095
    "minicpmv": run_minicpmv,
1096
    "mistral3": run_mistral3,
1097
    "mllama": run_mllama,
1098
    "llama4": run_llama4,
1099
    "molmo": run_molmo,
1100
    "NVLM_D": run_nvlm_d,
1101
    "ovis": run_ovis,
1102
1103
1104
    "paligemma": run_paligemma,
    "paligemma2": run_paligemma2,
    "phi3_v": run_phi3v,
1105
    "phi4_mm": run_phi4mm,
1106
    "pixtral_hf": run_pixtral_hf,
1107
    "qwen_vl": run_qwen_vl,
1108
    "qwen2_vl": run_qwen2_vl,
Roger Wang's avatar
Roger Wang committed
1109
    "qwen2_5_vl": run_qwen2_5_vl,
1110
    "qwen2_5_omni": run_qwen2_5_omni,
1111
    "skywork_chat": run_skyworkr1v,
1112
    "smolvlm": run_smolvlm,
汪志鹏's avatar
汪志鹏 committed
1113
    "tarsier": run_tarsier,
1114
1115
1116
}


1117
1118
1119
1120
1121
1122
1123
1124
1125
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
1126
        image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
1127
1128
1129
1130
1131
1132
        img_questions = [
            "What is the content of this image?",
            "Describe the content of this image in detail.",
            "What's in the image?",
            "Where is this image taken?",
        ]
1133
1134
1135

        return {
            "data": image,
1136
            "questions": img_questions,
1137
1138
1139
1140
        }

    if args.modality == "video":
        # Input video and question
1141
        video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
1142
        vid_questions = ["Why is this video funny?"]
1143
1144
1145

        return {
            "data": video,
1146
            "questions": vid_questions,
1147
1148
1149
1150
1151
1152
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


1153
1154
1155
1156
def apply_image_repeat(
    image_repeat_prob, num_prompts, data, prompts: list[str], modality
):
    """Repeats images with provided probability of "image_repeat_prob".
1157
1158
    Used to simulate hit/miss for the MM preprocessor cache.
    """
1159
    assert image_repeat_prob <= 1.0 and image_repeat_prob >= 0
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
    no_yes = [0, 1]
    probs = [1.0 - image_repeat_prob, image_repeat_prob]

    inputs = []
    cur_image = data
    for i in range(num_prompts):
        if image_repeat_prob is not None:
            res = random.choices(no_yes, probs)[0]
            if res == 0:
                # No repeat => Modify one pixel
                cur_image = cur_image.copy()
                new_val = (i // 256 // 256, i // 256, i % 256)
                cur_image.putpixel((0, 0), new_val)

1174
1175
1176
1177
        inputs.append(
            {
                "prompt": prompts[i % len(prompts)],
                "multi_modal_data": {modality: cur_image},
1178
            }
1179
        )
1180
1181
1182
1183

    return inputs


1184
1185
1186
1187
@contextmanager
def time_counter(enable: bool):
    if enable:
        import time
1188

1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
        start_time = time.time()
        yield
        elapsed_time = time.time() - start_time
        print("-" * 50)
        print("-- generate time = {}".format(elapsed_time))
        print("-" * 50)
    else:
        yield


1199
1200
def parse_args():
    parser = FlexibleArgumentParser(
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
        description="Demo on using vLLM for offline inference with "
        "vision language models for text generation"
    )
    parser.add_argument(
        "--model-type",
        "-m",
        type=str,
        default="llava",
        choices=model_example_map.keys(),
        help='Huggingface "model_type".',
    )
    parser.add_argument(
        "--num-prompts", type=int, default=4, help="Number of prompts to run."
    )
    parser.add_argument(
        "--modality",
        type=str,
        default="image",
        choices=["image", "video"],
        help="Modality of the input.",
    )
    parser.add_argument(
        "--num-frames",
        type=int,
        default=16,
        help="Number of frames to extract from the video.",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Set the seed when initializing `vllm.LLM`.",
    )
1234
1235

    parser.add_argument(
1236
        "--image-repeat-prob",
1237
1238
        type=float,
        default=None,
1239
1240
        help="Simulates the hit-ratio for multi-modal preprocessor cache (if enabled)",
    )
1241
1242

    parser.add_argument(
1243
1244
1245
1246
        "--disable-mm-preprocessor-cache",
        action="store_true",
        help="If True, disables caching of multi-modal preprocessor/mapper.",
    )
1247
1248

    parser.add_argument(
1249
1250
1251
1252
        "--time-generate",
        action="store_true",
        help="If True, then print the total generate() call time",
    )
1253
1254

    parser.add_argument(
1255
1256
1257
1258
1259
        "--use-different-prompt-per-request",
        action="store_true",
        help="If True, then use different prompt (with the same multi-modal "
        "data) for each request.",
    )
1260
1261
1262
    return parser.parse_args()


1263
1264
1265
1266
1267
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

1268
1269
1270
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
1271
    questions = mm_input["questions"]
1272

1273
1274
    req_data = model_example_map[model](questions, modality)

1275
1276
1277
    # Disable other modalities to save memory
    default_limits = {"image": 0, "video": 0, "audio": 0}
    req_data.engine_args.limit_mm_per_prompt = default_limits | dict(
1278
1279
        req_data.engine_args.limit_mm_per_prompt or {}
    )
1280
1281
1282
1283
1284

    engine_args = asdict(req_data.engine_args) | {
        "seed": args.seed,
        "disable_mm_preprocessor_cache": args.disable_mm_preprocessor_cache,
    }
1285
1286
    llm = LLM(**engine_args)

1287
    # Don't want to check the flag multiple times, so just hijack `prompts`.
1288
1289
1290
1291
1292
    prompts = (
        req_data.prompts
        if args.use_different_prompt_per_request
        else [req_data.prompts[0]]
    )
1293
1294
1295

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
1296
1297
1298
    sampling_params = SamplingParams(
        temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
    )
1299
1300
1301
1302
1303

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        inputs = {
1304
            "prompt": prompts[0],
1305
            "multi_modal_data": {modality: data},
1306
1307
1308
        }
    else:
        # Batch inference
1309
1310
        if args.image_repeat_prob is not None:
            # Repeat images with specified probability of "image_repeat_prob"
1311
1312
1313
            inputs = apply_image_repeat(
                args.image_repeat_prob, args.num_prompts, data, prompts, modality
            )
1314
1315
        else:
            # Use the same image for all prompts
1316
1317
1318
1319
1320
1321
1322
            inputs = [
                {
                    "prompt": prompts[i % len(prompts)],
                    "multi_modal_data": {modality: data},
                }
                for i in range(args.num_prompts)
            ]
1323

1324
    # Add LoRA request if applicable
1325
1326
1327
    lora_request = (
        req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
    )
1328

1329
1330
1331
1332
1333
1334
    with time_counter(args.time_generate):
        outputs = llm.generate(
            inputs,
            sampling_params=sampling_params,
            lora_request=lora_request,
        )
1335

1336
    print("-" * 50)
1337
1338
1339
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)
1340
        print("-" * 50)
1341
1342
1343


if __name__ == "__main__":
1344
    args = parse_args()
1345
    main(args)