"vllm/multimodal/processing/processor.py" did not exist on "0ad9951c416d33c5da4f7a504fb162cbe62386f5"
vision_language.py 37.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""
Cyrus Leung's avatar
Cyrus Leung committed
4
5
This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.
6
7
8
9

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
10

11
import os
12
import random
13
from contextlib import contextmanager
14
15
from dataclasses import asdict
from typing import NamedTuple, Optional
16

17
from huggingface_hub import snapshot_download
18
19
from transformers import AutoTokenizer

20
from vllm import LLM, EngineArgs, SamplingParams
21
from vllm.assets.image import ImageAsset
22
from vllm.assets.video import VideoAsset
23
from vllm.lora.request import LoRARequest
24
from vllm.multimodal.image import convert_image_mode
25
26
from vllm.utils import FlexibleArgumentParser

27
28
29
30
31
32
33
34

class ModelRequestData(NamedTuple):
    engine_args: EngineArgs
    prompts: list[str]
    stop_token_ids: Optional[list[int]] = None
    lora_requests: Optional[list[LoRARequest]] = None


35
36
37
38
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
# lower-end GPUs.
# Unless specified, these settings have been tested to work on a single L4.

39

40
# Aria
41
def run_aria(questions: list[str], modality: str) -> ModelRequestData:
42
43
44
    assert modality == "image"
    model_name = "rhymes-ai/Aria"

45
    # NOTE: Need L40 (or equivalent) to avoid OOM
46
47
48
49
50
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        dtype="bfloat16",
51
        limit_mm_per_prompt={modality: 1},
52
    )
53

54
55
56
57
58
59
60
    prompts = [
        (
            f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
            "<|im_end|>\n<|im_start|>assistant\n"
        )
        for question in questions
    ]
61
62

    stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
63
64
65
66
67
68

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
69
70


Jennifer Zhao's avatar
Jennifer Zhao committed
71
72
73
74
75
76
77
78
79
80
# Aya Vision
def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "CohereForAI/aya-vision-8b"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        mm_processor_kwargs={"crop_to_patches": True},
81
        limit_mm_per_prompt={modality: 1},
Jennifer Zhao's avatar
Jennifer Zhao committed
82
83
84
85
86
87
88
89
90
91
92
    )
    prompts = [
        f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
        for question in questions
    ]
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


93
# BLIP-2
94
def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
95
96
97
98
    assert modality == "image"

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
99
    prompts = [f"Question: {question} Answer:" for question in questions]
100
    engine_args = EngineArgs(
101
        model="Salesforce/blip2-opt-6.7b",
102
        limit_mm_per_prompt={modality: 1},
103
104
105
106
107
108
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
109
110
111


# Chameleon
112
def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:
113
114
    assert modality == "image"

115
    prompts = [f"{question}<image>" for question in questions]
116
117
118
119
    engine_args = EngineArgs(
        model="facebook/chameleon-7b",
        max_model_len=4096,
        max_num_seqs=2,
120
        limit_mm_per_prompt={modality: 1},
121
122
123
124
125
126
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
127
128


129
# Deepseek-VL2
130
def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:
131
132
    assert modality == "image"

133
    model_name = "deepseek-ai/deepseek-vl2-tiny"
134

135
136
137
138
139
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
140
        limit_mm_per_prompt={modality: 1},
141
    )
142

143
    prompts = [
144
        f"<|User|>: <image>\n{question}\n\n<|Assistant|>:" for question in questions
145
    ]
146
147
148
149
150

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
151
152


153
# Florence2
154
def run_florence2(questions: list[str], modality: str) -> ModelRequestData:
155
156
    assert modality == "image"

157
158
    engine_args = EngineArgs(
        model="microsoft/Florence-2-large",
159
        tokenizer="Isotr0py/Florence-2-tokenizer",
160
161
        max_model_len=4096,
        max_num_seqs=2,
162
163
        trust_remote_code=True,
        dtype="bfloat16",
164
        limit_mm_per_prompt={modality: 1},
165
    )
166

167
168
169
170
171
172
    prompts = ["<MORE_DETAILED_CAPTION>" for _ in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
173
174


175
# Fuyu
176
def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:
177
178
    assert modality == "image"

179
    prompts = [f"{question}\n" for question in questions]
180
181
182
183
    engine_args = EngineArgs(
        model="adept/fuyu-8b",
        max_model_len=2048,
        max_num_seqs=2,
184
        limit_mm_per_prompt={modality: 1},
185
186
187
188
189
190
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
191
192


193
# Gemma 3
194
def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:
195
196
197
    assert modality == "image"
    model_name = "google/gemma-3-4b-it"

198
    engine_args = EngineArgs(
199
200
201
202
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        mm_processor_kwargs={"do_pan_and_scan": True},
203
        limit_mm_per_prompt={modality: 1},
204
    )
205

206
207
208
209
210
211
212
213
    prompts = [
        (
            "<bos><start_of_turn>user\n"
            f"<start_of_image>{question}<end_of_turn>\n"
            "<start_of_turn>model\n"
        )
        for question in questions
    ]
214
215
216
217
218

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
219
220


221
# GLM-4v
222
def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:
223
224
225
    assert modality == "image"
    model_name = "THUDM/glm-4v-9b"

226
227
228
229
230
231
232
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=2048,
        max_num_seqs=2,
        trust_remote_code=True,
        enforce_eager=True,
        hf_overrides={"architectures": ["GLM4VForCausalLM"]},
233
        limit_mm_per_prompt={modality: 1},
234
    )
235

236
237
    prompts = [
        f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
238
239
        {question}<|assistant|>"
        for question in questions
240
    ]
241

242
    stop_token_ids = [151329, 151336, 151338]
243
244
245
246
247
248

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
249
250
251


# H2OVL-Mississippi
252
def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
253
254
    assert modality == "image"

255
    model_name = "h2oai/h2ovl-mississippi-800m"
256

257
    engine_args = EngineArgs(
258
259
260
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
261
        limit_mm_per_prompt={modality: 1},
262
263
    )

264
265
266
267
268
269
270
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
271
272

    # Stop tokens for H2OVL-Mississippi
273
    # https://huggingface.co/h2oai/h2ovl-mississippi-800m
274
    stop_token_ids = [tokenizer.eos_token_id]
275
276
277
278
279
280

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
281
282
283


# Idefics3-8B-Llama3
284
def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
285
286
287
    assert modality == "image"
    model_name = "HuggingFaceM4/Idefics3-8B-Llama3"

288
    engine_args = EngineArgs(
289
290
291
292
293
294
295
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        # if you are running out of memory, you can reduce the "longest_edge".
        # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
        mm_processor_kwargs={
296
            "size": {"longest_edge": 3 * 364},
297
        },
298
        limit_mm_per_prompt={modality: 1},
299
    )
300
301
302
303
    prompts = [
        (f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]
304
305
306
307
308

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
309
310


311
312
313
314
315
316
317
318
319
320
321
# SmolVLM2-2.2B-Instruct
def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        mm_processor_kwargs={
322
            "max_image_size": {"longest_edge": 384},
323
        },
324
        limit_mm_per_prompt={modality: 1},
325
326
327
328
329
330
331
332
333
334
335
336
    )
    prompts = [
        (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")
        for question in questions
    ]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


汪志鹏's avatar
汪志鹏 committed
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
# omni-research/Tarsier-7b
def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"
    model_name = "omni-research/Tarsier-7b"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        limit_mm_per_prompt={modality: 1},
    )
    prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


356
# InternVL
357
def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
358
    model_name = "OpenGVLab/InternVL3-2B"
359

360
    engine_args = EngineArgs(
361
362
        model=model_name,
        trust_remote_code=True,
363
        max_model_len=8192,
364
        limit_mm_per_prompt={modality: 1},
365
366
    )

367
368
369
370
371
    if modality == "image":
        placeholder = "<image>"
    elif modality == "video":
        placeholder = "<video>"

372
373
374
375
376
377
378
379
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"{placeholder}\n{question}"}]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
380
381
382
383
384
385
386

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
387
    stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
388
389
390
391
392
393

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
394
395


396
397
398
399
400
401
402
# Kimi-VL
def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    prompts = [
        "<|im_user|>user<|im_middle|><|media_start|>image<|media_content|>"
        f"<|media_pad|><|media_end|>{question}<|im_end|>"
403
404
        "<|im_assistant|>assistant<|im_middle|>"
        for question in questions
405
406
407
408
409
    ]

    engine_args = EngineArgs(
        model="moonshotai/Kimi-VL-A3B-Instruct",
        trust_remote_code=True,
Cyrus Leung's avatar
Cyrus Leung committed
410
        max_model_len=4096,
411
        limit_mm_per_prompt={modality: 1},
412
413
414
415
416
417
418
419
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


420
# LLaVA-1.5
421
def run_llava(questions: list[str], modality: str) -> ModelRequestData:
422
    assert modality == "image"
423

424
    prompts = [f"USER: <image>\n{question}\nASSISTANT:" for question in questions]
425

426
427
428
    engine_args = EngineArgs(
        model="llava-hf/llava-1.5-7b-hf",
        max_model_len=4096,
429
        limit_mm_per_prompt={modality: 1},
430
431
432
433
434
435
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
436
437
438


# LLaVA-1.6/LLaVA-NeXT
439
def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:
440
    assert modality == "image"
441

442
    prompts = [f"[INST] <image>\n{question} [/INST]" for question in questions]
443
444
445
    engine_args = EngineArgs(
        model="llava-hf/llava-v1.6-mistral-7b-hf",
        max_model_len=8192,
446
        limit_mm_per_prompt={modality: 1},
447
448
449
450
451
452
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
453
454
455
456


# LlaVA-NeXT-Video
# Currently only support for video input
457
def run_llava_next_video(questions: list[str], modality: str) -> ModelRequestData:
458
459
    assert modality == "video"

460
    prompts = [f"USER: <video>\n{question} ASSISTANT:" for question in questions]
461
462
463
    engine_args = EngineArgs(
        model="llava-hf/LLaVA-NeXT-Video-7B-hf",
        max_model_len=8192,
464
        max_num_seqs=2,
465
        limit_mm_per_prompt={modality: 1},
466
467
468
469
470
471
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
472
473


474
# LLaVA-OneVision
475
def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData:
476
    if modality == "video":
477
478
        prompts = [
            f"<|im_start|>user <video>\n{question}<|im_end|> \
479
480
        <|im_start|>assistant\n"
            for question in questions
481
        ]
482
483

    elif modality == "image":
484
485
        prompts = [
            f"<|im_start|>user <image>\n{question}<|im_end|> \
486
487
        <|im_start|>assistant\n"
            for question in questions
488
        ]
489

490
491
492
    engine_args = EngineArgs(
        model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
        max_model_len=16384,
493
        limit_mm_per_prompt={modality: 1},
494
495
496
497
498
499
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
500
501


502
# Mantis
503
def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
504
    assert modality == "image"
505

506
507
    llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"  # noqa: E501
    prompts = [llama3_template.format(f"{question}\n<image>") for question in questions]
508

509
    engine_args = EngineArgs(
510
        model="TIGER-Lab/Mantis-8B-siglip-llama3",
511
        max_model_len=4096,
512
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
513
        limit_mm_per_prompt={modality: 1},
514
    )
515
    stop_token_ids = [128009]
516
517
518
519
520
521

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
522
523
524


# MiniCPM-V
525
def run_minicpmv_base(questions: list[str], modality: str, model_name):
526
527
    assert modality in ["image", "video"]
    # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
528
529
530
531
532
533
534

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
535
536
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

537
    # 2.6
538
539
540
541
542
543
544
545
546
    # model_name = "openbmb/MiniCPM-V-2_6"
    # o2.6

    # modality supports
    # 2.0: image
    # 2.5: image
    # 2.6: image, video
    # o2.6: image, video, audio
    # model_name = "openbmb/MiniCPM-o-2_6"
547
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
548
    engine_args = EngineArgs(
549
        model=model_name,
550
551
        max_model_len=4096,
        max_num_seqs=2,
552
        trust_remote_code=True,
553
        limit_mm_per_prompt={modality: 1},
554
    )
555
556
557
558
559
560
561
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

562
    # 2.6 / o2.6
563
    stop_tokens = ["<|im_end|>", "<|endoftext|>"]
564
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
565

566
567
568
569
570
    modality_placeholder = {
        "image": "(<image>./</image>)",
        "video": "(<video>./</video>)",
    }

571
572
    prompts = [
        tokenizer.apply_chat_template(
573
574
575
576
577
578
            [
                {
                    "role": "user",
                    "content": f"{modality_placeholder[modality]}\n{question}",
                }
            ],
579
            tokenize=False,
580
581
582
            add_generation_prompt=True,
        )
        for question in questions
583
    ]
584
585
586
587
588
589

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )
590
591


592
def run_minicpmo(questions: list[str], modality: str) -> ModelRequestData:
593
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-o-2_6")
594
595


596
def run_minicpmv(questions: list[str], modality: str) -> ModelRequestData:
597
    return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-V-2_6")
598
599


600
601
602
603
604
605
606
607
608
609
610
611
# Mistral-3 HF-format
def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"

    # NOTE: Need L40 (or equivalent) to avoid OOM
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        tensor_parallel_size=2,
612
        limit_mm_per_prompt={modality: 1},
613
614
615
616
617
618
619
620
621
622
    )

    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


623
# LLama 3.2
624
def run_mllama(questions: list[str], modality: str) -> ModelRequestData:
625
626
    assert modality == "image"

627
    model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
628

629
630
631
632
633
    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (131072) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

    # The configuration below has been confirmed to launch on a single L40 GPU.
634
    engine_args = EngineArgs(
635
        model=model_name,
636
        max_model_len=8192,
637
        max_num_seqs=2,
638
        limit_mm_per_prompt={modality: 1},
639
640
    )

641
    tokenizer = AutoTokenizer.from_pretrained(model_name)
642
643
644
645
646
647
648
649
650
651
652
653
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": question}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )
654
655
656
657
658

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
659
660


661
def run_llama4(questions: list[str], modality: str) -> ModelRequestData:
662
663
664
665
666
667
668
669
670
671
    assert modality == "image"

    model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=4,
        tensor_parallel_size=8,
        gpu_memory_utilization=0.4,
672
        limit_mm_per_prompt={modality: 1},
673
674
675
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name)
676
677
678
679
680
681
682
683
684
685
686
687
    messages = [
        [
            {
                "role": "user",
                "content": [{"type": "image"}, {"type": "text", "text": f"{question}"}],
            }
        ]
        for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True, tokenize=False
    )
688
689
690
691
692
693
694
695
    stop_token_ids = None
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )


696
# Molmo
697
def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
698
699
    assert modality == "image"

700
    model_name = "allenai/Molmo-7B-D-0924"
701

702
    engine_args = EngineArgs(
703
        model=model_name,
704
        trust_remote_code=True,
705
        dtype="bfloat16",
706
        limit_mm_per_prompt={modality: 1},
707
    )
708

709
710
    prompts = [
        f"<|im_start|>user <image>\n{question}<|im_end|> \
711
712
        <|im_start|>assistant\n"
        for question in questions
713
    ]
714
715
716
717
718

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
719
720


721
# NVLM-D
722
def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:
723
724
725
726
727
    assert modality == "image"

    model_name = "nvidia/NVLM-D-72B"

    # Adjust this as necessary to fit in GPU
728
    engine_args = EngineArgs(
729
730
731
732
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        tensor_parallel_size=4,
733
        limit_mm_per_prompt={modality: 1},
734
735
    )

736
737
738
739
740
741
742
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
743
744
745
746
747

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
748
749


750
751
# Ovis
def run_ovis(questions: list[str], modality: str) -> ModelRequestData:
752
753
754
755
756
757
758
759
760
761
    assert modality == "image"

    model_name = "AIDC-AI/Ovis2-1B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        trust_remote_code=True,
        dtype="half",
762
        limit_mm_per_prompt={modality: 1},
763
764
    )

765
766
767
768
769
770
771
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
772
773
774
775
776
777
778

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


779
# PaliGemma
780
def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:
781
    assert modality == "image"
782

783
    # PaliGemma has special prompt format for VQA
784
785
786
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma-3b-mix-224",
787
        limit_mm_per_prompt={modality: 1},
788
    )
789
790
791
792
793

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
794
795


796
# PaliGemma 2
797
def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:
798
    assert modality == "image"
799

800
    # PaliGemma 2 has special prompt format for VQA
801
802
803
    prompts = ["caption en" for _ in questions]
    engine_args = EngineArgs(
        model="google/paligemma2-3b-ft-docci-448",
804
        limit_mm_per_prompt={modality: 1},
805
    )
806
807
808
809
810

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
811
812


813
# Phi-3-Vision
814
def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
815
816
    assert modality == "image"

817
818
819
820
    prompts = [
        f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
        for question in questions
    ]
821

822
823
824
825
826
827
828
829
830
831
832
833
    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
834
    engine_args = EngineArgs(
835
836
        model="microsoft/Phi-3.5-vision-instruct",
        trust_remote_code=True,
837
        max_model_len=4096,
838
        max_num_seqs=2,
839
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
840
        mm_processor_kwargs={"num_crops": 16},
841
        limit_mm_per_prompt={modality: 1},
842
    )
843
844
845
846
847

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
848
849


850
# Phi-4-multimodal-instruct
851
def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
852
853
854
855
856
857
858
859
860
861
    """
    Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
    show how to process image inputs.
    """
    assert modality == "image"
    model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
    # Since the vision-lora and speech-lora co-exist with the base model,
    # we have to manually specify the path of the lora weights.
    vision_lora_path = os.path.join(model_path, "vision-lora")
    prompts = [
862
        f"<|user|><|image_1|>{question}<|end|><|assistant|>" for question in questions
863
    ]
864
    engine_args = EngineArgs(
865
866
        model=model_path,
        trust_remote_code=True,
867
        max_model_len=5120,
868
        max_num_seqs=2,
869
        max_num_batched_tokens=12800,
870
871
        enable_lora=True,
        max_lora_rank=320,
872
873
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
        mm_processor_kwargs={"dynamic_hd": 16},
874
        limit_mm_per_prompt={modality: 1},
875
876
    )

877
878
879
880
881
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
    )
882
883


884
# Pixtral HF-format
885
def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
886
887
888
889
    assert modality == "image"

    model_name = "mistral-community/pixtral-12b"

890
    # NOTE: Need L40 (or equivalent) to avoid OOM
891
    engine_args = EngineArgs(
892
        model=model_name,
893
        max_model_len=6144,
894
        max_num_seqs=2,
895
        limit_mm_per_prompt={modality: 1},
896
897
    )

898
    prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
899
900
901
902
903

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
904
905


906
# Qwen
907
def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:
908
909
    assert modality == "image"

910
    engine_args = EngineArgs(
911
        model="Qwen/Qwen-VL",
912
        trust_remote_code=True,
913
914
        max_model_len=1024,
        max_num_seqs=2,
915
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
916
        limit_mm_per_prompt={modality: 1},
917
918
    )

919
    prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]
920
921
922
923
924

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
925
926


927
# Qwen2-VL
928
def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:
929
    model_name = "Qwen/Qwen2-VL-7B-Instruct"
930

931
    engine_args = EngineArgs(
932
        model=model_name,
933
934
935
        max_model_len=4096,
        max_num_seqs=5,
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
936
        mm_processor_kwargs={
937
938
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
939
        },
940
        limit_mm_per_prompt={modality: 1},
941
    )
942

943
944
945
946
947
    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

948
    prompts = [
949
950
951
952
953
954
955
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
956
    ]
957
958
959
960
961

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
962
963


Roger Wang's avatar
Roger Wang committed
964
# Qwen2.5-VL
965
def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
Roger Wang's avatar
Roger Wang committed
966
967
    model_name = "Qwen/Qwen2.5-VL-3B-Instruct"

968
    engine_args = EngineArgs(
Roger Wang's avatar
Roger Wang committed
969
970
971
972
973
974
975
976
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": 1,
        },
977
        limit_mm_per_prompt={modality: 1},
Roger Wang's avatar
Roger Wang committed
978
979
980
981
982
983
984
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

985
    prompts = [
986
987
988
989
990
991
992
        (
            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
            f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
993
    ]
994
995
996
997
998

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )
Roger Wang's avatar
Roger Wang committed
999
1000


1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
# Qwen2.5-Omni
def run_qwen2_5_omni(questions: list[str], modality: str):
    model_name = "Qwen/Qwen2.5-Omni-7B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": [1],
        },
1014
        limit_mm_per_prompt={modality: 1},
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
    )

    if modality == "image":
        placeholder = "<|IMAGE|>"
    elif modality == "video":
        placeholder = "<|VIDEO|>"

    default_system = (
        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
        "Group, capable of perceiving auditory and visual inputs, as well as "
1025
1026
        "generating text and speech."
    )
1027

1028
1029
1030
1031
1032
1033
1034
1035
1036
    prompts = [
        (
            f"<|im_start|>system\n{default_system}<|im_end|>\n"
            f"<|im_start|>user\n<|vision_bos|>{placeholder}<|vision_eos|>"
            f"{question}<|im_end|>\n"
            "<|im_start|>assistant\n"
        )
        for question in questions
    ]
1037
1038
1039
1040
1041
1042
    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
    )


1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
# SkyworkR1V
def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
    assert modality == "image"

    model_name = "Skywork/Skywork-R1V-38B"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
1053
        limit_mm_per_prompt={modality: 1},
1054
1055
    )

1056
1057
1058
1059
1060
1061
1062
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    messages = [
        [{"role": "user", "content": f"<image>\n{question}"}] for question in questions
    ]
    prompts = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075

    # Stop tokens for SkyworkR1V
    # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
    stop_tokens = ["<|end▁of▁sentence|>", "<|endoftext|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]

    return ModelRequestData(
        engine_args=engine_args,
        prompts=prompts,
        stop_token_ids=stop_token_ids,
    )


1076
model_example_map = {
1077
    "aria": run_aria,
Jennifer Zhao's avatar
Jennifer Zhao committed
1078
    "aya_vision": run_aya_vision,
1079
1080
    "blip-2": run_blip2,
    "chameleon": run_chameleon,
1081
    "deepseek_vl_v2": run_deepseek_vl2,
1082
    "florence2": run_florence2,
1083
    "fuyu": run_fuyu,
1084
    "gemma3": run_gemma3,
1085
1086
1087
1088
    "glm4v": run_glm4v,
    "h2ovl_chat": run_h2ovl,
    "idefics3": run_idefics3,
    "internvl_chat": run_internvl,
1089
    "kimi_vl": run_kimi_vl,
1090
1091
    "llava": run_llava,
    "llava-next": run_llava_next,
1092
    "llava-next-video": run_llava_next_video,
1093
    "llava-onevision": run_llava_onevision,
1094
    "mantis": run_mantis,
1095
    "minicpmo": run_minicpmo,
1096
    "minicpmv": run_minicpmv,
1097
    "mistral3": run_mistral3,
1098
    "mllama": run_mllama,
1099
    "llama4": run_llama4,
1100
    "molmo": run_molmo,
1101
    "NVLM_D": run_nvlm_d,
1102
    "ovis": run_ovis,
1103
1104
1105
    "paligemma": run_paligemma,
    "paligemma2": run_paligemma2,
    "phi3_v": run_phi3v,
1106
    "phi4_mm": run_phi4mm,
1107
    "pixtral_hf": run_pixtral_hf,
1108
    "qwen_vl": run_qwen_vl,
1109
    "qwen2_vl": run_qwen2_vl,
Roger Wang's avatar
Roger Wang committed
1110
    "qwen2_5_vl": run_qwen2_5_vl,
1111
    "qwen2_5_omni": run_qwen2_5_omni,
1112
    "skywork_chat": run_skyworkr1v,
1113
    "smolvlm": run_smolvlm,
汪志鹏's avatar
汪志鹏 committed
1114
    "tarsier": run_tarsier,
1115
1116
1117
}


1118
1119
1120
1121
1122
1123
1124
1125
1126
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
1127
        image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
1128
1129
1130
1131
1132
1133
        img_questions = [
            "What is the content of this image?",
            "Describe the content of this image in detail.",
            "What's in the image?",
            "Where is this image taken?",
        ]
1134
1135
1136

        return {
            "data": image,
1137
            "questions": img_questions,
1138
1139
1140
1141
        }

    if args.modality == "video":
        # Input video and question
1142
        video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
1143
        vid_questions = ["Why is this video funny?"]
1144
1145
1146

        return {
            "data": video,
1147
            "questions": vid_questions,
1148
1149
1150
1151
1152
1153
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


1154
1155
1156
1157
def apply_image_repeat(
    image_repeat_prob, num_prompts, data, prompts: list[str], modality
):
    """Repeats images with provided probability of "image_repeat_prob".
1158
1159
    Used to simulate hit/miss for the MM preprocessor cache.
    """
1160
    assert image_repeat_prob <= 1.0 and image_repeat_prob >= 0
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
    no_yes = [0, 1]
    probs = [1.0 - image_repeat_prob, image_repeat_prob]

    inputs = []
    cur_image = data
    for i in range(num_prompts):
        if image_repeat_prob is not None:
            res = random.choices(no_yes, probs)[0]
            if res == 0:
                # No repeat => Modify one pixel
                cur_image = cur_image.copy()
                new_val = (i // 256 // 256, i // 256, i % 256)
                cur_image.putpixel((0, 0), new_val)

1175
1176
1177
1178
        inputs.append(
            {
                "prompt": prompts[i % len(prompts)],
                "multi_modal_data": {modality: cur_image},
1179
            }
1180
        )
1181
1182
1183
1184

    return inputs


1185
1186
1187
1188
@contextmanager
def time_counter(enable: bool):
    if enable:
        import time
1189

1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
        start_time = time.time()
        yield
        elapsed_time = time.time() - start_time
        print("-" * 50)
        print("-- generate time = {}".format(elapsed_time))
        print("-" * 50)
    else:
        yield


1200
1201
def parse_args():
    parser = FlexibleArgumentParser(
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
        description="Demo on using vLLM for offline inference with "
        "vision language models for text generation"
    )
    parser.add_argument(
        "--model-type",
        "-m",
        type=str,
        default="llava",
        choices=model_example_map.keys(),
        help='Huggingface "model_type".',
    )
    parser.add_argument(
        "--num-prompts", type=int, default=4, help="Number of prompts to run."
    )
    parser.add_argument(
        "--modality",
        type=str,
        default="image",
        choices=["image", "video"],
        help="Modality of the input.",
    )
    parser.add_argument(
        "--num-frames",
        type=int,
        default=16,
        help="Number of frames to extract from the video.",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Set the seed when initializing `vllm.LLM`.",
    )
1235
1236

    parser.add_argument(
1237
        "--image-repeat-prob",
1238
1239
        type=float,
        default=None,
1240
1241
        help="Simulates the hit-ratio for multi-modal preprocessor cache (if enabled)",
    )
1242
1243

    parser.add_argument(
1244
1245
1246
1247
        "--disable-mm-preprocessor-cache",
        action="store_true",
        help="If True, disables caching of multi-modal preprocessor/mapper.",
    )
1248
1249

    parser.add_argument(
1250
1251
1252
1253
        "--time-generate",
        action="store_true",
        help="If True, then print the total generate() call time",
    )
1254
1255

    parser.add_argument(
1256
1257
1258
1259
1260
        "--use-different-prompt-per-request",
        action="store_true",
        help="If True, then use different prompt (with the same multi-modal "
        "data) for each request.",
    )
1261
1262
1263
    return parser.parse_args()


1264
1265
1266
1267
1268
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

1269
1270
1271
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
1272
    questions = mm_input["questions"]
1273

1274
1275
    req_data = model_example_map[model](questions, modality)

1276
1277
1278
    # Disable other modalities to save memory
    default_limits = {"image": 0, "video": 0, "audio": 0}
    req_data.engine_args.limit_mm_per_prompt = default_limits | dict(
1279
1280
        req_data.engine_args.limit_mm_per_prompt or {}
    )
1281
1282
1283
1284
1285

    engine_args = asdict(req_data.engine_args) | {
        "seed": args.seed,
        "disable_mm_preprocessor_cache": args.disable_mm_preprocessor_cache,
    }
1286
1287
    llm = LLM(**engine_args)

1288
    # Don't want to check the flag multiple times, so just hijack `prompts`.
1289
1290
1291
1292
1293
    prompts = (
        req_data.prompts
        if args.use_different_prompt_per_request
        else [req_data.prompts[0]]
    )
1294
1295
1296

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
1297
1298
1299
    sampling_params = SamplingParams(
        temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
    )
1300
1301
1302
1303
1304

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        inputs = {
1305
            "prompt": prompts[0],
1306
            "multi_modal_data": {modality: data},
1307
1308
1309
        }
    else:
        # Batch inference
1310
1311
        if args.image_repeat_prob is not None:
            # Repeat images with specified probability of "image_repeat_prob"
1312
1313
1314
            inputs = apply_image_repeat(
                args.image_repeat_prob, args.num_prompts, data, prompts, modality
            )
1315
1316
        else:
            # Use the same image for all prompts
1317
1318
1319
1320
1321
1322
1323
            inputs = [
                {
                    "prompt": prompts[i % len(prompts)],
                    "multi_modal_data": {modality: data},
                }
                for i in range(args.num_prompts)
            ]
1324

1325
    # Add LoRA request if applicable
1326
1327
1328
    lora_request = (
        req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
    )
1329

1330
1331
1332
1333
1334
1335
    with time_counter(args.time_generate):
        outputs = llm.generate(
            inputs,
            sampling_params=sampling_params,
            lora_request=lora_request,
        )
1336

1337
    print("-" * 50)
1338
1339
1340
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)
1341
        print("-" * 50)
1342
1343
1344


if __name__ == "__main__":
1345
    args = parse_args()
1346
    main(args)