"vscode:/vscode.git/clone" did not exist on "4f02cb2eac20a1281784914acca80cd1fe2bfd15"
vision_language_multi_image.py 42.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
"""
This example shows how to use vLLM for running offline inference with
Cyrus Leung's avatar
Cyrus Leung committed
5
6
multi-image input on vision language models for text generation,
using the chat template defined by the model.
7
"""
8

9
import os
10
from argparse import Namespace
11
from dataclasses import asdict
12
from typing import NamedTuple, Optional
13

14
from huggingface_hub import snapshot_download
15
from PIL.Image import Image
16
from transformers import AutoProcessor, AutoTokenizer
17

18
from vllm import LLM, EngineArgs, SamplingParams
19
from vllm.lora.request import LoRARequest
20
21
22
23
24
25
26
from vllm.multimodal.utils import fetch_image
from vllm.utils import FlexibleArgumentParser

QUESTION = "What is the content of each image?"
IMAGE_URLS = [
    "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg",
27
28
29
30
31
32
33
34
35
36
    "https://upload.wikimedia.org/wikipedia/commons/2/26/Ultramarine_Flycatcher_%28Ficedula_superciliaris%29_Naggar%2C_Himachal_Pradesh%2C_2013_%28cropped%29.JPG",
    "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e5/Anim1754_-_Flickr_-_NOAA_Photo_Library_%281%29.jpg/2560px-Anim1754_-_Flickr_-_NOAA_Photo_Library_%281%29.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/d/d4/Starfish%2C_Caswell_Bay_-_geograph.org.uk_-_409413.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/6/69/Grapevinesnail_01.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/thumb/0/0b/Texas_invasive_Musk_Thistle_1.jpg/1920px-Texas_invasive_Musk_Thistle_1.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/thumb/7/7a/Huskiesatrest.jpg/2880px-Huskiesatrest.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/thumb/6/68/Orange_tabby_cat_sitting_on_fallen_leaves-Hisashi-01A.jpg/1920px-Orange_tabby_cat_sitting_on_fallen_leaves-Hisashi-01A.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/3/30/George_the_amazing_guinea_pig.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/thumb/1/1f/Oryctolagus_cuniculus_Rcdo.jpg/1920px-Oryctolagus_cuniculus_Rcdo.jpg",
    "https://upload.wikimedia.org/wikipedia/commons/9/98/Horse-and-pony.jpg",
37
38
39
]


40
class ModelRequestData(NamedTuple):
41
    engine_args: EngineArgs
42
    prompt: str
43
    image_data: list[Image]
44
45
46
    stop_token_ids: Optional[list[int]] = None
    chat_template: Optional[str] = None
    lora_requests: Optional[list[LoRARequest]] = None
47
48


49
50
51
52
53
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
# lower-end GPUs.
# Unless specified, these settings have been tested to work on a single L4.


54
def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:
55
    model_name = "rhymes-ai/Aria"
56
57
58
59
60
61
62
    engine_args = EngineArgs(
        model=model_name,
        tokenizer_mode="slow",
        trust_remote_code=True,
        dtype="bfloat16",
        limit_mm_per_prompt={"image": len(image_urls)},
    )
63
    placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
64
65
66
    prompt = (
        f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n<|im_start|>assistant\n"
    )
67
    stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
68

69
    return ModelRequestData(
70
        engine_args=engine_args,
71
72
73
        prompt=prompt,
        stop_token_ids=stop_token_ids,
        image_data=[fetch_image(url) for url in image_urls],
74
    )
75

76

Jennifer Zhao's avatar
Jennifer Zhao committed
77
78
79
80
81
82
83
84
85
86
def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "CohereForAI/aya-vision-8b"

    engine_args = EngineArgs(
        model=model_name,
        max_num_seqs=2,
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
87
88
89
90
91
92
93
94
95
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]
Jennifer Zhao's avatar
Jennifer Zhao committed
96
97
98

    processor = AutoProcessor.from_pretrained(model_name)

99
100
101
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
Jennifer Zhao's avatar
Jennifer Zhao committed
102
103
104
105
106
107
108
109

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
def load_command_a_vision(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "CohereLabs/command-a-vision-07-2025"

    # NOTE: This model is 122B parameters and requires tensor parallelism
    # Recommended to use tp=4 on H100 GPUs
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=32768,
        tensor_parallel_size=4,
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]

    processor = AutoProcessor.from_pretrained(model_name)

    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


146
def load_deepseek_vl2(question: str, image_urls: list[str]) -> ModelRequestData:
147
    model_name = "deepseek-ai/deepseek-vl2-tiny"
148

149
150
151
152
153
154
155
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=2,
        hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
        limit_mm_per_prompt={"image": len(image_urls)},
    )
156

157
158
159
    placeholder = "".join(
        f"image_{i}:<image>\n" for i, _ in enumerate(image_urls, start=1)
    )
160
161
162
    prompt = f"<|User|>: {placeholder}{question}\n\n<|Assistant|>:"

    return ModelRequestData(
163
        engine_args=engine_args,
164
165
166
167
168
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


169
def load_gemma3(question: str, image_urls: list[str]) -> ModelRequestData:
170
171
    model_name = "google/gemma-3-4b-it"

172
    engine_args = EngineArgs(
173
174
175
176
177
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        limit_mm_per_prompt={"image": len(image_urls)},
    )
178
179

    placeholders = [{"type": "image", "image": url} for url in image_urls]
180
181
182
183
184
185
186
187
188
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]
189
190
191

    processor = AutoProcessor.from_pretrained(model_name)

192
193
194
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
195
196

    return ModelRequestData(
197
        engine_args=engine_args,
198
199
200
201
202
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


203
def load_h2ovl(question: str, image_urls: list[str]) -> ModelRequestData:
204
    model_name = "h2oai/h2ovl-mississippi-800m"
205

206
    engine_args = EngineArgs(
207
208
209
210
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
        limit_mm_per_prompt={"image": len(image_urls)},
211
        mm_processor_kwargs={"max_dynamic_patch": 4},
212
213
    )

214
215
216
217
    placeholders = "\n".join(
        f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
    )
    messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
218

219
220
221
222
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    prompt = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
223
224

    # Stop tokens for H2OVL-Mississippi
225
    # https://huggingface.co/h2oai/h2ovl-mississippi-800m
226
227
228
    stop_token_ids = [tokenizer.eos_token_id]

    return ModelRequestData(
229
        engine_args=engine_args,
230
231
232
233
234
235
        prompt=prompt,
        stop_token_ids=stop_token_ids,
        image_data=[fetch_image(url) for url in image_urls],
    )


236
237
238
239
240
def load_hyperclovax_seed_vision(
    question: str, image_urls: list[str]
) -> ModelRequestData:
    model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
241

242
    engine_args = EngineArgs(
243
        model=model_name,
244
245
        trust_remote_code=True,
        max_model_len=16384,
246
247
248
        limit_mm_per_prompt={"image": len(image_urls)},
    )

249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
    message = {"role": "user", "content": list()}
    for _image_url in image_urls:
        message["content"].append(
            {
                "type": "image",
                "image": _image_url,
                "ocr": "",
                "lens_keywords": "",
                "lens_local_keywords": "",
            }
        )
    message["content"].append(
        {
            "type": "text",
            "text": question,
        }
265
    )
266
267
268
269
270
271
272
273
274

    prompt = tokenizer.apply_chat_template(
        [
            message,
        ],
        tokenize=False,
        add_generation_prompt=True,
    )

275
    return ModelRequestData(
276
        engine_args=engine_args,
277
        prompt=prompt,
278
        stop_token_ids=None,
279
280
281
282
        image_data=[fetch_image(url) for url in image_urls],
    )


283
284
def load_idefics3(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
285
286
287
288
289
290
291
292

    # The configuration below has been confirmed to launch on a single L40 GPU.
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=16,
        enforce_eager=True,
        limit_mm_per_prompt={"image": len(image_urls)},
293
294
        # if you are running out of memory, you can reduce the "longest_edge".
        # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
295
        mm_processor_kwargs={
296
            "size": {"longest_edge": 2 * 364},
297
298
299
        },
    )

300
301
302
    placeholders = "\n".join(
        f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
    )
303
    prompt = f"<|begin_of_text|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:"  # noqa: E501
304
305
306
307
308
309
310
    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


Lyu Han's avatar
Lyu Han committed
311
def load_interns1(question: str, image_urls: list[str]) -> ModelRequestData:
312
    model_name = "internlm/Intern-S1-mini"
Lyu Han's avatar
Lyu Han committed
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = "\n".join(
        f"Image-{i}: <IMG_CONTEXT>\n" for i, _ in enumerate(image_urls, start=1)
    )
    messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]

    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    prompt = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


338
def load_internvl(question: str, image_urls: list[str]) -> ModelRequestData:
339
340
    model_name = "OpenGVLab/InternVL2-2B"

341
    engine_args = EngineArgs(
342
343
344
345
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        limit_mm_per_prompt={"image": len(image_urls)},
346
        mm_processor_kwargs={"max_dynamic_patch": 4},
347
348
    )

349
350
351
352
    placeholders = "\n".join(
        f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
    )
    messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
353

354
355
356
357
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    prompt = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
358
359
360
361

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
362
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
363
364
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
365

366
    return ModelRequestData(
367
        engine_args=engine_args,
368
369
370
371
        prompt=prompt,
        stop_token_ids=stop_token_ids,
        image_data=[fetch_image(url) for url in image_urls],
    )
372
373


374
375
def load_keye_vl(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "Kwai-Keye/Keye-VL-8B-Preview"
376

377
378
    engine_args = EngineArgs(
        model=model_name,
379
380
381
        trust_remote_code=True,
        max_model_len=8192,
        max_num_seqs=5,
382
383
384
385
386
387
388
389
390
391
392
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
393
        },
394
395
    ]

396
    processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
397
398
399
400
401

    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

402
403
    image_data = [fetch_image(url) for url in image_urls]

404
405
406
    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
407
        image_data=image_data,
408
409
410
    )


411
412
413
def load_keye_vl1_5(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "Kwai-Keye/Keye-VL-1_5-8B"

414
415
    engine_args = EngineArgs(
        model=model_name,
416
417
418
        trust_remote_code=True,
        max_model_len=32768,
        max_num_seqs=5,
419
420
421
422
423
424
425
426
427
428
429
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
430
        },
431
432
    ]

433
    processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
434
435
436
437
438

    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

439
440
    image_data = [fetch_image(url) for url in image_urls]

441
442
443
    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
444
        image_data=image_data,
445
446
447
    )


448
449
450
def load_kimi_vl(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "moonshotai/Kimi-VL-A3B-Instruct"

451
452
    engine_args = EngineArgs(
        model=model_name,
453
454
455
        trust_remote_code=True,
        max_model_len=4096,
        max_num_seqs=4,
456
457
458
459
460
461
462
463
464
465
466
467
468
469
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]

470
    processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
471
472
473
474
475
476
477
478
479
480
481
482

    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


483
484
485
def load_llama4(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"

486
487
    engine_args = EngineArgs(
        model=model_name,
488
489
        max_model_len=131072,
        tensor_parallel_size=8,
490
491
492
493
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
494
495
496
497
498
499
500
501
502
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]
503
504
505

    processor = AutoProcessor.from_pretrained(model_name)

506
507
508
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
509
510
511
512
513
514
515
516

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


517
518
519
520
def load_llava(question: str, image_urls: list[str]) -> ModelRequestData:
    # NOTE: CAUTION! Original Llava models wasn't really trained on multi-image inputs,
    # it will generate poor response for multi-image inputs!
    model_name = "llava-hf/llava-1.5-7b-hf"
521
522
    engine_args = EngineArgs(
        model=model_name,
523
        max_num_seqs=16,
524
525
526
527
528
529
530
531
532
533
534
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
535
        }
536
537
    ]

538
    processor = AutoProcessor.from_pretrained(model_name)
539
540
541
542
543
544
545
546

    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
547
        image_data=[fetch_image(url) for url in image_urls],
548
549
550
    )


551
552
def load_llava_next(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "llava-hf/llava-v1.6-mistral-7b-hf"
553
554
    engine_args = EngineArgs(
        model=model_name,
555
556
        max_model_len=8192,
        max_num_seqs=16,
557
558
559
560
561
562
563
564
565
566
567
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
568
        }
569
570
    ]

571
    processor = AutoProcessor.from_pretrained(model_name)
572
573
574
575
576
577
578
579

    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
580
        image_data=[fetch_image(url) for url in image_urls],
581
582
583
    )


584
585
def load_llava_onevision(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "llava-hf/llava-onevision-qwen2-7b-ov-hf"
586
587
    engine_args = EngineArgs(
        model=model_name,
588
589
        max_model_len=16384,
        max_num_seqs=16,
590
591
592
593
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
594
595
596
597
598
599
600
601
602
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]
603

604
    processor = AutoProcessor.from_pretrained(model_name)
605

606
607
608
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
609
610
611
612
613
614
615
616

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


617
618
619
620
621
622
623
624
625
626
def load_mistral3(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"

    # Adjust this as necessary to fit in GPU
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        tensor_parallel_size=2,
        limit_mm_per_prompt={"image": len(image_urls)},
627
        ignore_patterns=["consolidated.safetensors"],
628
629
630
631
632
633
634
635
636
637
638
639
    )

    placeholders = "[IMG]" * len(image_urls)
    prompt = f"<s>[INST]{question}\n{placeholders}[/INST]"

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


640
def load_nvlm_d(question: str, image_urls: list[str]) -> ModelRequestData:
641
642
643
    model_name = "nvidia/NVLM-D-72B"

    # Adjust this as necessary to fit in GPU
644
    engine_args = EngineArgs(
645
646
647
648
649
650
651
652
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
        tensor_parallel_size=4,
        limit_mm_per_prompt={"image": len(image_urls)},
        mm_processor_kwargs={"max_dynamic_patch": 4},
    )

653
654
655
656
    placeholders = "\n".join(
        f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
    )
    messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
657

658
659
660
661
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    prompt = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
662
663

    return ModelRequestData(
664
        engine_args=engine_args,
665
666
667
668
669
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


670
671
# Ovis
def load_ovis(question: str, image_urls: list[str]) -> ModelRequestData:
672
673
674
675
676
677
678
679
680
681
682
    model_name = "AIDC-AI/Ovis2-1B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        trust_remote_code=True,
        dtype="half",
        limit_mm_per_prompt={"image": len(image_urls)},
    )

683
684
685
686
    placeholders = "\n".join(
        f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
    )
    messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
687

688
689
690
691
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
    prompt = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
692
693
694
695
696
697
698
699

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
# ovis2_5
def load_ovis2_5(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "AIDC-AI/Ovis2.5-2B"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        trust_remote_code=True,
        dtype="half",
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = "\n".join(
        f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
    )
716
717
718
    prompt = (
        f"<|im_start|>user\n\n{placeholders}\n{question}<|im_end|>\n"
        "<|im_start|>assistant\n"
719
720
721
722
723
724
725
726
727
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


728
def load_pixtral_hf(question: str, image_urls: list[str]) -> ModelRequestData:
729
730
731
    model_name = "mistral-community/pixtral-12b"

    # Adjust this as necessary to fit in GPU
732
    engine_args = EngineArgs(
733
734
735
736
737
738
739
740
741
742
743
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        tensor_parallel_size=2,
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = "[IMG]" * len(image_urls)
    prompt = f"<s>[INST]{question}\n{placeholders}[/INST]"

    return ModelRequestData(
744
        engine_args=engine_args,
745
746
747
748
749
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


750
def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData:
751
752
753
754
755
756
757
758
759
760
761
762
    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
763
    engine_args = EngineArgs(
764
765
766
767
768
769
770
        model="microsoft/Phi-3.5-vision-instruct",
        trust_remote_code=True,
        max_model_len=4096,
        max_num_seqs=2,
        limit_mm_per_prompt={"image": len(image_urls)},
        mm_processor_kwargs={"num_crops": 4},
    )
771
772
773
    placeholders = "\n".join(
        f"<|image_{i}|>" for i, _ in enumerate(image_urls, start=1)
    )
774
775
776
    prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"

    return ModelRequestData(
777
        engine_args=engine_args,
778
779
780
781
782
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


783
784
785
786
787
788
789
790
791
792
def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
    """
    Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
    show how to process multi images inputs.
    """

    model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
    # Since the vision-lora and speech-lora co-exist with the base model,
    # we have to manually specify the path of the lora weights.
    vision_lora_path = os.path.join(model_path, "vision-lora")
793
    engine_args = EngineArgs(
794
795
        model=model_path,
        trust_remote_code=True,
796
        max_model_len=4096,
797
798
799
800
        max_num_seqs=2,
        limit_mm_per_prompt={"image": len(image_urls)},
        enable_lora=True,
        max_lora_rank=320,
801
802
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
        mm_processor_kwargs={"dynamic_hd": 4},
803
804
    )

805
    placeholders = "".join(f"<|image_{i}|>" for i, _ in enumerate(image_urls, start=1))
806
807
808
    prompt = f"<|user|>{placeholders}{question}<|end|><|assistant|>"

    return ModelRequestData(
809
        engine_args=engine_args,
810
811
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
812
        lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
813
814
815
    )


816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
def load_phi4_multimodal(question: str, image_urls: list[str]) -> ModelRequestData:
    """
    Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
    show how to process multi images inputs.
    """

    model_path = snapshot_download(
        "microsoft/Phi-4-multimodal-instruct", revision="refs/pr/70"
    )
    # Since the vision-lora and speech-lora co-exist with the base model,
    # we have to manually specify the path of the lora weights.
    vision_lora_path = os.path.join(model_path, "vision-lora")
    engine_args = EngineArgs(
        model=model_path,
        max_model_len=4096,
        max_num_seqs=2,
        limit_mm_per_prompt={"image": len(image_urls)},
        enable_lora=True,
        max_lora_rank=320,
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
        mm_processor_kwargs={"dynamic_hd": 4},
    )

    placeholders = "<|image|>" * len(image_urls)
    prompt = f"<|user|>{placeholders}{question}<|end|><|assistant|>"

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
        lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
    )


850
def load_qwen_vl_chat(question: str, image_urls: list[str]) -> ModelRequestData:
851
    model_name = "Qwen/Qwen-VL-Chat"
852
    engine_args = EngineArgs(
853
854
855
856
        model=model_name,
        trust_remote_code=True,
        max_model_len=1024,
        max_num_seqs=2,
857
        hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},
858
859
        limit_mm_per_prompt={"image": len(image_urls)},
    )
860
861
862
    placeholders = "".join(
        f"Picture {i}: <img></img>\n" for i, _ in enumerate(image_urls, start=1)
    )
863
864
865
866
867

    # This model does not have a chat_template attribute on its tokenizer,
    # so we need to explicitly pass it. We use ChatML since it's used in the
    # generation utils of the model:
    # https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/qwen_generation_utils.py#L265
868
    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
869
870
871
872

    # Copied from: https://huggingface.co/docs/transformers/main/en/chat_templating
    chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"  # noqa: E501

873
874
875
876
877
878
879
    messages = [{"role": "user", "content": f"{placeholders}\n{question}"}]
    prompt = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True,
        chat_template=chat_template,
    )
880
881
882

    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
883

884
    return ModelRequestData(
885
        engine_args=engine_args,
886
887
888
889
890
891
892
        prompt=prompt,
        stop_token_ids=stop_token_ids,
        image_data=[fetch_image(url) for url in image_urls],
        chat_template=chat_template,
    )


893
def load_qwen2_vl(question: str, image_urls: list[str]) -> ModelRequestData:
894
    try:
汪志鹏's avatar
汪志鹏 committed
895
        from qwen_vl_utils import smart_resize
896
    except ModuleNotFoundError:
897
898
899
900
901
        print(
            "WARNING: `qwen-vl-utils` not installed, input images will not "
            "be automatically resized. You can enable this functionality by "
            "`pip install qwen-vl-utils`."
        )
汪志鹏's avatar
汪志鹏 committed
902
        smart_resize = None
903
904
905

    model_name = "Qwen/Qwen2-VL-7B-Instruct"

906
    # Tested on L40
907
    engine_args = EngineArgs(
908
        model=model_name,
汪志鹏's avatar
汪志鹏 committed
909
        max_model_len=32768 if smart_resize is None else 4096,
910
        max_num_seqs=5,
911
912
913
914
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
915
916
917
918
919
920
921
922
923
924
    messages = [
        {"role": "system", "content": "You are a helpful assistant."},
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        },
    ]
925
926
927

    processor = AutoProcessor.from_pretrained(model_name)

928
929
930
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
931

汪志鹏's avatar
汪志鹏 committed
932
    if smart_resize is None:
933
934
        image_data = [fetch_image(url) for url in image_urls]
    else:
汪志鹏's avatar
汪志鹏 committed
935
936
937
938
939
940
941
942
943

        def post_process_image(image: Image) -> Image:
            width, height = image.size
            resized_height, resized_width = smart_resize(
                height, width, max_pixels=1024 * 28 * 28
            )
            return image.resize((resized_width, resized_height))

        image_data = [post_process_image(fetch_image(url)) for url in image_urls]
944

945
    return ModelRequestData(
946
        engine_args=engine_args,
947
948
949
        prompt=prompt,
        image_data=image_data,
    )
950
951


952
def load_qwen2_5_vl(question: str, image_urls: list[str]) -> ModelRequestData:
Roger Wang's avatar
Roger Wang committed
953
    try:
汪志鹏's avatar
汪志鹏 committed
954
        from qwen_vl_utils import smart_resize
Roger Wang's avatar
Roger Wang committed
955
    except ModuleNotFoundError:
956
957
958
959
960
        print(
            "WARNING: `qwen-vl-utils` not installed, input images will not "
            "be automatically resized. You can enable this functionality by "
            "`pip install qwen-vl-utils`."
        )
汪志鹏's avatar
汪志鹏 committed
961
        smart_resize = None
Roger Wang's avatar
Roger Wang committed
962
963
964

    model_name = "Qwen/Qwen2.5-VL-3B-Instruct"

965
    engine_args = EngineArgs(
Roger Wang's avatar
Roger Wang committed
966
        model=model_name,
汪志鹏's avatar
汪志鹏 committed
967
        max_model_len=32768 if smart_resize is None else 4096,
Roger Wang's avatar
Roger Wang committed
968
969
970
971
972
        max_num_seqs=5,
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
973
974
975
976
977
978
979
980
981
982
    messages = [
        {"role": "system", "content": "You are a helpful assistant."},
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        },
    ]
Roger Wang's avatar
Roger Wang committed
983
984
985

    processor = AutoProcessor.from_pretrained(model_name)

986
987
988
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
Roger Wang's avatar
Roger Wang committed
989

汪志鹏's avatar
汪志鹏 committed
990
    if smart_resize is None:
Roger Wang's avatar
Roger Wang committed
991
992
        image_data = [fetch_image(url) for url in image_urls]
    else:
汪志鹏's avatar
汪志鹏 committed
993
994
995
996
997
998
999
1000
1001

        def post_process_image(image: Image) -> Image:
            width, height = image.size
            resized_height, resized_width = smart_resize(
                height, width, max_pixels=1024 * 28 * 28
            )
            return image.resize((resized_width, resized_height))

        image_data = [post_process_image(fetch_image(url)) for url in image_urls]
Roger Wang's avatar
Roger Wang committed
1002
1003

    return ModelRequestData(
1004
        engine_args=engine_args,
Roger Wang's avatar
Roger Wang committed
1005
1006
1007
1008
1009
        prompt=prompt,
        image_data=image_data,
    )


1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
def load_r_vl(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "YannQi/R-4B"
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=16384,
        max_num_seqs=16,
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]

    processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)

    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
def load_smolvlm(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"

    # The configuration below has been confirmed to launch on a single L40 GPU.
    engine_args = EngineArgs(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=16,
        enforce_eager=True,
        limit_mm_per_prompt={"image": len(image_urls)},
        mm_processor_kwargs={
            "max_image_size": {"longest_edge": 384},
        },
    )

    placeholders = "\n".join(
        f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)
    )
    prompt = (
        f"<|im_start|>User:{placeholders}\n{question}<end_of_utterance>\nAssistant:"  # noqa: E501
    )
    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=[fetch_image(url) for url in image_urls],
    )


def load_step3(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "stepfun-ai/step3-fp8"

    # NOTE: Below are verified configurations for step3-fp8
    # on 8xH100 GPUs.
    engine_args = EngineArgs(
        model=model_name,
        max_num_batched_tokens=4096,
        gpu_memory_utilization=0.85,
        tensor_parallel_size=8,
        limit_mm_per_prompt={"image": len(image_urls)},
        reasoning_parser="step3",
    )

    prompt = (
        "<|begin▁of▁sentence|> You are a helpful assistant. <|BOT|>user\n "
        f"{'<im_patch>' * len(image_urls)}{question} <|EOT|><|BOT|"
        ">assistant\n<think>\n"
    )
    image_data = [fetch_image(url) for url in image_urls]

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=image_data,
    )


汪志鹏's avatar
汪志鹏 committed
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
def load_tarsier(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "omni-research/Tarsier-7b"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        limit_mm_per_prompt={"image": len(image_urls)},
    )

    prompt = f"USER: {'<image>' * len(image_urls)}\n{question}\n ASSISTANT:"
    image_data = [fetch_image(url) for url in image_urls]

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=image_data,
    )


1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "omni-research/Tarsier2-Recap-7b"

    engine_args = EngineArgs(
        model=model_name,
        trust_remote_code=True,
        max_model_len=32768,
        limit_mm_per_prompt={"image": len(image_urls)},
        hf_overrides={"architectures": ["Tarsier2ForConditionalGeneration"]},
    )

    prompt = (
        "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
        f"<|im_start|>user\n<|vision_start|>{'<|image_pad|>' * len(image_urls)}"
        f"<|vision_end|>{question}<|im_end|>\n"
        "<|im_start|>assistant\n"
    )
    image_data = [fetch_image(url) for url in image_urls]

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=image_data,
    )


1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
# GLM-4.5V
def load_glm4_5v(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "zai-org/GLM-4.5V"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=32768,
        max_num_seqs=2,
        limit_mm_per_prompt={"image": len(image_urls)},
        enforce_eager=True,
        tensor_parallel_size=4,
    )
    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]
    processor = AutoProcessor.from_pretrained(model_name)
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
    image_data = [fetch_image(url) for url in image_urls]

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=image_data,
    )


# GLM-4.5V-FP8
def load_glm4_5v_fp8(question: str, image_urls: list[str]) -> ModelRequestData:
    model_name = "zai-org/GLM-4.5V-FP8"

    engine_args = EngineArgs(
        model=model_name,
        max_model_len=32768,
        max_num_seqs=2,
        limit_mm_per_prompt={"image": len(image_urls)},
        enforce_eager=True,
        tensor_parallel_size=4,
    )
    placeholders = [{"type": "image", "image": url} for url in image_urls]
    messages = [
        {
            "role": "user",
            "content": [
                *placeholders,
                {"type": "text", "text": question},
            ],
        }
    ]
    processor = AutoProcessor.from_pretrained(model_name)
    prompt = processor.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
    image_data = [fetch_image(url) for url in image_urls]

    return ModelRequestData(
        engine_args=engine_args,
        prompt=prompt,
        image_data=image_data,
    )


1215
model_example_map = {
1216
    "aria": load_aria,
Jennifer Zhao's avatar
Jennifer Zhao committed
1217
    "aya_vision": load_aya_vision,
1218
    "command_a_vision": load_command_a_vision,
1219
    "deepseek_vl_v2": load_deepseek_vl2,
1220
    "gemma3": load_gemma3,
1221
    "h2ovl_chat": load_h2ovl,
1222
    "hyperclovax_seed_vision": load_hyperclovax_seed_vision,
1223
    "idefics3": load_idefics3,
Lyu Han's avatar
Lyu Han committed
1224
    "interns1": load_interns1,
1225
    "internvl_chat": load_internvl,
1226
    "keye_vl": load_keye_vl,
1227
    "keye_vl1_5": load_keye_vl1_5,
1228
    "kimi_vl": load_kimi_vl,
1229
    "llama4": load_llama4,
1230
1231
1232
    "llava": load_llava,
    "llava-next": load_llava_next,
    "llava-onevision": load_llava_onevision,
1233
    "mistral3": load_mistral3,
1234
    "NVLM_D": load_nvlm_d,
1235
    "ovis": load_ovis,
1236
    "ovis2_5": load_ovis2_5,
1237
    "phi3_v": load_phi3v,
1238
    "phi4_mm": load_phi4mm,
1239
    "phi4_multimodal": load_phi4_multimodal,
1240
1241
    "pixtral_hf": load_pixtral_hf,
    "qwen_vl_chat": load_qwen_vl_chat,
1242
    "qwen2_vl": load_qwen2_vl,
Roger Wang's avatar
Roger Wang committed
1243
    "qwen2_5_vl": load_qwen2_5_vl,
1244
    "rvl": load_r_vl,
1245
    "smolvlm": load_smolvlm,
1246
    "step3": load_step3,
汪志鹏's avatar
汪志鹏 committed
1247
    "tarsier": load_tarsier,
1248
    "tarsier2": load_tarsier2,
1249
1250
    "glm4_5v": load_glm4_5v,
    "glm4_5v_fp8": load_glm4_5v_fp8,
1251
1252
1253
}


1254
def run_generate(model, question: str, image_urls: list[str], seed: Optional[int]):
1255
    req_data = model_example_map[model](question, image_urls)
1256

1257
1258
1259
    engine_args = asdict(req_data.engine_args) | {"seed": args.seed}
    llm = LLM(**engine_args)

1260
1261
1262
    sampling_params = SamplingParams(
        temperature=0.0, max_tokens=256, stop_token_ids=req_data.stop_token_ids
    )
1263

1264
    outputs = llm.generate(
1265
        {
1266
            "prompt": req_data.prompt,
1267
            "multi_modal_data": {"image": req_data.image_data},
1268
        },
1269
1270
1271
        sampling_params=sampling_params,
        lora_request=req_data.lora_requests,
    )
1272

1273
    print("-" * 50)
1274
1275
1276
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)
1277
        print("-" * 50)
1278
1279


1280
def run_chat(model: str, question: str, image_urls: list[str], seed: Optional[int]):
1281
    req_data = model_example_map[model](question, image_urls)
1282

1283
1284
1285
    # Disable other modalities to save memory
    default_limits = {"image": 0, "video": 0, "audio": 0}
    req_data.engine_args.limit_mm_per_prompt = default_limits | dict(
1286
1287
        req_data.engine_args.limit_mm_per_prompt or {}
    )
1288

1289
1290
1291
    engine_args = asdict(req_data.engine_args) | {"seed": seed}
    llm = LLM(**engine_args)

1292
1293
1294
    sampling_params = SamplingParams(
        temperature=0.0, max_tokens=256, stop_token_ids=req_data.stop_token_ids
    )
1295
    outputs = llm.chat(
1296
1297
1298
1299
1300
1301
1302
        [
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": question,
1303
                    },
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
                    *(
                        {
                            "type": "image_url",
                            "image_url": {"url": image_url},
                        }
                        for image_url in image_urls
                    ),
                ],
            }
        ],
1314
        sampling_params=sampling_params,
1315
        chat_template=req_data.chat_template,
1316
        lora_request=req_data.lora_requests,
1317
    )
1318

1319
    print("-" * 50)
1320
1321
1322
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)
1323
        print("-" * 50)
1324
1325


1326
def parse_args():
1327
    parser = FlexibleArgumentParser(
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
        description="Demo on using vLLM for offline inference with "
        "vision language models that support multi-image input for text "
        "generation"
    )
    parser.add_argument(
        "--model-type",
        "-m",
        type=str,
        default="phi3_v",
        choices=model_example_map.keys(),
        help='Huggingface "model_type".',
    )
    parser.add_argument(
        "--method",
        type=str,
        default="generate",
        choices=["generate", "chat"],
        help="The method to run in `vllm.LLM`.",
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=None,
        help="Set the seed when initializing `vllm.LLM`.",
    )
1353
1354
1355
    parser.add_argument(
        "--num-images",
        "-n",
1356
        type=int,
1357
        choices=list(range(1, len(IMAGE_URLS) + 1)),  # the max number of images
1358
        default=2,
1359
1360
        help="Number of images to use for the demo.",
    )
1361
1362
    return parser.parse_args()

1363

1364
1365
1366
1367
1368
def main(args: Namespace):
    model = args.model_type
    method = args.method
    seed = args.seed

1369
    image_urls = IMAGE_URLS[: args.num_images]
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380

    if method == "generate":
        run_generate(model, QUESTION, image_urls, seed)
    elif method == "chat":
        run_chat(model, QUESTION, image_urls, seed)
    else:
        raise ValueError(f"Invalid method: {method}")


if __name__ == "__main__":
    args = parse_args()
1381
    main(args)