vision_language.py 23.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
"""
Cyrus Leung's avatar
Cyrus Leung committed
3
4
This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.
5
6
7
8

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
9
10
import random

11
12
13
14
from transformers import AutoTokenizer

from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
15
from vllm.assets.video import VideoAsset
16
17
from vllm.utils import FlexibleArgumentParser

18
19
20
21
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
# lower-end GPUs.
# Unless specified, these settings have been tested to work on a single L4.

22

23
24
25
26
27
# Aria
def run_aria(question: str, modality: str):
    assert modality == "image"
    model_name = "rhymes-ai/Aria"

28
    # NOTE: Need L40 (or equivalent) to avoid OOM
29
    llm = LLM(model=model_name,
30
31
              max_model_len=4096,
              max_num_seqs=2,
32
              dtype="bfloat16",
33
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
34

35
    prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
36
37
38
39
40
41
42
43
44
45
46
47
48
49
              "<|im_end|>\n<|im_start|>assistant\n")

    stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
    return llm, prompt, stop_token_ids


# BLIP-2
def run_blip2(question: str, modality: str):
    assert modality == "image"

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
    prompt = f"Question: {question} Answer:"
    llm = LLM(model="Salesforce/blip2-opt-2.7b",
50
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
51
52
53
54
55
56
57
58
59
60
61
    stop_token_ids = None
    return llm, prompt, stop_token_ids


# Chameleon
def run_chameleon(question: str, modality: str):
    assert modality == "image"

    prompt = f"{question}<image>"
    llm = LLM(model="facebook/chameleon-7b",
              max_model_len=4096,
62
              max_num_seqs=2,
63
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
64
65
66
67
    stop_token_ids = None
    return llm, prompt, stop_token_ids


68
69
70
71
# Deepseek-VL2
def run_deepseek_vl2(question: str, modality: str):
    assert modality == "image"

72
    model_name = "deepseek-ai/deepseek-vl2-tiny"
73
74
75
76
77
78
79
80
81
82
83
84

    llm = LLM(model=model_name,
              max_model_len=4096,
              max_num_seqs=2,
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
              hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]})

    prompt = f"<|User|>: <image>\n{question}\n\n<|Assistant|>:"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


85
86
87
88
89
90
91
92
# Fuyu
def run_fuyu(question: str, modality: str):
    assert modality == "image"

    prompt = f"{question}\n"
    llm = LLM(model="adept/fuyu-8b",
              max_model_len=2048,
              max_num_seqs=2,
93
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    stop_token_ids = None
    return llm, prompt, stop_token_ids


# GLM-4v
def run_glm4v(question: str, modality: str):
    assert modality == "image"
    model_name = "THUDM/glm-4v-9b"

    llm = LLM(model=model_name,
              max_model_len=2048,
              max_num_seqs=2,
              trust_remote_code=True,
              enforce_eager=True,
108
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
109
110
111
    prompt = f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
        {question}<|assistant|>"

112
113
114
115
116
117
118
119
120
121
122
123
124
125
    stop_token_ids = [151329, 151336, 151338]
    return llm, prompt, stop_token_ids


# H2OVL-Mississippi
def run_h2ovl(question: str, modality: str):
    assert modality == "image"

    model_name = "h2oai/h2ovl-mississippi-2b"

    llm = LLM(
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
126
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)

    # Stop tokens for H2OVL-Mississippi
    # https://huggingface.co/h2oai/h2ovl-mississippi-2b
    stop_token_ids = [tokenizer.eos_token_id]
    return llm, prompt, stop_token_ids


# Idefics3-8B-Llama3
def run_idefics3(question: str, modality: str):
    assert modality == "image"
    model_name = "HuggingFaceM4/Idefics3-8B-Llama3"

    llm = LLM(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        # if you are running out of memory, you can reduce the "longest_edge".
        # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
        mm_processor_kwargs={
            "size": {
                "longest_edge": 3 * 364
            },
        },
159
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
    )
    prompt = (
        f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
    )
    stop_token_ids = None
    return llm, prompt, stop_token_ids


# InternVL
def run_internvl(question: str, modality: str):
    assert modality == "image"

    model_name = "OpenGVLab/InternVL2-2B"

    llm = LLM(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
178
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
    return llm, prompt, stop_token_ids


197
# LLaVA-1.5
198
def run_llava(question: str, modality: str):
199
    assert modality == "image"
200
201
202

    prompt = f"USER: <image>\n{question}\nASSISTANT:"

203
204
    llm = LLM(model="llava-hf/llava-1.5-7b-hf",
              max_model_len=4096,
205
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
206
207
    stop_token_ids = None
    return llm, prompt, stop_token_ids
208
209
210


# LLaVA-1.6/LLaVA-NeXT
211
def run_llava_next(question: str, modality: str):
212
    assert modality == "image"
213
214

    prompt = f"[INST] <image>\n{question} [/INST]"
215
216
    llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
              max_model_len=8192,
217
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
218
219
220
221
222
223
    stop_token_ids = None
    return llm, prompt, stop_token_ids


# LlaVA-NeXT-Video
# Currently only support for video input
224
def run_llava_next_video(question: str, modality: str):
225
226
    assert modality == "video"

227
    prompt = f"USER: <video>\n{question} ASSISTANT:"
228
229
    llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
              max_model_len=8192,
230
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
231
232
    stop_token_ids = None
    return llm, prompt, stop_token_ids
233
234


235
# LLaVA-OneVision
236
def run_llava_onevision(question: str, modality: str):
237
238
239
240
241
242
243
244
245
246

    if modality == "video":
        prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    elif modality == "image":
        prompt = f"<|im_start|>user <image>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
247
              max_model_len=16384,
248
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
249
250
251
252
    stop_token_ids = None
    return llm, prompt, stop_token_ids


253
254
# Mantis
def run_mantis(question: str, modality: str):
255
    assert modality == "image"
256

257
258
    llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n'  # noqa: E501
    prompt = llama3_template.format(f"{question}\n<image>")
259
260

    llm = LLM(
261
        model="TIGER-Lab/Mantis-8B-siglip-llama3",
262
        max_model_len=4096,
263
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
264
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
265
    )
266
    stop_token_ids = [128009]
267
    return llm, prompt, stop_token_ids
268
269
270


# MiniCPM-V
271
272
273
def run_minicpmv_base(question: str, modality: str, model_name):
    assert modality in ["image", "video"]
    # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
274
275
276
277
278
279
280

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
281
282
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

283
    # 2.6
284
285
286
287
288
289
290
291
292
    # model_name = "openbmb/MiniCPM-V-2_6"
    # o2.6

    # modality supports
    # 2.0: image
    # 2.5: image
    # 2.6: image, video
    # o2.6: image, video, audio
    # model_name = "openbmb/MiniCPM-o-2_6"
293
294
295
296
    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    llm = LLM(
        model=model_name,
297
298
        max_model_len=4096,
        max_num_seqs=2,
299
        trust_remote_code=True,
300
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
301
    )
302
303
304
305
306
307
308
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

309
    # 2.6 / o2.6
310
311
    stop_tokens = ['<|im_end|>', '<|endoftext|>']
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
312

313
314
315
316
317
    modality_placeholder = {
        "image": "(<image>./</image>)",
        "video": "(<video>./</video>)",
    }

318
319
    messages = [{
        'role': 'user',
320
        'content': f'{modality_placeholder[modality]}\n{question}'
321
322
323
324
    }]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)
325
    return llm, prompt, stop_token_ids
326
327


328
329
330
331
332
333
334
335
def run_minicpmo(question: str, modality: str):
    return run_minicpmv_base(question, modality, "openbmb/MiniCPM-o-2_6")


def run_minicpmv(question: str, modality: str):
    return run_minicpmv_base(question, modality, "openbmb/MiniCPM-V-2_6")


336
337
# LLama 3.2
def run_mllama(question: str, modality: str):
338
339
    assert modality == "image"

340
    model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
341

342
343
344
345
346
    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (131072) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

    # The configuration below has been confirmed to launch on a single L40 GPU.
347
348
    llm = LLM(
        model=model_name,
349
350
        max_model_len=4096,
        max_num_seqs=16,
351
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
352
353
    )

354
355
356
357
358
359
360
361
362
363
364
365
366
367
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    messages = [{
        "role":
        "user",
        "content": [{
            "type": "image"
        }, {
            "type": "text",
            "text": f"{question}"
        }]
    }]
    prompt = tokenizer.apply_chat_template(messages,
                                           add_generation_prompt=True,
                                           tokenize=False)
368
    stop_token_ids = None
369
370
371
    return llm, prompt, stop_token_ids


372
373
# Molmo
def run_molmo(question, modality):
374
375
    assert modality == "image"

376
    model_name = "allenai/Molmo-7B-D-0924"
377

378
    llm = LLM(
379
        model=model_name,
380
        trust_remote_code=True,
381
        dtype="bfloat16",
382
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
383
    )
384

385
386
    prompt = question
    stop_token_ids = None
387
    return llm, prompt, stop_token_ids
388
389


390
391
392
393
394
395
396
397
398
399
400
401
# NVLM-D
def run_nvlm_d(question: str, modality: str):
    assert modality == "image"

    model_name = "nvidia/NVLM-D-72B"

    # Adjust this as necessary to fit in GPU
    llm = LLM(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        tensor_parallel_size=4,
402
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
403
404
405
406
407
408
409
410
411
412
413
414
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)
    stop_token_ids = None
    return llm, prompt, stop_token_ids


415
416
# PaliGemma
def run_paligemma(question: str, modality: str):
417
    assert modality == "image"
418

419
420
421
    # PaliGemma has special prompt format for VQA
    prompt = "caption en"
    llm = LLM(model="google/paligemma-3b-mix-224",
422
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
423
424
    stop_token_ids = None
    return llm, prompt, stop_token_ids
425
426


427
428
# PaliGemma 2
def run_paligemma2(question: str, modality: str):
429
    assert modality == "image"
430

431
432
433
    # PaliGemma 2 has special prompt format for VQA
    prompt = "caption en"
    llm = LLM(model="google/paligemma2-3b-ft-docci-448",
434
              disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
435
436
437
438
    stop_token_ids = None
    return llm, prompt, stop_token_ids


439
440
# Phi-3-Vision
def run_phi3v(question: str, modality: str):
441
442
    assert modality == "image"

443
    prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
444

445
446
447
448
449
450
451
452
453
454
455
456
    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
457
    llm = LLM(
458
459
        model="microsoft/Phi-3.5-vision-instruct",
        trust_remote_code=True,
460
        max_model_len=4096,
461
        max_num_seqs=2,
462
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
463
        mm_processor_kwargs={"num_crops": 16},
464
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
465
466
467
468
469
    )
    stop_token_ids = None
    return llm, prompt, stop_token_ids


470
471
472
473
474
475
# Pixtral HF-format
def run_pixtral_hf(question: str, modality: str):
    assert modality == "image"

    model_name = "mistral-community/pixtral-12b"

476
    # NOTE: Need L40 (or equivalent) to avoid OOM
477
478
479
    llm = LLM(
        model=model_name,
        max_model_len=8192,
480
        max_num_seqs=2,
481
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
482
483
484
485
486
487
488
    )

    prompt = f"<s>[INST]{question}\n[IMG][/INST]"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


489
490
# Qwen
def run_qwen_vl(question: str, modality: str):
491
492
493
    assert modality == "image"

    llm = LLM(
494
        model="Qwen/Qwen-VL",
495
        trust_remote_code=True,
496
497
        max_model_len=1024,
        max_num_seqs=2,
498
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
499
500
    )

501
    prompt = f"{question}Picture 1: <img></img>\n"
502
503
504
505
    stop_token_ids = None
    return llm, prompt, stop_token_ids


506
507
# Qwen2-VL
def run_qwen2_vl(question: str, modality: str):
508

509
    model_name = "Qwen/Qwen2-VL-7B-Instruct"
510

511
512
    llm = LLM(
        model=model_name,
513
514
515
        max_model_len=4096,
        max_num_seqs=5,
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
516
        mm_processor_kwargs={
517
518
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
519
        },
520
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
521
    )
522

523
524
525
526
527
    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

528
    prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
529
              f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
530
531
532
              f"{question}<|im_end|>\n"
              "<|im_start|>assistant\n")
    stop_token_ids = None
533
534
535
    return llm, prompt, stop_token_ids


Roger Wang's avatar
Roger Wang committed
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
# Qwen2.5-VL
def run_qwen2_5_vl(question: str, modality: str):

    model_name = "Qwen/Qwen2.5-VL-3B-Instruct"

    llm = LLM(
        model=model_name,
        max_model_len=4096,
        max_num_seqs=5,
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
            "fps": 1,
        },
        disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
    )

    if modality == "image":
        placeholder = "<|image_pad|>"
    elif modality == "video":
        placeholder = "<|video_pad|>"

    prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
              f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
              f"{question}<|im_end|>\n"
              "<|im_start|>assistant\n")
    stop_token_ids = None
    return llm, prompt, stop_token_ids


566
model_example_map = {
567
568
569
    "aria": run_aria,
    "blip-2": run_blip2,
    "chameleon": run_chameleon,
570
    "deepseek_vl_v2": run_deepseek_vl2,
571
572
573
574
575
    "fuyu": run_fuyu,
    "glm4v": run_glm4v,
    "h2ovl_chat": run_h2ovl,
    "idefics3": run_idefics3,
    "internvl_chat": run_internvl,
576
577
    "llava": run_llava,
    "llava-next": run_llava_next,
578
    "llava-next-video": run_llava_next_video,
579
    "llava-onevision": run_llava_onevision,
580
    "mantis": run_mantis,
581
    "minicpmo": run_minicpmo,
582
    "minicpmv": run_minicpmv,
583
584
    "mllama": run_mllama,
    "molmo": run_molmo,
585
    "NVLM_D": run_nvlm_d,
586
587
588
589
    "paligemma": run_paligemma,
    "paligemma2": run_paligemma2,
    "phi3_v": run_phi3v,
    "pixtral_hf": run_pixtral_hf,
590
    "qwen_vl": run_qwen_vl,
591
    "qwen2_vl": run_qwen2_vl,
Roger Wang's avatar
Roger Wang committed
592
    "qwen2_5_vl": run_qwen2_5_vl,
593
594
595
}


596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
        image = ImageAsset("cherry_blossom") \
            .pil_image.convert("RGB")
        img_question = "What is the content of this image?"

        return {
            "data": image,
            "question": img_question,
        }

    if args.modality == "video":
        # Input video and question
        video = VideoAsset(name="sample_demo_1.mp4",
                           num_frames=args.num_frames).np_ndarrays
        vid_question = "Why is this video funny?"

        return {
            "data": video,
            "question": vid_question,
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
def apply_image_repeat(image_repeat_prob, num_prompts, data, prompt, modality):
    """Repeats images with provided probability of "image_repeat_prob". 
    Used to simulate hit/miss for the MM preprocessor cache.
    """
    assert (image_repeat_prob <= 1.0 and image_repeat_prob >= 0)
    no_yes = [0, 1]
    probs = [1.0 - image_repeat_prob, image_repeat_prob]

    inputs = []
    cur_image = data
    for i in range(num_prompts):
        if image_repeat_prob is not None:
            res = random.choices(no_yes, probs)[0]
            if res == 0:
                # No repeat => Modify one pixel
                cur_image = cur_image.copy()
                new_val = (i // 256 // 256, i // 256, i % 256)
                cur_image.putpixel((0, 0), new_val)

        inputs.append({
            "prompt": prompt,
            "multi_modal_data": {
                modality: cur_image
            }
        })

    return inputs


658
659
660
661
662
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

663
664
665
666
667
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
    question = mm_input["question"]

668
    llm, prompt, stop_token_ids = model_example_map[model](question, modality)
669
670
671

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
672
673
674
    sampling_params = SamplingParams(temperature=0.2,
                                     max_tokens=64,
                                     stop_token_ids=stop_token_ids)
675
676
677
678
679
680
681

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        inputs = {
            "prompt": prompt,
            "multi_modal_data": {
682
                modality: data
683
684
685
686
687
            },
        }

    else:
        # Batch inference
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
        if args.image_repeat_prob is not None:
            # Repeat images with specified probability of "image_repeat_prob"
            inputs = apply_image_repeat(args.image_repeat_prob,
                                        args.num_prompts, data, prompt,
                                        modality)
        else:
            # Use the same image for all prompts
            inputs = [{
                "prompt": prompt,
                "multi_modal_data": {
                    modality: data
                },
            } for _ in range(args.num_prompts)]

    if args.time_generate:
        import time
        start_time = time.time()
        outputs = llm.generate(inputs, sampling_params=sampling_params)
        elapsed_time = time.time() - start_time
        print("-- generate time = {}".format(elapsed_time))
708

709
710
    else:
        outputs = llm.generate(inputs, sampling_params=sampling_params)
711
712
713
714
715
716
717
718
719

    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)


if __name__ == "__main__":
    parser = FlexibleArgumentParser(
        description='Demo on using vLLM for offline inference with '
Cyrus Leung's avatar
Cyrus Leung committed
720
        'vision language models for text generation')
721
722
723
724
725
726
727
728
    parser.add_argument('--model-type',
                        '-m',
                        type=str,
                        default="llava",
                        choices=model_example_map.keys(),
                        help='Huggingface "model_type".')
    parser.add_argument('--num-prompts',
                        type=int,
729
                        default=4,
730
                        help='Number of prompts to run.')
731
732
733
    parser.add_argument('--modality',
                        type=str,
                        default="image",
734
                        choices=['image', 'video'],
735
736
737
738
739
                        help='Modality of the input.')
    parser.add_argument('--num-frames',
                        type=int,
                        default=16,
                        help='Number of frames to extract from the video.')
740
741
742
743
744
745
746
747
748

    parser.add_argument(
        '--image-repeat-prob',
        type=float,
        default=None,
        help='Simulates the hit-ratio for multi-modal preprocessor cache'
        ' (if enabled)')

    parser.add_argument(
749
        '--disable-mm-preprocessor-cache',
750
        action='store_true',
751
        help='If True, disables caching of multi-modal preprocessor/mapper.')
752
753
754
755
756
757

    parser.add_argument(
        '--time-generate',
        action='store_true',
        help='If True, then print the total generate() call time')

758
    args = parser.parse_args()
759
    main(args)