offline_inference_vision_language.py 20.4 KB
Newer Older
1
"""
Cyrus Leung's avatar
Cyrus Leung committed
2
3
This example shows how to use vLLM for running offline inference with
the correct prompt format on vision language models for text generation.
4
5
6
7

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
8
9
import random

10
11
12
13
from transformers import AutoTokenizer

from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
14
from vllm.assets.video import VideoAsset
15
16
from vllm.utils import FlexibleArgumentParser

17
18
19
20
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
# lower-end GPUs.
# Unless specified, these settings have been tested to work on a single L4.

21
22

# LLaVA-1.5
23
def run_llava(question: str, modality: str):
24
    assert modality == "image"
25
26
27

    prompt = f"USER: <image>\n{question}\nASSISTANT:"

28
29
30
    llm = LLM(model="llava-hf/llava-1.5-7b-hf",
              max_model_len=4096,
              mm_cache_preprocessor=args.mm_cache_preprocessor)
31
32
    stop_token_ids = None
    return llm, prompt, stop_token_ids
33
34
35


# LLaVA-1.6/LLaVA-NeXT
36
def run_llava_next(question: str, modality: str):
37
    assert modality == "image"
38
39

    prompt = f"[INST] <image>\n{question} [/INST]"
40
41
42
    llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
              max_model_len=8192,
              mm_cache_preprocessor=args.mm_cache_preprocessor)
43
44
45
46
47
48
    stop_token_ids = None
    return llm, prompt, stop_token_ids


# LlaVA-NeXT-Video
# Currently only support for video input
49
def run_llava_next_video(question: str, modality: str):
50
51
    assert modality == "video"

52
    prompt = f"USER: <video>\n{question} ASSISTANT:"
53
54
55
    llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
              max_model_len=8192,
              mm_cache_preprocessor=args.mm_cache_preprocessor)
56
57
    stop_token_ids = None
    return llm, prompt, stop_token_ids
58
59


60
# LLaVA-OneVision
61
def run_llava_onevision(question: str, modality: str):
62
63
64
65
66
67
68
69
70
71

    if modality == "video":
        prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    elif modality == "image":
        prompt = f"<|im_start|>user <image>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
72
73
              max_model_len=16384,
              mm_cache_preprocessor=args.mm_cache_preprocessor)
74
75
76
77
    stop_token_ids = None
    return llm, prompt, stop_token_ids


78
# Fuyu
79
def run_fuyu(question: str, modality: str):
80
    assert modality == "image"
81
82

    prompt = f"{question}\n"
83
84
85
86
    llm = LLM(model="adept/fuyu-8b",
              max_model_len=2048,
              max_num_seqs=2,
              mm_cache_preprocessor=args.mm_cache_preprocessor)
87
88
    stop_token_ids = None
    return llm, prompt, stop_token_ids
89
90
91


# Phi-3-Vision
92
def run_phi3v(question: str, modality: str):
93
    assert modality == "image"
94

95
    prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
96

97
98
99
100
101
102
103
104
105
106
107
108
    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
109
    llm = LLM(
110
        model="microsoft/Phi-3.5-vision-instruct",
111
        trust_remote_code=True,
112
113
        max_model_len=4096,
        max_num_seqs=2,
114
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
115
        mm_processor_kwargs={"num_crops": 16},
116
        mm_cache_preprocessor=args.mm_cache_preprocessor,
117
    )
118
119
    stop_token_ids = None
    return llm, prompt, stop_token_ids
120
121
122


# PaliGemma
123
def run_paligemma(question: str, modality: str):
124
    assert modality == "image"
125

126
127
    # PaliGemma has special prompt format for VQA
    prompt = "caption en"
128
129
    llm = LLM(model="google/paligemma-3b-mix-224",
              mm_cache_preprocessor=args.mm_cache_preprocessor)
130
131
    stop_token_ids = None
    return llm, prompt, stop_token_ids
132
133


Jani Monoses's avatar
Jani Monoses committed
134
135
136
137
138
139
140
141
142
143
144
145
# PaliGemma 2
def run_paligemma2(question: str, modality: str):
    assert modality == "image"

    # PaliGemma 2 has special prompt format for VQA
    prompt = "caption en"
    llm = LLM(model="google/paligemma2-3b-ft-docci-448",
              mm_cache_preprocessor=args.mm_cache_preprocessor)
    stop_token_ids = None
    return llm, prompt, stop_token_ids


146
# Chameleon
147
def run_chameleon(question: str, modality: str):
148
    assert modality == "image"
149
150

    prompt = f"{question}<image>"
151
152
153
    llm = LLM(model="facebook/chameleon-7b",
              max_model_len=4096,
              mm_cache_preprocessor=args.mm_cache_preprocessor)
154
155
    stop_token_ids = None
    return llm, prompt, stop_token_ids
156
157
158


# MiniCPM-V
159
def run_minicpmv(question: str, modality: str):
160
    assert modality == "image"
161
162
163
164
165
166
167

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
168
169
170
171
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

    #2.6
    model_name = "openbmb/MiniCPM-V-2_6"
172
173
174
175
    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    llm = LLM(
        model=model_name,
176
177
        max_model_len=4096,
        max_num_seqs=2,
178
        trust_remote_code=True,
179
        mm_cache_preprocessor=args.mm_cache_preprocessor,
180
    )
181
182
183
184
185
186
187
188
189
190
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

    # 2.6
    stop_tokens = ['<|im_end|>', '<|endoftext|>']
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
191
192
193
194
195
196
197
198

    messages = [{
        'role': 'user',
        'content': f'(<image>./</image>)\n{question}'
    }]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)
199
    return llm, prompt, stop_token_ids
200
201


202
203
204
205
206
207
208
209
210
211
# H2OVL-Mississippi
def run_h2ovl(question: str, modality: str):
    assert modality == "image"

    model_name = "h2oai/h2ovl-mississippi-2b"

    llm = LLM(
        model=model_name,
        trust_remote_code=True,
        max_model_len=8192,
212
        mm_cache_preprocessor=args.mm_cache_preprocessor,
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)

    # Stop tokens for H2OVL-Mississippi
    # https://huggingface.co/h2oai/h2ovl-mississippi-2b
    stop_token_ids = [tokenizer.eos_token_id]
    return llm, prompt, stop_token_ids


228
# InternVL
229
def run_internvl(question: str, modality: str):
230
231
    assert modality == "image"

232
233
    model_name = "OpenGVLab/InternVL2-2B"

234
    llm = LLM(
235
        model=model_name,
236
        trust_remote_code=True,
237
        max_model_len=4096,
238
        mm_cache_preprocessor=args.mm_cache_preprocessor,
239
    )
240
241
242
243
244
245
246
247
248
249
250

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
251
    # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
252
253
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
254
    return llm, prompt, stop_token_ids
255
256


257
258
259
260
261
262
263
264
265
266
267
268
# NVLM-D
def run_nvlm_d(question: str, modality: str):
    assert modality == "image"

    model_name = "nvidia/NVLM-D-72B"

    # Adjust this as necessary to fit in GPU
    llm = LLM(
        model=model_name,
        trust_remote_code=True,
        max_model_len=4096,
        tensor_parallel_size=4,
269
        mm_cache_preprocessor=args.mm_cache_preprocessor,
270
271
272
273
274
275
276
277
278
279
280
281
    )

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)
    stop_token_ids = None
    return llm, prompt, stop_token_ids


282
# BLIP-2
283
def run_blip2(question: str, modality: str):
284
    assert modality == "image"
285
286
287
288

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
    prompt = f"Question: {question} Answer:"
289
290
    llm = LLM(model="Salesforce/blip2-opt-2.7b",
              mm_cache_preprocessor=args.mm_cache_preprocessor)
291
292
    stop_token_ids = None
    return llm, prompt, stop_token_ids
293
294


295
# Qwen
296
def run_qwen_vl(question: str, modality: str):
297
    assert modality == "image"
298
299
300
301

    llm = LLM(
        model="Qwen/Qwen-VL",
        trust_remote_code=True,
302
303
        max_model_len=1024,
        max_num_seqs=2,
304
        mm_cache_preprocessor=args.mm_cache_preprocessor,
305
306
307
308
309
310
311
    )

    prompt = f"{question}Picture 1: <img></img>\n"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


312
# Qwen2-VL
313
def run_qwen2_vl(question: str, modality: str):
314
315
    assert modality == "image"

316
317
318
319
    model_name = "Qwen/Qwen2-VL-7B-Instruct"

    llm = LLM(
        model=model_name,
320
        max_model_len=4096,
321
        max_num_seqs=5,
322
323
324
325
326
        # Note - mm_processor_kwargs can also be passed to generate/chat calls
        mm_processor_kwargs={
            "min_pixels": 28 * 28,
            "max_pixels": 1280 * 28 * 28,
        },
327
        mm_cache_preprocessor=args.mm_cache_preprocessor,
328
329
330
331
332
333
334
335
336
337
    )

    prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
              "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
              f"{question}<|im_end|>\n"
              "<|im_start|>assistant\n")
    stop_token_ids = None
    return llm, prompt, stop_token_ids


338
339
340
341
342
343
344
345
346
# Pixtral HF-format
def run_pixtral_hf(question: str, modality: str):
    assert modality == "image"

    model_name = "mistral-community/pixtral-12b"

    llm = LLM(
        model=model_name,
        max_model_len=8192,
347
        mm_cache_preprocessor=args.mm_cache_preprocessor,
348
349
350
351
352
353
354
    )

    prompt = f"<s>[INST]{question}\n[IMG][/INST]"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


355
356
# LLama 3.2
def run_mllama(question: str, modality: str):
357
358
359
360
361
362
363
364
    assert modality == "image"

    model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"

    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (131072) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

365
    # The configuration below has been confirmed to launch on a single L40 GPU.
366
367
    llm = LLM(
        model=model_name,
368
        max_model_len=4096,
369
370
        max_num_seqs=16,
        enforce_eager=True,
371
        mm_cache_preprocessor=args.mm_cache_preprocessor,
372
373
374
375
376
377
378
    )

    prompt = f"<|image|><|begin_of_text|>{question}"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


379
380
381
382
383
384
385
386
387
388
# Molmo
def run_molmo(question, modality):
    assert modality == "image"

    model_name = "allenai/Molmo-7B-D-0924"

    llm = LLM(
        model=model_name,
        trust_remote_code=True,
        dtype="bfloat16",
389
        mm_cache_preprocessor=args.mm_cache_preprocessor,
390
391
392
393
394
395
396
    )

    prompt = question
    stop_token_ids = None
    return llm, prompt, stop_token_ids


397
398
399
400
401
402
403
404
405
# GLM-4v
def run_glm4v(question: str, modality: str):
    assert modality == "image"
    model_name = "THUDM/glm-4v-9b"

    llm = LLM(model=model_name,
              max_model_len=2048,
              max_num_seqs=2,
              trust_remote_code=True,
406
407
              enforce_eager=True,
              mm_cache_preprocessor=args.mm_cache_preprocessor)
408
409
410
411
412
    prompt = question
    stop_token_ids = [151329, 151336, 151338]
    return llm, prompt, stop_token_ids


413
414
415
416
417
# Idefics3-8B-Llama3
def run_idefics3(question: str, modality: str):
    assert modality == "image"
    model_name = "HuggingFaceM4/Idefics3-8B-Llama3"

418
419
420
421
422
423
424
425
426
427
428
429
    llm = LLM(
        model=model_name,
        max_model_len=8192,
        max_num_seqs=2,
        enforce_eager=True,
        # if you are running out of memory, you can reduce the "longest_edge".
        # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
        mm_processor_kwargs={
            "size": {
                "longest_edge": 3 * 364
            },
        },
430
        mm_cache_preprocessor=args.mm_cache_preprocessor,
431
    )
432
433
434
435
436
437
438
    prompt = (
        f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
    )
    stop_token_ids = None
    return llm, prompt, stop_token_ids


439
440
441
442
443
444
445
446
# Aria
def run_aria(question: str, modality: str):
    assert modality == "image"
    model_name = "rhymes-ai/Aria"

    llm = LLM(model=model_name,
              tokenizer_mode="slow",
              trust_remote_code=True,
447
448
              dtype="bfloat16",
              mm_cache_preprocessor=args.mm_cache_preprocessor)
449
450
451
452
453
454
455
456

    prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
              "<|im_end|>\n<|im_start|>assistant\n")

    stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
    return llm, prompt, stop_token_ids


457
458
459
460
461
462
463
464
465
466
467
# Mantis
def run_mantis(question: str, modality: str):
    assert modality == "image"

    llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n'  # noqa: E501
    prompt = llama3_template.format(f"{question}\n<image>")

    llm = LLM(
        model="TIGER-Lab/Mantis-8B-siglip-llama3",
        max_model_len=4096,
        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
468
        mm_cache_preprocessor=args.mm_cache_preprocessor,
469
470
471
472
473
    )
    stop_token_ids = [128009]
    return llm, prompt, stop_token_ids


474
475
476
model_example_map = {
    "llava": run_llava,
    "llava-next": run_llava_next,
477
    "llava-next-video": run_llava_next_video,
478
    "llava-onevision": run_llava_onevision,
479
480
481
    "fuyu": run_fuyu,
    "phi3_v": run_phi3v,
    "paligemma": run_paligemma,
Jani Monoses's avatar
Jani Monoses committed
482
    "paligemma2": run_paligemma2,
483
484
    "chameleon": run_chameleon,
    "minicpmv": run_minicpmv,
485
    "blip-2": run_blip2,
486
    "h2ovl_chat": run_h2ovl,
487
    "internvl_chat": run_internvl,
488
    "NVLM_D": run_nvlm_d,
489
    "qwen_vl": run_qwen_vl,
490
    "qwen2_vl": run_qwen2_vl,
491
    "pixtral_hf": run_pixtral_hf,
492
    "mllama": run_mllama,
493
    "molmo": run_molmo,
494
    "glm4v": run_glm4v,
495
    "idefics3": run_idefics3,
496
    "aria": run_aria,
497
    "mantis": run_mantis,
498
499
500
}


501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
        image = ImageAsset("cherry_blossom") \
            .pil_image.convert("RGB")
        img_question = "What is the content of this image?"

        return {
            "data": image,
            "question": img_question,
        }

    if args.modality == "video":
        # Input video and question
        video = VideoAsset(name="sample_demo_1.mp4",
                           num_frames=args.num_frames).np_ndarrays
        vid_question = "Why is this video funny?"

        return {
            "data": video,
            "question": vid_question,
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
def apply_image_repeat(image_repeat_prob, num_prompts, data, prompt, modality):
    """Repeats images with provided probability of "image_repeat_prob". 
    Used to simulate hit/miss for the MM preprocessor cache.
    """
    assert (image_repeat_prob <= 1.0 and image_repeat_prob >= 0)
    no_yes = [0, 1]
    probs = [1.0 - image_repeat_prob, image_repeat_prob]

    inputs = []
    cur_image = data
    for i in range(num_prompts):
        if image_repeat_prob is not None:
            res = random.choices(no_yes, probs)[0]
            if res == 0:
                # No repeat => Modify one pixel
                cur_image = cur_image.copy()
                new_val = (i // 256 // 256, i // 256, i % 256)
                cur_image.putpixel((0, 0), new_val)

        inputs.append({
            "prompt": prompt,
            "multi_modal_data": {
                modality: cur_image
            }
        })

    return inputs


563
564
565
566
567
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

568
569
570
571
572
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
    question = mm_input["question"]

573
    llm, prompt, stop_token_ids = model_example_map[model](question, modality)
574
575
576

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
577
578
579
    sampling_params = SamplingParams(temperature=0.2,
                                     max_tokens=64,
                                     stop_token_ids=stop_token_ids)
580
581
582
583
584
585
586

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        inputs = {
            "prompt": prompt,
            "multi_modal_data": {
587
                modality: data
588
589
590
591
592
            },
        }

    else:
        # Batch inference
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
        if args.image_repeat_prob is not None:
            # Repeat images with specified probability of "image_repeat_prob"
            inputs = apply_image_repeat(args.image_repeat_prob,
                                        args.num_prompts, data, prompt,
                                        modality)
        else:
            # Use the same image for all prompts
            inputs = [{
                "prompt": prompt,
                "multi_modal_data": {
                    modality: data
                },
            } for _ in range(args.num_prompts)]

    if args.time_generate:
        import time
        start_time = time.time()
        outputs = llm.generate(inputs, sampling_params=sampling_params)
        elapsed_time = time.time() - start_time
        print("-- generate time = {}".format(elapsed_time))
613

614
615
    else:
        outputs = llm.generate(inputs, sampling_params=sampling_params)
616
617
618
619
620
621
622
623
624

    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)


if __name__ == "__main__":
    parser = FlexibleArgumentParser(
        description='Demo on using vLLM for offline inference with '
Cyrus Leung's avatar
Cyrus Leung committed
625
        'vision language models for text generation')
626
627
628
629
630
631
632
633
    parser.add_argument('--model-type',
                        '-m',
                        type=str,
                        default="llava",
                        choices=model_example_map.keys(),
                        help='Huggingface "model_type".')
    parser.add_argument('--num-prompts',
                        type=int,
634
                        default=4,
635
                        help='Number of prompts to run.')
636
637
638
    parser.add_argument('--modality',
                        type=str,
                        default="image",
639
                        choices=['image', 'video'],
640
641
642
643
644
                        help='Modality of the input.')
    parser.add_argument('--num-frames',
                        type=int,
                        default=16,
                        help='Number of frames to extract from the video.')
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662

    parser.add_argument(
        '--image-repeat-prob',
        type=float,
        default=None,
        help='Simulates the hit-ratio for multi-modal preprocessor cache'
        ' (if enabled)')

    parser.add_argument(
        '--mm-cache-preprocessor',
        action='store_true',
        help='If True, enable caching of multi-modal preprocessor/mapper.')

    parser.add_argument(
        '--time-generate',
        action='store_true',
        help='If True, then print the total generate() call time')

663
    args = parser.parse_args()
664
    main(args)