offline_inference_vision_language.py 11.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
"""
This example shows how to use vLLM for running offline inference 
with the correct prompt format on vision language models.

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
from transformers import AutoTokenizer

from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
12
from vllm.assets.video import VideoAsset
13
14
15
16
from vllm.utils import FlexibleArgumentParser


# LLaVA-1.5
17
18
def run_llava(question, modality):
    assert modality == "image"
19
20
21
22

    prompt = f"USER: <image>\n{question}\nASSISTANT:"

    llm = LLM(model="llava-hf/llava-1.5-7b-hf")
23
24
    stop_token_ids = None
    return llm, prompt, stop_token_ids
25
26
27


# LLaVA-1.6/LLaVA-NeXT
28
29
def run_llava_next(question, modality):
    assert modality == "image"
30
31

    prompt = f"[INST] <image>\n{question} [/INST]"
32
33
34
35
36
37
38
    llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
    stop_token_ids = None
    return llm, prompt, stop_token_ids


# LlaVA-NeXT-Video
# Currently only support for video input
39
40
41
def run_llava_next_video(question, modality):
    assert modality == "video"

42
43
    prompt = f"USER: <video>\n{question} ASSISTANT:"
    llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
44
45
    stop_token_ids = None
    return llm, prompt, stop_token_ids
46
47


48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# LLaVA-OneVision
def run_llava_onevision(question, modality):

    if modality == "video":
        prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    elif modality == "image":
        prompt = f"<|im_start|>user <image>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
              max_model_len=32768)
    stop_token_ids = None
    return llm, prompt, stop_token_ids


65
# Fuyu
66
67
def run_fuyu(question, modality):
    assert modality == "image"
68
69
70

    prompt = f"{question}\n"
    llm = LLM(model="adept/fuyu-8b")
71
72
    stop_token_ids = None
    return llm, prompt, stop_token_ids
73
74
75


# Phi-3-Vision
76
77
def run_phi3v(question, modality):
    assert modality == "image"
78
79
80
81
82
83
84
85

    prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"  # noqa: E501
    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (128k) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

    # In this example, we override max_num_seqs to 5 while
    # keeping the original context length of 128k.
86
87
88
89
90
91
92
93
94
95
96
97
98

    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
99
100
101
102
    llm = LLM(
        model="microsoft/Phi-3-vision-128k-instruct",
        trust_remote_code=True,
        max_num_seqs=5,
103
        mm_processor_kwargs={"num_crops": 16},
104
    )
105
106
    stop_token_ids = None
    return llm, prompt, stop_token_ids
107
108
109


# PaliGemma
110
111
def run_paligemma(question, modality):
    assert modality == "image"
112

113
114
    # PaliGemma has special prompt format for VQA
    prompt = "caption en"
115
    llm = LLM(model="google/paligemma-3b-mix-224")
116
117
    stop_token_ids = None
    return llm, prompt, stop_token_ids
118
119
120


# Chameleon
121
122
def run_chameleon(question, modality):
    assert modality == "image"
123
124
125

    prompt = f"{question}<image>"
    llm = LLM(model="facebook/chameleon-7b")
126
127
    stop_token_ids = None
    return llm, prompt, stop_token_ids
128
129
130


# MiniCPM-V
131
132
def run_minicpmv(question, modality):
    assert modality == "image"
133
134
135
136
137
138
139

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
140
141
142
143
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

    #2.6
    model_name = "openbmb/MiniCPM-V-2_6"
144
145
146
147
148
149
    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    llm = LLM(
        model=model_name,
        trust_remote_code=True,
    )
150
151
152
153
154
155
156
157
158
159
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

    # 2.6
    stop_tokens = ['<|im_end|>', '<|endoftext|>']
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
160
161
162
163
164
165
166
167

    messages = [{
        'role': 'user',
        'content': f'(<image>./</image>)\n{question}'
    }]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)
168
    return llm, prompt, stop_token_ids
169
170


171
# InternVL
172
173
174
def run_internvl(question, modality):
    assert modality == "image"

175
176
    model_name = "OpenGVLab/InternVL2-2B"

177
    llm = LLM(
178
        model=model_name,
179
180
181
        trust_remote_code=True,
        max_num_seqs=5,
    )
182
183
184
185
186
187
188
189
190
191
192
193
194
195

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B#service
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
196
    return llm, prompt, stop_token_ids
197
198


199
# BLIP-2
200
201
def run_blip2(question, modality):
    assert modality == "image"
202
203
204
205
206

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
    prompt = f"Question: {question} Answer:"
    llm = LLM(model="Salesforce/blip2-opt-2.7b")
207
208
    stop_token_ids = None
    return llm, prompt, stop_token_ids
209
210


211
# Qwen
212
213
def run_qwen_vl(question, modality):
    assert modality == "image"
214
215
216
217
218
219
220
221
222
223
224
225

    llm = LLM(
        model="Qwen/Qwen-VL",
        trust_remote_code=True,
        max_num_seqs=5,
    )

    prompt = f"{question}Picture 1: <img></img>\n"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


226
# Qwen2-VL
227
228
229
def run_qwen2_vl(question, modality):
    assert modality == "image"

230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
    model_name = "Qwen/Qwen2-VL-7B-Instruct"

    llm = LLM(
        model=model_name,
        max_num_seqs=5,
    )

    prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
              "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
              f"{question}<|im_end|>\n"
              "<|im_start|>assistant\n")
    stop_token_ids = None
    return llm, prompt, stop_token_ids


245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
# LLama
def run_mllama(question, modality):
    assert modality == "image"

    model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"

    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (131072) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

    # The configuration below has been confirmed to launch on a
    # single H100 GPU.
    llm = LLM(
        model=model_name,
        max_num_seqs=16,
        enforce_eager=True,
    )

    prompt = f"<|image|><|begin_of_text|>{question}"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


268
269
270
model_example_map = {
    "llava": run_llava,
    "llava-next": run_llava_next,
271
    "llava-next-video": run_llava_next_video,
272
    "llava-onevision": run_llava_onevision,
273
274
275
276
277
    "fuyu": run_fuyu,
    "phi3_v": run_phi3v,
    "paligemma": run_paligemma,
    "chameleon": run_chameleon,
    "minicpmv": run_minicpmv,
278
    "blip-2": run_blip2,
279
    "internvl_chat": run_internvl,
280
    "qwen_vl": run_qwen_vl,
281
    "qwen2_vl": run_qwen2_vl,
282
    "mllama": run_mllama,
283
284
285
}


286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
        image = ImageAsset("cherry_blossom") \
            .pil_image.convert("RGB")
        img_question = "What is the content of this image?"

        return {
            "data": image,
            "question": img_question,
        }

    if args.modality == "video":
        # Input video and question
        video = VideoAsset(name="sample_demo_1.mp4",
                           num_frames=args.num_frames).np_ndarrays
        vid_question = "Why is this video funny?"

        return {
            "data": video,
            "question": vid_question,
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


319
320
321
322
323
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

324
325
326
327
328
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
    question = mm_input["question"]

329
    llm, prompt, stop_token_ids = model_example_map[model](question, modality)
330
331
332

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
333
334
335
    sampling_params = SamplingParams(temperature=0.2,
                                     max_tokens=64,
                                     stop_token_ids=stop_token_ids)
336
337
338
339
340
341
342

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        inputs = {
            "prompt": prompt,
            "multi_modal_data": {
343
                modality: data
344
345
346
347
348
349
350
351
            },
        }

    else:
        # Batch inference
        inputs = [{
            "prompt": prompt,
            "multi_modal_data": {
352
                modality: data
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
            },
        } for _ in range(args.num_prompts)]

    outputs = llm.generate(inputs, sampling_params=sampling_params)

    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)


if __name__ == "__main__":
    parser = FlexibleArgumentParser(
        description='Demo on using vLLM for offline inference with '
        'vision language models')
    parser.add_argument('--model-type',
                        '-m',
                        type=str,
                        default="llava",
                        choices=model_example_map.keys(),
                        help='Huggingface "model_type".')
    parser.add_argument('--num-prompts',
                        type=int,
375
                        default=4,
376
                        help='Number of prompts to run.')
377
378
379
    parser.add_argument('--modality',
                        type=str,
                        default="image",
380
                        choices=['image', 'video'],
381
382
383
384
385
                        help='Modality of the input.')
    parser.add_argument('--num-frames',
                        type=int,
                        default=16,
                        help='Number of frames to extract from the video.')
386
387
    args = parser.parse_args()
    main(args)