offline_inference_vision_language.py 10.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
"""
This example shows how to use vLLM for running offline inference 
with the correct prompt format on vision language models.

For most models, the prompt format should follow corresponding examples
on HuggingFace model repository.
"""
from transformers import AutoTokenizer

from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
12
from vllm.assets.video import VideoAsset
13
14
15
16
from vllm.utils import FlexibleArgumentParser


# LLaVA-1.5
17
18
def run_llava(question, modality):
    assert modality == "image"
19
20
21
22

    prompt = f"USER: <image>\n{question}\nASSISTANT:"

    llm = LLM(model="llava-hf/llava-1.5-7b-hf")
23
24
    stop_token_ids = None
    return llm, prompt, stop_token_ids
25
26
27


# LLaVA-1.6/LLaVA-NeXT
28
29
def run_llava_next(question, modality):
    assert modality == "image"
30
31

    prompt = f"[INST] <image>\n{question} [/INST]"
32
33
34
35
36
37
38
    llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
    stop_token_ids = None
    return llm, prompt, stop_token_ids


# LlaVA-NeXT-Video
# Currently only support for video input
39
40
41
def run_llava_next_video(question, modality):
    assert modality == "video"

42
43
    prompt = f"USER: <video>\n{question} ASSISTANT:"
    llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
44
45
    stop_token_ids = None
    return llm, prompt, stop_token_ids
46
47


48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# LLaVA-OneVision
def run_llava_onevision(question, modality):

    if modality == "video":
        prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    elif modality == "image":
        prompt = f"<|im_start|>user <image>\n{question}<|im_end|> \
        <|im_start|>assistant\n"

    llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
              max_model_len=32768)
    stop_token_ids = None
    return llm, prompt, stop_token_ids


65
# Fuyu
66
67
def run_fuyu(question, modality):
    assert modality == "image"
68
69
70

    prompt = f"{question}\n"
    llm = LLM(model="adept/fuyu-8b")
71
72
    stop_token_ids = None
    return llm, prompt, stop_token_ids
73
74
75


# Phi-3-Vision
76
77
def run_phi3v(question, modality):
    assert modality == "image"
78
79
80
81
82
83
84
85

    prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"  # noqa: E501
    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (128k) for this model may cause OOM.
    # You may lower either to run this example on lower-end GPUs.

    # In this example, we override max_num_seqs to 5 while
    # keeping the original context length of 128k.
86
87
88
89
90
91
92
93
94
95
96
97
98

    # num_crops is an override kwarg to the multimodal image processor;
    # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
    # to use 16 for single frame scenarios, and 4 for multi-frame.
    #
    # Generally speaking, a larger value for num_crops results in more
    # tokens per image instance, because it may scale the image more in
    # the image preprocessing. Some references in the model docs and the
    # formula for image tokens after the preprocessing
    # transform can be found below.
    #
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
    # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
99
100
101
102
    llm = LLM(
        model="microsoft/Phi-3-vision-128k-instruct",
        trust_remote_code=True,
        max_num_seqs=5,
103
        mm_processor_kwargs={"num_crops": 16},
104
    )
105
106
    stop_token_ids = None
    return llm, prompt, stop_token_ids
107
108
109


# PaliGemma
110
111
def run_paligemma(question, modality):
    assert modality == "image"
112

113
114
    # PaliGemma has special prompt format for VQA
    prompt = "caption en"
115
    llm = LLM(model="google/paligemma-3b-mix-224")
116
117
    stop_token_ids = None
    return llm, prompt, stop_token_ids
118
119
120


# Chameleon
121
122
def run_chameleon(question, modality):
    assert modality == "image"
123
124
125

    prompt = f"{question}<image>"
    llm = LLM(model="facebook/chameleon-7b")
126
127
    stop_token_ids = None
    return llm, prompt, stop_token_ids
128
129
130


# MiniCPM-V
131
132
def run_minicpmv(question, modality):
    assert modality == "image"
133
134
135
136
137
138
139

    # 2.0
    # The official repo doesn't work yet, so we need to use a fork for now
    # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
    # model_name = "HwwwH/MiniCPM-V-2"

    # 2.5
140
141
142
143
    # model_name = "openbmb/MiniCPM-Llama3-V-2_5"

    #2.6
    model_name = "openbmb/MiniCPM-V-2_6"
144
145
146
147
148
149
    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    llm = LLM(
        model=model_name,
        trust_remote_code=True,
    )
150
151
152
153
154
155
156
157
158
159
    # NOTE The stop_token_ids are different for various versions of MiniCPM-V
    # 2.0
    # stop_token_ids = [tokenizer.eos_id]

    # 2.5
    # stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]

    # 2.6
    stop_tokens = ['<|im_end|>', '<|endoftext|>']
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
160
161
162
163
164
165
166
167

    messages = [{
        'role': 'user',
        'content': f'(<image>./</image>)\n{question}'
    }]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)
168
    return llm, prompt, stop_token_ids
169
170


171
# InternVL
172
173
174
def run_internvl(question, modality):
    assert modality == "image"

175
176
    model_name = "OpenGVLab/InternVL2-2B"

177
    llm = LLM(
178
        model=model_name,
179
180
181
        trust_remote_code=True,
        max_num_seqs=5,
    )
182
183
184
185
186
187
188
189
190
191
192
193
194
195

    tokenizer = AutoTokenizer.from_pretrained(model_name,
                                              trust_remote_code=True)
    messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
    prompt = tokenizer.apply_chat_template(messages,
                                           tokenize=False,
                                           add_generation_prompt=True)

    # Stop tokens for InternVL
    # models variants may have different stop tokens
    # please refer to the model card for the correct "stop words":
    # https://huggingface.co/OpenGVLab/InternVL2-2B#service
    stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
    stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
196
    return llm, prompt, stop_token_ids
197
198


199
# BLIP-2
200
201
def run_blip2(question, modality):
    assert modality == "image"
202
203
204
205
206

    # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
    # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
    prompt = f"Question: {question} Answer:"
    llm = LLM(model="Salesforce/blip2-opt-2.7b")
207
208
    stop_token_ids = None
    return llm, prompt, stop_token_ids
209
210


211
# Qwen
212
213
def run_qwen_vl(question, modality):
    assert modality == "image"
214
215
216
217
218
219
220
221
222
223
224
225

    llm = LLM(
        model="Qwen/Qwen-VL",
        trust_remote_code=True,
        max_num_seqs=5,
    )

    prompt = f"{question}Picture 1: <img></img>\n"
    stop_token_ids = None
    return llm, prompt, stop_token_ids


226
# Qwen2-VL
227
228
229
def run_qwen2_vl(question, modality):
    assert modality == "image"

230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
    model_name = "Qwen/Qwen2-VL-7B-Instruct"

    llm = LLM(
        model=model_name,
        max_num_seqs=5,
    )

    prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
              "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
              f"{question}<|im_end|>\n"
              "<|im_start|>assistant\n")
    stop_token_ids = None
    return llm, prompt, stop_token_ids


245
246
247
model_example_map = {
    "llava": run_llava,
    "llava-next": run_llava_next,
248
    "llava-next-video": run_llava_next_video,
249
    "llava-onevision": run_llava_onevision,
250
251
252
253
254
    "fuyu": run_fuyu,
    "phi3_v": run_phi3v,
    "paligemma": run_paligemma,
    "chameleon": run_chameleon,
    "minicpmv": run_minicpmv,
255
    "blip-2": run_blip2,
256
    "internvl_chat": run_internvl,
257
    "qwen_vl": run_qwen_vl,
258
    "qwen2_vl": run_qwen2_vl,
259
260
261
}


262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def get_multi_modal_input(args):
    """
    return {
        "data": image or video,
        "question": question,
    }
    """
    if args.modality == "image":
        # Input image and question
        image = ImageAsset("cherry_blossom") \
            .pil_image.convert("RGB")
        img_question = "What is the content of this image?"

        return {
            "data": image,
            "question": img_question,
        }

    if args.modality == "video":
        # Input video and question
        video = VideoAsset(name="sample_demo_1.mp4",
                           num_frames=args.num_frames).np_ndarrays
        vid_question = "Why is this video funny?"

        return {
            "data": video,
            "question": vid_question,
        }

    msg = f"Modality {args.modality} is not supported."
    raise ValueError(msg)


295
296
297
298
299
def main(args):
    model = args.model_type
    if model not in model_example_map:
        raise ValueError(f"Model type {model} is not supported.")

300
301
302
303
304
    modality = args.modality
    mm_input = get_multi_modal_input(args)
    data = mm_input["data"]
    question = mm_input["question"]

305
    llm, prompt, stop_token_ids = model_example_map[model](question, modality)
306
307
308

    # We set temperature to 0.2 so that outputs can be different
    # even when all prompts are identical when running batch inference.
309
310
311
    sampling_params = SamplingParams(temperature=0.2,
                                     max_tokens=64,
                                     stop_token_ids=stop_token_ids)
312
313
314
315
316
317
318

    assert args.num_prompts > 0
    if args.num_prompts == 1:
        # Single inference
        inputs = {
            "prompt": prompt,
            "multi_modal_data": {
319
                modality: data
320
321
322
323
324
325
326
327
            },
        }

    else:
        # Batch inference
        inputs = [{
            "prompt": prompt,
            "multi_modal_data": {
328
                modality: data
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
            },
        } for _ in range(args.num_prompts)]

    outputs = llm.generate(inputs, sampling_params=sampling_params)

    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)


if __name__ == "__main__":
    parser = FlexibleArgumentParser(
        description='Demo on using vLLM for offline inference with '
        'vision language models')
    parser.add_argument('--model-type',
                        '-m',
                        type=str,
                        default="llava",
                        choices=model_example_map.keys(),
                        help='Huggingface "model_type".')
    parser.add_argument('--num-prompts',
                        type=int,
351
                        default=4,
352
                        help='Number of prompts to run.')
353
354
355
    parser.add_argument('--modality',
                        type=str,
                        default="image",
356
                        choices=['image', 'video'],
357
358
359
360
361
                        help='Modality of the input.')
    parser.add_argument('--num-frames',
                        type=int,
                        default=16,
                        help='Number of frames to extract from the video.')
362
363
    args = parser.parse_args()
    main(args)