"vllm/vscode:/vscode.git/clone" did not exist on "37c9859fab60bbc346be20a662387479eb0760de"
phi3v_example.py 1.11 KB
Newer Older
1
from vllm import LLM, SamplingParams
2
from vllm.assets.image import ImageAsset
3

4
5
6

def run_phi3v():
    model_path = "microsoft/Phi-3-vision-128k-instruct"
7

8
9
    # Note: The default setting of max_num_seqs (256) and
    # max_model_len (128k) for this model may cause OOM.
10
11
    # You may lower either to run this example on lower-end GPUs.

12
13
    # In this example, we override max_num_seqs to 5 while
    # keeping the original context length of 128k.
14
15
16
    llm = LLM(
        model=model_path,
        trust_remote_code=True,
17
        max_num_seqs=5,
18
19
    )

20
    image = ImageAsset("cherry_blossom").pil_image
21
22
23
24
25

    # single-image prompt
    prompt = "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n"  # noqa: E501
    sampling_params = SamplingParams(temperature=0, max_tokens=64)

26
27
28
    outputs = llm.generate(
        {
            "prompt": prompt,
29
30
31
            "multi_modal_data": {
                "image": image
            },
32
33
        },
        sampling_params=sampling_params)
34
35
36
37
38
39
40
    for o in outputs:
        generated_text = o.outputs[0].text
        print(generated_text)


if __name__ == "__main__":
    run_phi3v()