Skip to content

GitLab

Unverified Commit 360ddbd3 authored by Roger Wang's avatar Roger Wang Committed by GitHub
Browse files

[Misc] Update Pixtral example (#8431)

parent a480939e
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 2 deletions
examples/offline_inference_pixtral.py
View file @ 360ddbd3
...@@ -11,7 +11,7 @@ from vllm.sampling_params import SamplingParams ...@@ -11,7 +11,7 @@ from vllm.sampling_params import SamplingParams
# - Server: # - Server:
# #
# ```bash # ```bash
# vllm serve mistralai/Pixtral-12B-2409 --tokenizer_mode mistral --limit_mm_per_prompt 'image=4' --max_num_batched_tokens 16384 # vllm serve mistralai/Pixtral-12B-2409 --tokenizer-mode mistral --limit-mm-per-prompt 'image=4' --max-model-len 16384
# ``` # ```
# #
# - Client: # - Client:
...@@ -45,6 +45,7 @@ def run_simple_demo(): ...@@ -45,6 +45,7 @@ def run_simple_demo():
model_name = "mistralai/Pixtral-12B-2409" model_name = "mistralai/Pixtral-12B-2409"
sampling_params = SamplingParams(max_tokens=8192) sampling_params = SamplingParams(max_tokens=8192)
# Lower max_num_seqs or max_model_len on low-VRAM GPUs.
llm = LLM(model=model_name, tokenizer_mode="mistral") llm = LLM(model=model_name, tokenizer_mode="mistral")
prompt = "Describe this image in one sentence." prompt = "Describe this image in one sentence."
...@@ -83,7 +84,7 @@ def run_advanced_demo(): ...@@ -83,7 +84,7 @@ def run_advanced_demo():
model=model_name, model=model_name,
tokenizer_mode="mistral", tokenizer_mode="mistral",
limit_mm_per_prompt={"image": max_img_per_msg}, limit_mm_per_prompt={"image": max_img_per_msg},
max_num_batched_tokens=max_img_per_msg * max_tokens_per_img, max_model_len=max_img_per_msg * max_tokens_per_img,
) )
prompt = "Describe the following image." prompt = "Describe the following image."
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment