[Feature] Warm up readonly multimodal processor during renderer startup (#40797)

Signed-off-by: Chenguang ZHENG <645327136@qq.com>
Co-authored-by: OpenAI Codex <codex@openai.com>

tests/entrypoints/openai/chat_completion/test_chat_error.py

@@ -164,6 +164,58 @@ async def test_chat_error_non_stream():
         await serving_chat.create_chat_completion(request)
 
 
+@pytest.mark.asyncio
+async def test_openai_chat_keeps_mm_cache_for_engine_execution():
+    mock_engine = MagicMock(spec=AsyncLLM)
+    mock_engine.errored = False
+    mock_engine.model_config = MockModelConfig()
+    mock_engine.input_processor = MagicMock()
+    mock_engine.renderer = _build_renderer(mock_engine.model_config)
+
+    serving_chat = _build_serving_chat(mock_engine)
+
+    request = ChatCompletionRequest(
+        model=MODEL_NAME,
+        messages=[{"role": "user", "content": "Test prompt"}],
+    )
+
+    result = await serving_chat.render_chat_request(request)
+
+    assert isinstance(result, tuple)
+    assert (
+        serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
+            "skip_mm_cache"
+        ]
+        is False
+    )
+
+
+@pytest.mark.asyncio
+async def test_renderer_only_chat_request_skips_mm_cache():
+    mock_engine = MagicMock(spec=AsyncLLM)
+    mock_engine.errored = False
+    mock_engine.model_config = MockModelConfig()
+    mock_engine.input_processor = MagicMock()
+    mock_engine.renderer = _build_renderer(mock_engine.model_config)
+
+    serving_chat = _build_serving_chat(mock_engine)
+
+    request = ChatCompletionRequest(
+        model=MODEL_NAME,
+        messages=[{"role": "user", "content": "Test prompt"}],
+    )
+
+    result = await serving_chat.openai_serving_render.render_chat_request(request)
+
+    assert result.token_ids == [1, 2, 3]
+    assert (
+        serving_chat.openai_serving_render.preprocess_chat.call_args.kwargs[
+            "skip_mm_cache"
+        ]
+        is True
+    )
+
+
 @pytest.mark.asyncio
 async def test_chat_error_stream():
     """test finish_reason='error' returns 500 InternalServerError (streaming)"""

tests/entrypoints/openai/completion/test_completion_error.py

@@ -3,7 +3,7 @@
 
 from dataclasses import dataclass, field
 from typing import Any
-from unittest.mock import MagicMock
+from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
@@ -148,6 +148,66 @@ async def test_completion_error_non_stream():
         await serving_completion.create_completion(request)
 
 
+@pytest.mark.asyncio
+async def test_openai_completion_keeps_mm_cache_for_engine_execution():
+    mock_engine = MagicMock(spec=AsyncLLM)
+    mock_engine.errored = False
+    mock_engine.model_config = MockModelConfig()
+    mock_engine.input_processor = MagicMock()
+    mock_engine.renderer = _build_renderer(mock_engine.model_config)
+
+    serving_completion = _build_serving_completion(mock_engine)
+    serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
+        return_value=[{"prompt_token_ids": [1, 2, 3]}]
+    )
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt="Test prompt",
+    )
+
+    result = await serving_completion.render_completion_request(request)
+
+    assert isinstance(result, list)
+    assert (
+        serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
+            "skip_mm_cache"
+        ]
+        is False
+    )
+
+
+@pytest.mark.asyncio
+async def test_renderer_only_completion_request_skips_mm_cache():
+    mock_engine = MagicMock(spec=AsyncLLM)
+    mock_engine.errored = False
+    mock_engine.model_config = MockModelConfig()
+    mock_engine.input_processor = MagicMock()
+    mock_engine.renderer = _build_renderer(mock_engine.model_config)
+
+    serving_completion = _build_serving_completion(mock_engine)
+    serving_completion.openai_serving_render.preprocess_completion = AsyncMock(
+        return_value=[{"prompt_token_ids": [1, 2, 3]}]
+    )
+
+    request = CompletionRequest(
+        model=MODEL_NAME,
+        prompt="Test prompt",
+    )
+
+    result = await serving_completion.openai_serving_render.render_completion_request(
+        request
+    )
+
+    assert isinstance(result, list)
+    assert (
+        serving_completion.openai_serving_render.preprocess_completion.call_args.kwargs[
+            "skip_mm_cache"
+        ]
+        is True
+    )
+
+
 @pytest.mark.asyncio
 async def test_completion_error_stream():
     """test finish_reason='error' returns 500 InternalServerError (streaming)"""

tests/entrypoints/serve/disagg/test_generate_stream.py

@@ -12,7 +12,10 @@ from vllm.config.multimodal import MultiModalConfig
 from vllm.entrypoints.openai.engine.protocol import StreamOptions
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
+from vllm.entrypoints.serve.disagg.protocol import (
+    GenerateRequest,
+    GenerateResponse,
+)
 from vllm.entrypoints.serve.disagg.serving import ServingTokens
 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 from vllm.logprobs import Logprob
@@ -164,6 +167,36 @@ def _parse_sse_chunks(chunks: list[str]) -> list[Any]:
     return parsed
 
 
+@pytest.mark.asyncio
+async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution():
+    engine = _mock_engine()
+
+    async def mock_generate(*args, **kwargs):
+        yield _make_request_output(
+            "req-1", token_ids=[10], finish_reason="stop", finished=True
+        )
+
+    engine.generate = MagicMock(side_effect=mock_generate)
+    serving = _build_serving_tokens(engine)
+
+    request = GenerateRequest(
+        token_ids=[1, 2, 3],
+        sampling_params=SamplingParams(max_tokens=1),
+        model=MODEL_NAME,
+        stream=False,
+    )
+
+    response = await serving.serve_tokens(request)
+
+    assert isinstance(response, GenerateResponse)
+    assert (
+        serving.openai_serving_render.preprocess_completion.call_args.kwargs[
+            "skip_mm_cache"
+        ]
+        is True
+    )
+
+
 @pytest.mark.asyncio
 async def test_stream_basic():
     """Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""

tests/entrypoints/serve/tokenize/test_serving_tokenization.py

+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from dataclasses import dataclass, field
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from vllm.config.multimodal import MultiModalConfig
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
+from vllm.entrypoints.serve.tokenize.protocol import (
+    TokenizeChatRequest,
+    TokenizeCompletionRequest,
+)
+from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
+from vllm.v1.engine.async_llm import AsyncLLM
+
+MODEL_NAME = "openai-community/gpt2"
+BASE_MODEL_PATHS = [
+    BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME),
+]
+
+
+@dataclass
+class MockHFConfig:
+    model_type: str = "any"
+
+
+@dataclass
+class MockModelConfig:
+    task = "generate"
+    runner_type = "generate"
+    model = MODEL_NAME
+    tokenizer = MODEL_NAME
+    trust_remote_code = False
+    tokenizer_mode = "auto"
+    max_model_len = 100
+    tokenizer_revision = None
+    multimodal_config = MultiModalConfig()
+    hf_config = MockHFConfig()
+    hf_text_config = MockHFConfig()
+    logits_processors: list[str] | None = None
+    diff_sampling_param: dict | None = None
+    allowed_local_media_path: str = ""
+    allowed_media_domains: list[str] | None = None
+    encoder_config = None
+    generation_config: str = "auto"
+    media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict)
+    skip_tokenizer_init = False
+    is_encoder_decoder: bool = False
+    is_multimodal_model: bool = False
+    renderer_num_workers: int = 1
+
+    def get_diff_sampling_param(self):
+        return self.diff_sampling_param or {}
+
+
+def _build_serving_tokenization(engine: AsyncLLM) -> OpenAIServingTokenization:
+    models = OpenAIServingModels(
+        engine_client=engine,
+        base_model_paths=BASE_MODEL_PATHS,
+    )
+    serving_render = OpenAIServingRender(
+        model_config=engine.model_config,
+        renderer=engine.renderer,
+        model_registry=models.registry,
+        request_logger=None,
+        chat_template=None,
+        chat_template_content_format="auto",
+    )
+    return OpenAIServingTokenization(
+        engine,
+        models,
+        openai_serving_render=serving_render,
+        request_logger=None,
+        chat_template=None,
+        chat_template_content_format="auto",
+    )
+
+
+@pytest.mark.asyncio
+async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path():
+    mock_engine = MagicMock(spec=AsyncLLM)
+    mock_engine.errored = False
+    mock_engine.model_config = MockModelConfig()
+    mock_engine.input_processor = MagicMock()
+    mock_engine.renderer = MagicMock()
+
+    serving = _build_serving_tokenization(mock_engine)
+    serving.openai_serving_render.preprocess_chat = AsyncMock(
+        return_value=(
+            [{"role": "user", "content": "Test"}],
+            [{"prompt_token_ids": [1, 2, 3]}],
+        )
+    )
+
+    request = TokenizeChatRequest(
+        model=MODEL_NAME,
+        messages=[{"role": "user", "content": "Test prompt"}],
+    )
+
+    response = await serving.create_tokenize(request, MagicMock(headers={}))
+
+    assert response.tokens == [1, 2, 3]
+    assert (
+        serving.openai_serving_render.preprocess_chat.call_args.kwargs["skip_mm_cache"]
+        is True
+    )
+
+
+@pytest.mark.asyncio
+async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path():
+    mock_engine = MagicMock(spec=AsyncLLM)
+    mock_engine.errored = False
+    mock_engine.model_config = MockModelConfig()
+    mock_engine.input_processor = MagicMock()
+    mock_engine.renderer = MagicMock()
+
+    serving = _build_serving_tokenization(mock_engine)
+    serving.openai_serving_render.preprocess_completion = AsyncMock(
+        return_value=[{"prompt_token_ids": [1, 2, 3]}]
+    )
+
+    request = TokenizeCompletionRequest(
+        model=MODEL_NAME,
+        prompt="Test prompt",
+    )
+
+    response = await serving.create_tokenize(request, MagicMock(headers={}))
+
+    assert response.tokens == [1, 2, 3]
+    assert (
+        serving.openai_serving_render.preprocess_completion.call_args.kwargs[
+            "skip_mm_cache"
+        ]
+        is True
+    )

tests/renderers/test_warmup.py

@@ -34,6 +34,14 @@ def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock:
     mm_processor = MagicMock()
     mm_processor.info.allowed_mm_limits = mm_limits
     renderer.mm_processor = mm_processor
+    renderer._readonly_mm_processor = None
+    renderer._warmup_mm_processor = BaseRenderer._warmup_mm_processor.__get__(
+        renderer, BaseRenderer
+    )
+    renderer._clear_processor_cache = BaseRenderer._clear_processor_cache
+    renderer.clear_mm_cache = MagicMock()
+    renderer.model_config.max_model_len = 128
+    renderer.model_config.get_multimodal_config.return_value.limit_per_prompt = {}
 
     return renderer
 
@@ -109,3 +117,19 @@ class TestMmWarmupSkippedWhenNoProcessor:
         BaseRenderer.warmup(renderer, ChatParams())
 
         renderer.model_config.get_multimodal_config.assert_not_called()
+
+
+class TestReadonlyMmWarmup:
+    """Readonly MM processor warmup must mirror the render path behavior."""
+
+    def test_readonly_processor_apply_called_and_cache_cleared(self):
+        renderer = _make_renderer_mock({"image": 1})
+        readonly_mm_processor = MagicMock()
+        readonly_mm_processor.info.allowed_mm_limits = {"image": 1}
+        renderer._readonly_mm_processor = readonly_mm_processor
+
+        with patch("vllm.multimodal.processing.TimingContext", autospec=True):
+            BaseRenderer.warmup(renderer, ChatParams())
+
+        readonly_mm_processor.apply.assert_called_once()
+        readonly_mm_processor.cache.clear_cache.assert_called_once()

vllm/entrypoints/serve/render/serving.py

@@ -136,7 +136,7 @@ class OpenAIServingRender:
                 "Beam search is not supported by the render endpoint"
             )
 
-        result = await self.render_chat(request)
+        result = await self.render_chat(request, skip_mm_cache=True)
         if isinstance(result, ErrorResponse):
             return result
 
@@ -184,6 +184,8 @@ class OpenAIServingRender:
     async def render_chat(
         self,
         request: ChatCompletionRequest,
+        *,
+        skip_mm_cache: bool = False,
     ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
         """Core preprocessing logic for chat requests (no model/engine check).
 
@@ -252,7 +254,7 @@ class OpenAIServingRender:
                 default_template_kwargs=self.default_chat_template_kwargs,
                 tool_dicts=tool_dicts,
                 tool_parser=tool_parser,
-                skip_mm_cache=True,
+                skip_mm_cache=skip_mm_cache,
                 reasoning_parser=self.reasoning_parser,
             )
         else:
@@ -276,7 +278,7 @@ class OpenAIServingRender:
         error_check_ret = await self._check_model(request)
         if error_check_ret is not None:
             return error_check_ret
-        result = await self.render_completion(request)
+        result = await self.render_completion(request, skip_mm_cache=True)
         if isinstance(result, ErrorResponse):
             return result
         generate_requests: list[GenerateRequest] = []
@@ -322,6 +324,8 @@ class OpenAIServingRender:
     async def render_completion(
         self,
         request: CompletionRequest,
+        *,
+        skip_mm_cache: bool = False,
     ) -> list[EngineInput] | ErrorResponse:
         """Core preprocessing logic for completion requests (no model/engine check).
 
@@ -344,7 +348,7 @@ class OpenAIServingRender:
             request,
             prompt_input=request.prompt,
             prompt_embeds=request.prompt_embeds,
-            skip_mm_cache=True,
+            skip_mm_cache=skip_mm_cache,
         )
 
         return engine_inputs

vllm/renderers/base.py

@@ -198,6 +198,40 @@ class BaseRenderer(ABC, Generic[_T]):
         if self._mm_cache_stats is not None:
             self._mm_cache_stats.reset = True
 
+    @staticmethod
+    def _clear_processor_cache(
+        processor: "BaseMultiModalProcessor | None",
+    ) -> None:
+        if processor is None:
+            return
+
+        processor_cache = processor.cache
+        if processor_cache is not None:
+            processor_cache.clear_cache()
+
+    def _warmup_mm_processor(
+        self,
+        processor: "BaseMultiModalProcessor",
+        *,
+        log_prefix: str,
+    ) -> None:
+        from vllm.multimodal.processing import TimingContext
+
+        model_config = self.model_config
+        mm_config = model_config.get_multimodal_config()
+        mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
+
+        start_time = time.perf_counter()
+        processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
+            seq_len=model_config.max_model_len,
+            mm_counts=dict.fromkeys(mm_limits, 1),
+            mm_options=mm_config.limit_per_prompt,
+        )
+        _ = processor.apply(processor_inputs, timing_ctx=TimingContext(enabled=False))
+
+        elapsed = time.perf_counter() - start_time
+        logger.info("%s warmup completed in %.3fs", log_prefix, elapsed)
+
     def warmup(self, chat_params: ChatParams) -> None:
         """
         Warm up this renderer to avoid first-request latency.
@@ -221,35 +255,29 @@ class BaseRenderer(ABC, Generic[_T]):
             logger.warning("Chat template warmup failed", exc_info=True)
 
         if self.mm_processor:
-            from vllm.multimodal.processing import TimingContext
-
-            model_config = self.model_config
-            mm_config = model_config.get_multimodal_config()
-            processor = self.mm_processor
-            mm_limits = {
-                k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0
-            }
-
             try:
                 logger.debug("Warming up multi-modal processing...")
-                start_time = time.perf_counter()
-
-                processor_inputs = processor.dummy_inputs.get_dummy_processor_inputs(
-                    seq_len=model_config.max_model_len,
-                    mm_counts=dict.fromkeys(mm_limits, 1),
-                    mm_options=mm_config.limit_per_prompt,
-                )
-                _ = processor.apply(
-                    processor_inputs, timing_ctx=TimingContext(enabled=False)
+                self._warmup_mm_processor(
+                    self.mm_processor,
+                    log_prefix="Multi-modal",
                 )
-
-                elapsed = time.perf_counter() - start_time
-                logger.info("Multi-modal warmup completed in %.3fs", elapsed)
             except Exception:
                 logger.warning("Multi-modal warmup failed")
             finally:
                 self.clear_mm_cache()
 
+        if self._readonly_mm_processor is not None:
+            try:
+                logger.debug("Warming up readonly multi-modal processing...")
+                self._warmup_mm_processor(
+                    self._readonly_mm_processor,
+                    log_prefix="Readonly multi-modal",
+                )
+            except Exception:
+                logger.warning("Readonly multi-modal warmup failed")
+            finally:
+                self._clear_processor_cache(self._readonly_mm_processor)
+
     async def clear_mm_cache_async(self) -> None:
         """Serialize clear_mm_cache through the shared executor to avoid
         races with concurrent process_inputs on the mm_processor_cache."""