fix: Wrap default_multimodal_input_loader in asyncio.to_thread (#5945)

components/src/dynamo/trtllm/encode_helper.py

@@ -298,9 +298,11 @@ class EncodeHelper:
         # for tokenizer loading. `model_type` is needed to retrieve the correct
         # multimodal placeholders and apply model-specific preprocessing.
 
-        # Pass tokenizer to reuse the pre-initialized tokenizer instead of
-        # creating a new one per request
-        inputs = default_multimodal_input_loader(
+        # NOTE: default_multimodal_input_loader downloads images and preprocesses them
+        # synchronously. Wrap in asyncio.to_thread to allow concurrent image loading
+        # across multiple requests, improving throughput at high concurrency.
+        inputs = await asyncio.to_thread(
+            lambda: default_multimodal_input_loader(
                 tokenizer=tokenizer,
                 model_dir=model_dir,
                 model_type=model_type,
@@ -308,6 +310,7 @@ class EncodeHelper:
                 prompts=[text_prompt],
                 media=image_urls[0],
             )
+        )
 
         # NOTE: MultimodalEncoder.generate() is synchronous. Run it off-thread to avoid
         # blocking the encode worker's event loop under concurrency.

components/src/dynamo/trtllm/multimodal_processor.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import logging
 import time
 from io import BytesIO
@@ -211,10 +212,11 @@ class MultimodalRequestProcessor:
             ]
             logging.info(f"Using embedding paths: {embedding_paths}")
 
-        # Process with default_multimodal_input_loader
-        # Pass self.tokenizer to reuse the pre-initialized tokenizer instead of
-        # creating a new one per request
-        processed_inputs = default_multimodal_input_loader(
+        # NOTE: default_multimodal_input_loader downloads images and preprocesses them
+        # synchronously. Wrap in asyncio.to_thread to allow concurrent image loading
+        # across multiple requests, improving throughput at high concurrency.
+        processed_inputs = await asyncio.to_thread(
+            lambda: default_multimodal_input_loader(
                 tokenizer=self.tokenizer,
                 model_dir=self.model_dir,
                 model_type=self.model_type,
@@ -224,6 +226,7 @@ class MultimodalRequestProcessor:
                 device="cuda",
                 **loader_kwargs,
             )
+        )
 
         # Return the first processed input if available
         if processed_inputs: