[Bugfix] Fix startup hang for Granite Speech (#33699)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

vllm/multimodal/budget.py

@@ -54,6 +54,7 @@ class MultiModalBudget:
         self.max_model_len = model_config.max_model_len
         self.max_num_reqs = scheduler_config.max_num_seqs
 
+        with set_default_torch_num_threads():  # Avoid hang during startup
             cache = mm_registry.processor_only_cache_from_config(vllm_config)
             processor = mm_registry.create_processor(model_config, cache=cache)
 
@@ -64,7 +65,6 @@ class MultiModalBudget:
                 modality for modality, limit in mm_limits.items() if limit > 0
             }
 
-        with set_default_torch_num_threads():  # Avoid hang during startup
             all_mm_max_toks_per_item = get_mm_max_toks_per_item(
                 model_config,
                 mm_registry,