Hint users when weight update timeouts (#6570)

5ccf8fe1 · fzyzcjy · GitHub · 3f23d8cd · 5ccf8fe1
Unverified Commit 5ccf8fe1 authored May 26, 2025 by fzyzcjy Committed by GitHub May 25, 2025
Hide whitespace changes
Inline Side-by-side

Showing with 8 additions and 2 deletions

python/sglang/srt/model_executor/expert_location_updater.py python/sglang/srt/model_executor/expert_location_updater.py +8 -2

No files found.
--- a/python/sglang/srt/model_executor/expert_location_updater.py
+++ b/python/sglang/srt/model_executor/expert_location_updater.py
@@ -338,8 +338,14 @@ def update_expert_weights_single_layer(
            return
        reqs = torch.distributed.batch_isend_irecv(p2p_ops)
-        for req in reqs:
+        try:
-            req.wait()
+            for req in reqs:
+                req.wait(timeout=30)
+        except RuntimeError:
+            logger.error(
+                f"Context: {rank=} {old_physical_to_logical_map=} {new_physical_to_logical_map=} {num_local_physical_experts=} {num_gpu_per_node=}"
+            )
+            raise
    def _execute_buffer2weight_copies(buffer2weight_copy_infos):
        for (