Address performance regression: disable multiple streams on ROCm (#6412)

6317c5c6 · HAI · GitHub · cba1cdbc · 6317c5c6
Unverified Commit 6317c5c6 authored May 19, 2025 by HAI Committed by GitHub May 19, 2025
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 1 deletion

python/sglang/srt/models/deepseek_v2.py python/sglang/srt/models/deepseek_v2.py +2 -1

No files found.
--- a/python/sglang/srt/models/deepseek_v2.py
+++ b/python/sglang/srt/models/deepseek_v2.py
@@ -1315,7 +1315,8 @@ class DeepseekV2Model(nn.Module):
            config.hidden_size,
            enable_tp=not global_server_args_dict["enable_dp_attention"],
        )
-        self.alt_stream = torch.cuda.Stream()
+        # TODO(haishaw): multi-stream performance on ROCm
+        self.alt_stream = None if _is_hip else torch.cuda.Stream()
        self.layers = nn.ModuleList(
            [
                DeepseekV2DecoderLayer(