Update batch size limitation of dsv3_router_gemm kernel to 16 (#8051)

e7e5a305 · Baizhou Zhang · GitHub · dd7ca006 · e7e5a305
Unverified Commit e7e5a305 authored Jul 31, 2025 by Baizhou Zhang Committed by GitHub Aug 01, 2025
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 2 deletions

python/sglang/srt/models/deepseek_v2.py python/sglang/srt/models/deepseek_v2.py +1 -2

No files found.
--- a/python/sglang/srt/models/deepseek_v2.py
+++ b/python/sglang/srt/models/deepseek_v2.py
@@ -252,8 +252,7 @@ class MoEGate(nn.Module):
        # NOTE: For some unknown reason, router_gemm seems degrade accept length.
        if (
            _is_cuda
-            and not self.is_nextn
-            and hidden_states.shape[0] < 4
+            and hidden_states.shape[0] <= 16
            and hidden_states.shape[1] == 7168
            and self.weight.shape[0] == 256
            and _device_sm >= 90