[Fix] Fix NaN issues by fixing the cuda graph padding values for flashinfer (#1779)

0089c4bc · Lianmin Zheng · GitHub · 72e7b57a · 0089c4bc
Unverified Commit 0089c4bc authored Oct 24, 2024 by Lianmin Zheng Committed by GitHub Oct 24, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

python/sglang/srt/model_executor/cuda_graph_runner.py python/sglang/srt/model_executor/cuda_graph_runner.py +1 -1

No files found.
--- a/python/sglang/srt/model_executor/cuda_graph_runner.py
+++ b/python/sglang/srt/model_executor/cuda_graph_runner.py
@@ -290,7 +290,7 @@ class CudaGraphRunner:
        index = bisect.bisect_left(self.capture_bs, raw_bs)
        bs = self.capture_bs[index]
        if bs != raw_bs:
-            self.seq_lens.fill_(self.seq_len_fill_value)
+            self.seq_lens.fill_(1)
            self.out_cache_loc.zero_()

        # Common inputs