[PD] Allow customizing reserved tokens to avoid KV cache waste (#6002)

3008db9c · fzyzcjy · GitHub · 357fb2db · 3008db9c
Unverified Commit 3008db9c authored May 05, 2025 by fzyzcjy Committed by GitHub May 05, 2025
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

python/sglang/srt/disaggregation/decode.py python/sglang/srt/disaggregation/decode.py +3 -1

No files found.
--- a/python/sglang/srt/disaggregation/decode.py
+++ b/python/sglang/srt/disaggregation/decode.py
@@ -97,7 +97,9 @@ class DecodePreallocQueue:
        self.tp_size = tp_size
        self.bootstrap_port = bootstrap_port
-        self.num_reserved_decode_tokens = 512
+        self.num_reserved_decode_tokens = int(
+            os.environ.get("SGLANG_NUM_RESERVED_DECODE_TOKENS", "512")
+        )
        # Queue for requests pending pre-allocation
        self.queue: List[DecodeRequest] = []