perf: reduce ttft and itl with stream_interval 1 (#658)

ac971ff6 · zhyncs · GitHub · e1792cca · ac971ff6
Unverified Commit ac971ff6 authored Jul 20, 2024 by zhyncs Committed by GitHub Jul 19, 2024
Show whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

python/sglang/srt/server_args.py python/sglang/srt/server_args.py +1 -1

No files found.
--- a/python/sglang/srt/server_args.py
+++ b/python/sglang/srt/server_args.py
@@ -33,7 +33,7 @@ class ServerArgs:

    # Other runtime options
    tp_size: int = 1
-    stream_interval: int = 8
+    stream_interval: int = 1
    random_seed: Optional[int] = None

    # Logging