Increase interpreter parallelism (#46)

2b079f89 · Lianmin Zheng · GitHub · 05b4c398 · 2b079f89
Unverified Commit 2b079f89 authored Jan 18, 2024 by Lianmin Zheng Committed by GitHub Jan 18, 2024
Show whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

python/sglang/lang/interpreter.py python/sglang/lang/interpreter.py +1 -1

No files found.
--- a/python/sglang/lang/interpreter.py
+++ b/python/sglang/lang/interpreter.py
@@ -80,7 +80,7 @@ def run_program_batch(

    # Run all programs
    if num_threads == "auto":
-        num_threads = multiprocessing.cpu_count()
+        num_threads = min(64, multiprocessing.cpu_count() * 8)
    num_threads = min(num_threads, len(batch_arguments))

    if num_threads == 1: