set default_max_num_batched_tokens = 10240

6d25880c · zhuwenwen · 1dc70839 · 6d25880c
Commit 6d25880c authored Nov 12, 2025 by zhuwenwen
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

vllm/engine/arg_utils.py vllm/engine/arg_utils.py +1 -1

No files found.
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -1664,7 +1664,7 @@ class EngineArgs:
            # TODO(woosuk): Tune the default values for other hardware.
            default_max_num_batched_tokens = {
                UsageContext.LLM_CLASS: 8192,
-                UsageContext.OPENAI_API_SERVER: 2048,
+                UsageContext.OPENAI_API_SERVER: 10240,
            }
            default_max_num_seqs = {
                UsageContext.LLM_CLASS: 256,