set DEFAULT_MAX_NUM_BATCHED_TOKENS=10240

98a008db · zhuwenwen · 2c8a16d6 · 98a008db
Commit 98a008db authored Nov 09, 2025 by zhuwenwen
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

vllm/utils/__init__.py vllm/utils/__init__.py +1 -1

No files found.
--- a/vllm/utils/__init__.py
+++ b/vllm/utils/__init__.py
@@ -81,7 +81,7 @@ logger = init_logger(__name__)

 # This value is chosen to have a balance between ITL and TTFT. Note it is
 # not optimized for throughput.
-DEFAULT_MAX_NUM_BATCHED_TOKENS = 2048
+DEFAULT_MAX_NUM_BATCHED_TOKENS = 10240
 POOLING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
 MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS = 5120