Fix max_new_tokens for limited memory

fa7a696d · Lianmin Zheng · bef0b359 · fa7a696d
Commit fa7a696d authored Jan 24, 2024 by Lianmin Zheng
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 0 deletions

python/sglang/srt/managers/router/model_rpc.py python/sglang/srt/managers/router/model_rpc.py +1 -0

No files found.
--- a/python/sglang/srt/managers/router/model_rpc.py
+++ b/python/sglang/srt/managers/router/model_rpc.py
@@ -229,6 +229,7 @@ class ModelRpcServer(rpyc.Service):
        req.sampling_params.max_new_tokens = min(
            req.sampling_params.max_new_tokens,
            self.model_config.context_len - 1 - len(req.input_ids),
+            self.max_total_num_token - 128 - len(req.input_ids),
        )
        self.forward_queue.append(req)