`TokenizerManager.context_len` should inherit from `server_args.conte… (#654)

1b7adbb5 · shrirajh · GitHub · a9ef49c1 · 1b7adbb5
Unverified Commit 1b7adbb5 authored Jul 19, 2024 by shrirajh Committed by GitHub Jul 18, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 1 deletion

python/sglang/srt/managers/tokenizer_manager.py python/sglang/srt/managers/tokenizer_manager.py +4 -1

No files found.
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -69,7 +69,10 @@ class TokenizerManager:
            trust_remote_code=server_args.trust_remote_code,
            model_overide_args=model_overide_args,
        )
-        self.context_len = get_context_length(self.hf_config)
+        if server_args.context_length is not None:
+            self.context_len = server_args.context_length
+        else:
+            self.context_len = get_context_length(self.hf_config)

        if is_multimodal_model(self.model_path):
            self.processor = get_processor(