Skip to content

GitLab

Unverified Commit c3f1aac8 authored by Ying Sheng's avatar Ying Sheng Committed by GitHub
Browse files

Tune params (#696)

parent d198791f
Hide whitespace changes
Inline Side-by-side
Showing with 5 additions and 5 deletions
python/sglang/srt/server_args.py
View file @ c3f1aac8
...@@ -70,15 +70,15 @@ class ServerArgs: ...@@ -70,15 +70,15 @@ class ServerArgs:
self.tokenizer_path = self.model_path self.tokenizer_path = self.model_path
if self.mem_fraction_static is None: if self.mem_fraction_static is None:
if self.tp_size >= 16: if self.tp_size >= 16:
self.mem_fraction_static = 0.74 self.mem_fraction_static = 0.80
elif self.tp_size >= 8: elif self.tp_size >= 8:
self.mem_fraction_static = 0.78 self.mem_fraction_static = 0.84
elif self.tp_size >= 4: elif self.tp_size >= 4:
self.mem_fraction_static = 0.82 self.mem_fraction_static = 0.86
elif self.tp_size >= 2: elif self.tp_size >= 2:
self.mem_fraction_static = 0.85
else:
self.mem_fraction_static = 0.88 self.mem_fraction_static = 0.88
else:
self.mem_fraction_static = 0.89
if isinstance(self.additional_ports, int): if isinstance(self.additional_ports, int):
self.additional_ports = [self.additional_ports] self.additional_ports = [self.additional_ports]
elif self.additional_ports is None: elif self.additional_ports is None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment