Skip to content

GitLab

Unverified Commit cfc9f9ab authored by Yi Liu's avatar Yi Liu Committed by GitHub
Browse files

Fix gpu mem check on CPU (#6317) 


Signed-off-by: default avataryiliu30 <yi4.liu@intel.com>
parent fb4959b2
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
python/sglang/srt/server_args.py
View file @ cfc9f9ab
...@@ -229,7 +229,7 @@ class ServerArgs: ...@@ -229,7 +229,7 @@ class ServerArgs:
# Set mem fraction static, which depends on the tensor parallelism size # Set mem fraction static, which depends on the tensor parallelism size
if self.mem_fraction_static is None: if self.mem_fraction_static is None:
parallel_size = self.tp_size * self.pp_size parallel_size = self.tp_size * self.pp_size
if gpu_mem <= 81920: if gpu_mem is not None and gpu_mem <= 81920:
if parallel_size >= 16: if parallel_size >= 16:
self.mem_fraction_static = 0.79 self.mem_fraction_static = 0.79
elif parallel_size >= 8: elif parallel_size >= 8:
...@@ -242,7 +242,7 @@ class ServerArgs: ...@@ -242,7 +242,7 @@ class ServerArgs:
self.mem_fraction_static = 0.88 self.mem_fraction_static = 0.88
else: else:
self.mem_fraction_static = 0.88 self.mem_fraction_static = 0.88
if gpu_mem > 96 * 1024: if gpu_mem is not None and gpu_mem > 96 * 1024:
mem_fraction = self.mem_fraction_static mem_fraction = self.mem_fraction_static
self.mem_fraction_static = min( self.mem_fraction_static = min(
mem_fraction + 48 * 1024 * (1 - mem_fraction) / gpu_mem, mem_fraction + 48 * 1024 * (1 - mem_fraction) / gpu_mem,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment