Unverified Commit 7e944246 authored by Kebe's avatar Kebe Committed by GitHub
Browse files

Add memory_saver check (#4986)


Signed-off-by: default avatarKebe <mail@kebe7jun.com>
parent a086a113
...@@ -381,6 +381,18 @@ class ServerArgs: ...@@ -381,6 +381,18 @@ class ServerArgs:
self.disable_radix_cache = True self.disable_radix_cache = True
logger.warning("KV cache is forced as chunk cache for decode server") logger.warning("KV cache is forced as chunk cache for decode server")
if self.enable_memory_saver:
try:
import torch_memory_saver
except ImportError:
logger.warning(
"enable_memory_saver is enabled, but "
"torch-memory-saver is not installed. Please install it "
"via `pip3 uninstall torch-memory-saver`. "
"For normal operation, it will be disabled."
)
raise
os.environ["SGLANG_ENABLE_TORCH_COMPILE"] = ( os.environ["SGLANG_ENABLE_TORCH_COMPILE"] = (
"1" if self.enable_torch_compile else "0" "1" if self.enable_torch_compile else "0"
) )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment