Skip to content

GitLab

"deploy/operator/vscode:/vscode.git/clone" did not exist on "0f47842ac7037b246ce1eebc064cec261b0767c4"
Unverified Commit 6b432625 authored by ishandhanani's avatar ishandhanani Committed by GitHub
Browse files

fix: set `SGLANG_BLOCK_NONZERO_RANK_CHILDREN` to be no-op (#4527)

parent 36f58e36
Show whitespace changes
Inline Side-by-side
Showing with 8 additions and 2 deletions
components/src/dynamo/sglang/main.py
View file @ 6b432625
...@@ -102,8 +102,11 @@ async def init(runtime: DistributedRuntime, config: Config): ...@@ -102,8 +102,11 @@ async def init(runtime: DistributedRuntime, config: Config):
server_args, dynamo_args = config.server_args, config.dynamo_args server_args, dynamo_args = config.server_args, config.dynamo_args
# Prevent SGLang from blocking on non-leader nodes # Prevent SGLang from blocking on non-leader nodes
# We can switch this to 0 and leverage our own metrics
# after https://github.com/sgl-project/sglang/pull/13686
# is merged in
if server_args.node_rank >= 1: if server_args.node_rank >= 1:
os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "1"
engine = sgl.Engine(server_args=server_args) engine = sgl.Engine(server_args=server_args)
...@@ -188,8 +191,11 @@ async def init_prefill(runtime: DistributedRuntime, config: Config): ...@@ -188,8 +191,11 @@ async def init_prefill(runtime: DistributedRuntime, config: Config):
server_args, dynamo_args = config.server_args, config.dynamo_args server_args, dynamo_args = config.server_args, config.dynamo_args
# Prevent SGLang from blocking on non-leader nodes # Prevent SGLang from blocking on non-leader nodes
# We can switch this to 0 and leverage our own metrics
# after https://github.com/sgl-project/sglang/pull/13686
# is merged in
if server_args.node_rank >= 1: if server_args.node_rank >= 1:
os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "1"
engine = sgl.Engine(server_args=server_args) engine = sgl.Engine(server_args=server_args)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment