"lib/vscode:/vscode.git/clone" did not exist on "6bfb41dee42f4eda88162ea146ba7f4541d6865c"
Unverified Commit 960dc896 authored by Biswa Panda's avatar Biswa Panda Committed by GitHub
Browse files

fix: replace hard coded dynamo namespace with env var (#3048)


Signed-off-by: default avatarBiswa Panda <biswa.panda@gmail.com>
parent f77511ff
......@@ -5,6 +5,7 @@
import argparse
import logging
import os
import sys
from typing import Optional
......@@ -17,7 +18,8 @@ from dynamo.runtime.logging import configure_dynamo_logging
from . import __version__
DEFAULT_ENDPOINT = "dyn://dynamo.backend.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.backend.generate"
configure_dynamo_logging()
......
......@@ -20,7 +20,9 @@ from dynamo.sglang import __version__
configure_dynamo_logging()
DEFAULT_ENDPOINT = "dyn://dynamo.backend.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.backend.generate"
DYNAMO_ARGS: Dict[str, Dict[str, Any]] = {
"endpoint": {
"flags": ["--endpoint"],
......
......@@ -176,8 +176,9 @@ class VllmEncodeWorker:
@classmethod
def parse_args(cls) -> Tuple[argparse.Namespace, Config]:
DEFAULT_ENDPOINT = "dyn://dynamo.encoder.generate"
DEFAULT_DOWNSTREAM_ENDPOINT = "dyn://dynamo.llm.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.encoder.generate"
DEFAULT_DOWNSTREAM_ENDPOINT = f"dyn://{DYN_NAMESPACE}.llm.generate"
parser = FlexibleArgumentParser(
description="vLLM based encoder for Dynamo LLM."
......
......@@ -63,8 +63,9 @@ class Processor(ProcessMixIn):
@classmethod
def parse_args(cls) -> Tuple[argparse.Namespace, Config]:
DEFAULT_ENDPOINT = "dyn://dynamo.processor.generate"
DEFAULT_DOWNSTREAM_ENDPOINT = "dyn://dynamo.encoder.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.processor.generate"
DEFAULT_DOWNSTREAM_ENDPOINT = f"dyn://{DYN_NAMESPACE}.encoder.generate"
parser = FlexibleArgumentParser(
description="vLLM based processor for Dynamo LLM."
......
......@@ -217,8 +217,9 @@ class VllmEncodeWorker:
@classmethod
def parse_args(cls) -> Tuple[argparse.Namespace, Config]:
DEFAULT_ENDPOINT = "dyn://dynamo.encoder.generate"
DEFAULT_DOWNSTREAM_ENDPOINT = "dyn://dynamo.llm.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.encoder.generate"
DEFAULT_DOWNSTREAM_ENDPOINT = f"dyn://{DYN_NAMESPACE}.llm.generate"
parser = FlexibleArgumentParser(
description="vLLM based encoder for Dynamo LLM."
......
......@@ -84,17 +84,23 @@ class VllmBaseWorker:
# use endpoint_overwrite to set the default endpoint based on worker type
def endpoint_overwrite(args):
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
# default endpoint for this worker
if args.worker_type == "prefill":
args.endpoint = args.endpoint or "dyn://dynamo.llm.generate"
args.endpoint = args.endpoint or f"dyn://{DYN_NAMESPACE}.llm.generate"
elif args.worker_type == "decode":
args.endpoint = args.endpoint or "dyn://dynamo.decoder.generate"
args.endpoint = (
args.endpoint or f"dyn://{DYN_NAMESPACE}.decoder.generate"
)
elif args.worker_type == "encode_prefill":
args.endpoint = args.endpoint or "dyn://dynamo.encoder.generate"
args.endpoint = (
args.endpoint or f"dyn://{DYN_NAMESPACE}.encoder.generate"
)
# set downstream endpoint for disaggregated workers
if args.enable_disagg:
args.downstream_endpoint = (
args.downstream_endpoint or "dyn://dynamo.decoder.generate"
args.downstream_endpoint
or f"dyn://{DYN_NAMESPACE}.decoder.generate"
)
return args
......
......@@ -29,7 +29,8 @@ from vllm.engine.arg_utils import AsyncEngineArgs
logger = logging.getLogger(__name__)
DEFAULT_ENDPOINT = "dyn://dynamo.backend.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.backend.generate"
class Config:
......
......@@ -21,6 +21,7 @@
import argparse
import asyncio
import os
import sys
import sglang
......@@ -30,7 +31,8 @@ from sglang.srt.server_args import ServerArgs
from dynamo.llm import ModelInput, ModelType, register_llm
from dynamo.runtime import DistributedRuntime, dynamo_worker
DEFAULT_ENDPOINT = "dyn://dynamo.backend.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.backend.generate"
DEFAULT_MODEL = "Qwen/Qwen3-0.6B"
DEFAULT_TEMPERATURE = 0.7
......
......@@ -13,6 +13,7 @@
import argparse
import asyncio
import os
import sys
import sglang
......@@ -21,7 +22,8 @@ from sglang.srt.server_args import ServerArgs
from dynamo.runtime import DistributedRuntime, dynamo_worker
DEFAULT_ENDPOINT = "dyn://dynamo.backend.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.backend.generate"
DEFAULT_MODEL = "Qwen/Qwen3-0.6B"
DEFAULT_TEMPERATURE = 0.7
......
......@@ -22,6 +22,7 @@
import argparse
import asyncio
import os
import sys
import time
......@@ -34,7 +35,8 @@ from sglang.srt.server_args import ServerArgs
from dynamo.llm import ModelInput, ModelType, register_llm
from dynamo.runtime import DistributedRuntime, dynamo_worker
DEFAULT_ENDPOINT = "dyn://dynamo.backend.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.backend.generate"
DEFAULT_MODEL = "Qwen/Qwen3-0.6B"
......
......@@ -29,6 +29,7 @@
import argparse
import asyncio
import os
import sys
import uvloop
......@@ -42,7 +43,8 @@ from vllm.inputs import TokensPrompt
from dynamo.llm import ModelInput, ModelType, register_llm
from dynamo.runtime import DistributedRuntime, dynamo_worker
DEFAULT_ENDPOINT = "dyn://dynamo.backend.generate"
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.backend.generate"
DEFAULT_MODEL = "Qwen/Qwen3-0.6B"
DEFAULT_TEMPERATURE = 0.7
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment