Unverified Commit 905c920b authored by Biswa Panda's avatar Biswa Panda Committed by GitHub
Browse files

fix: dyn namespace scoping for trtllm (#2970)


Signed-off-by: default avatarBiswa Panda <biswa.panda@gmail.com>
parent ae38bd43
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
import argparse import argparse
import os
from typing import Optional from typing import Optional
from tensorrt_llm.llmapi import BuildConfig from tensorrt_llm.llmapi import BuildConfig
...@@ -13,11 +14,13 @@ from dynamo.trtllm.request_handlers.handler_base import ( ...@@ -13,11 +14,13 @@ from dynamo.trtllm.request_handlers.handler_base import (
DisaggregationStrategy, DisaggregationStrategy,
) )
DYN_NAMESPACE = os.environ.get("DYN_NAMESPACE", "dynamo")
# Default endpoint for the next worker. # Default endpoint for the next worker.
DEFAULT_ENDPOINT = "dyn://dynamo.tensorrt_llm.generate" DEFAULT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.tensorrt_llm.generate"
DEFAULT_MODEL_PATH = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" DEFAULT_MODEL_PATH = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
DEFAULT_NEXT_ENDPOINT = "dyn://dynamo.tensorrt_llm_next.generate" DEFAULT_NEXT_ENDPOINT = f"dyn://{DYN_NAMESPACE}.tensorrt_llm_next.generate"
DEFAULT_ENCODE_ENDPOINT = "dyn://dynamo.tensorrt_llm_encode.generate" DEFAULT_ENCODE_ENDPOINT = f"dyn://{DYN_NAMESPACE}.tensorrt_llm_encode.generate"
DEFAULT_DISAGGREGATION_STRATEGY = DisaggregationStrategy.DECODE_FIRST DEFAULT_DISAGGREGATION_STRATEGY = DisaggregationStrategy.DECODE_FIRST
DEFAULT_DISAGGREGATION_MODE = DisaggregationMode.AGGREGATED DEFAULT_DISAGGREGATION_MODE = DisaggregationMode.AGGREGATED
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment