Unverified Commit 49517f2a authored by Biswa Panda's avatar Biswa Panda Committed by GitHub
Browse files

fix: trtllm example (#909)

parent a0a09df0
...@@ -148,8 +148,8 @@ def main( ...@@ -148,8 +148,8 @@ def main(
from dynamo.runtime.logging import configure_dynamo_logging from dynamo.runtime.logging import configure_dynamo_logging
# Setup signal handlers for graceful shutdown # TODO: completely disable signal handlers in serve_dynamo. It interferes with arbiter shutdown
setup_signal_handlers() # setup_signal_handlers()
run_id = service_name run_id = service_name
dynamo_context["service_name"] = service_name dynamo_context["service_name"] = service_name
......
...@@ -19,6 +19,7 @@ from pathlib import Path ...@@ -19,6 +19,7 @@ from pathlib import Path
from components.processor import Processor from components.processor import Processor
from components.worker import TensorRTLLMWorker from components.worker import TensorRTLLMWorker
from fastapi import FastAPI
from pydantic import BaseModel from pydantic import BaseModel
from dynamo import sdk from dynamo import sdk
...@@ -45,9 +46,14 @@ class FrontendConfig(BaseModel): ...@@ -45,9 +46,14 @@ class FrontendConfig(BaseModel):
@service( @service(
dynamo={
"enabled": True,
"namespace": "dynamo",
},
resources={"cpu": "10", "memory": "20Gi"}, resources={"cpu": "10", "memory": "20Gi"},
workers=1, workers=1,
image=DYNAMO_IMAGE, image=DYNAMO_IMAGE,
app=FastAPI(title="TensorRT LLM Example"),
) )
# todo this should be called ApiServer # todo this should be called ApiServer
class Frontend: class Frontend:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment