Skip to content

GitLab

"examples/tensorrt_llm/configs/disagg_router.yaml" did not exist on "b8120504735bcc6f981fdddcce044026bd155e0e"
Commit 8621d914 authored by Biswa Panda's avatar Biswa Panda Committed by GitHub
Browse files

feat: dynamo deploy hello world example to k8s (#205)

parent 988378ab
Expand all Show whitespace changes
Inline Side-by-side
Showing with 998 additions and 249 deletions
deploy/dynamo/operator/internal/controller/dynamonimdeployment_controller.go
View file @ 8621d914
......@@ -1163,7 +1163,7 @@ func (r *DynamoNimDeploymentReconciler) createOrUpdateVirtualService(ctx context
Route: []*istioNetworking.HTTPRouteDestination{
{
Destination: &istioNetworking.Destination{
Host: fmt.Sprintf("%s.yatai.svc.cluster.local", dynamoNimDeployment.Name),
Host: dynamoNimDeployment.Name,
Port: &istioNetworking.PortSelector{
Number: 3000,
},
......@@ -1186,6 +1186,11 @@ func (r *DynamoNimDeploymentReconciler) createOrUpdateVirtualService(ctx context
vsEnabled := dynamoNimDeployment.Spec.Ingress.Enabled && dynamoNimDeployment.Spec.Ingress.UseVirtualService != nil && *dynamoNimDeployment.Spec.Ingress.UseVirtualService
if err := ctrl.SetControllerReference(dynamoNimDeployment, vs, r.Scheme); err != nil {
log.Error(err, "Failed to set controller reference for the VirtualService")
return false, err
}
if err != nil {
if vsEnabled {
log.Info("VirtualService not found, creating new one")
......@@ -1209,7 +1214,7 @@ func (r *DynamoNimDeploymentReconciler) createOrUpdateVirtualService(ctx context
}
log.Info("VirtualService found, updating", "OldVirtualService", oldVS)
vs.ObjectMeta.ResourceVersion = oldVS.ObjectMeta.ResourceVersion
if err := r.Update(ctx, vs); err != nil {
log.Error(err, "Failed to update VirtualService")
return false, err
......@@ -1764,34 +1769,12 @@ monitoring.options.insecure=true`
// do nothing
}
livenessProbe := &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 20,
FailureThreshold: 6,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/livez",
Port: intstr.FromString(commonconsts.BentoContainerPortName),
},
},
}
var livenessProbe *corev1.Probe
if opt.dynamoNimDeployment.Spec.LivenessProbe != nil {
livenessProbe = opt.dynamoNimDeployment.Spec.LivenessProbe
}
readinessProbe := &corev1.Probe{
InitialDelaySeconds: 5,
TimeoutSeconds: 5,
FailureThreshold: 12,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/readyz",
Port: intstr.FromString(commonconsts.BentoContainerPortName),
},
},
}
var readinessProbe *corev1.Probe
if opt.dynamoNimDeployment.Spec.ReadinessProbe != nil {
readinessProbe = opt.dynamoNimDeployment.Spec.ReadinessProbe
}
......@@ -1803,11 +1786,9 @@ monitoring.options.insecure=true`
args = append(args, "uv", "run", "dynamo", "start")
if opt.dynamoNimDeployment.Spec.ServiceName != "" {
args = append(args, []string{"--service-name", opt.dynamoNimDeployment.Spec.ServiceName}...)
}
if len(opt.dynamoNimDeployment.Spec.ExternalServices) > 0 {
// todo : remove this line when https://github.com/ai-dynamo/dynamo/issues/345 is fixed
enableDependsOption := false
if len(opt.dynamoNimDeployment.Spec.ExternalServices) > 0 && enableDependsOption {
serviceSuffix := fmt.Sprintf("%s.svc.cluster.local:3000", opt.dynamoNimDeployment.Namespace)
keys := make([]string, 0, len(opt.dynamoNimDeployment.Spec.ExternalServices))
......@@ -1823,13 +1804,18 @@ monitoring.options.insecure=true`
if service.DeploymentSelectorKey == "name" {
dependsFlag := fmt.Sprintf("--depends \"%s=http://%s.%s\"", key, service.DeploymentSelectorValue, serviceSuffix)
args = append(args, dependsFlag)
} else if service.DeploymentSelectorKey == "nova" {
dependsFlag := fmt.Sprintf("--depends \"%s=nova://%s\"", key, service.DeploymentSelectorValue)
} else if service.DeploymentSelectorKey == "dynamo" {
dependsFlag := fmt.Sprintf("--depends \"%s=dynamo://%s\"", key, service.DeploymentSelectorValue)
args = append(args, dependsFlag)
} else {
return nil, errors.Errorf("DeploymentSelectorKey '%s' not supported. Only 'name' and 'nova' are supported", service.DeploymentSelectorKey)
return nil, errors.Errorf("DeploymentSelectorKey '%s' not supported. Only 'name' and 'dynamo' are supported", service.DeploymentSelectorKey)
}
}
}
if opt.dynamoNimDeployment.Spec.ServiceName != "" {
args = append(args, []string{"--service-name", opt.dynamoNimDeployment.Spec.ServiceName}...)
args = append(args, "src."+opt.dynamoNimDeployment.Spec.DynamoTag)
}
yataiResources := opt.dynamoNimDeployment.Spec.Resources
......
deploy/dynamo/operator/internal/nim/nim.go
View file @ 8621d914
......@@ -44,7 +44,7 @@ import (
)
// ServiceConfig represents the YAML configuration structure for a service
type NovaConfig struct {
type DynamoConfig struct {
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace"`
Name string `yaml:"name"`
......@@ -67,7 +67,7 @@ type Autoscaling struct {
}
type Config struct {
Nova *NovaConfig `yaml:"nova,omitempty"`
Dynamo *DynamoConfig `yaml:"dynamo,omitempty"`
Resources *Resources `yaml:"resources,omitempty"`
Traffic *Traffic `yaml:"traffic,omitempty"`
Autoscaling *Autoscaling `yaml:"autoscaling,omitempty"`
......@@ -131,7 +131,9 @@ func RetrieveDynamoNimDownloadURL(ctx context.Context, dynamoDeployment *v1alpha
// ServicesConfig represents the top-level YAML structure of a dynamoNim yaml file stored in a dynamoNim tar file
type DynamoNIMConfig struct {
DynamoTag string `yaml:"service"`
Services []ServiceConfig `yaml:"services"`
EntryService string `yaml:"entry_service"`
}
type EventRecorder interface {
......@@ -249,16 +251,24 @@ func GetDynamoNIMConfig(ctx context.Context, dynamoDeployment *v1alpha1.DynamoDe
// generate DynamoNIMDeployment from config
func GenerateDynamoNIMDeployments(parentDynamoDeployment *v1alpha1.DynamoDeployment, config *DynamoNIMConfig) (map[string]*v1alpha1.DynamoNimDeployment, error) {
novaServices := make(map[string]string)
dynamoServices := make(map[string]string)
deployments := make(map[string]*v1alpha1.DynamoNimDeployment)
for _, service := range config.Services {
deployment := &v1alpha1.DynamoNimDeployment{}
deployment.Name = fmt.Sprintf("%s-%s", parentDynamoDeployment.Name, strings.ToLower(service.Name))
deployment.Namespace = parentDynamoDeployment.Namespace
deployment.Spec.DynamoTag = config.DynamoTag
deployment.Spec.DynamoNim = strings.Split(parentDynamoDeployment.Spec.DynamoNim, ":")[0]
deployment.Spec.ServiceName = service.Name
if service.Config.Nova != nil && service.Config.Nova.Enabled {
novaServices[service.Name] = fmt.Sprintf("%s/%s", service.Config.Nova.Name, service.Config.Nova.Namespace)
if service.Config.Dynamo != nil && service.Config.Dynamo.Enabled {
dynamoServices[service.Name] = fmt.Sprintf("%s/%s", service.Config.Dynamo.Name, service.Config.Dynamo.Namespace)
} else {
// dynamo is not enabled
if config.EntryService == service.Name {
// enable virtual service for the entry service
deployment.Spec.Ingress.Enabled = true
deployment.Spec.Ingress.UseVirtualService = &deployment.Spec.Ingress.Enabled
}
}
if service.Config.Resources != nil {
deployment.Spec.Resources = &compounaiCommon.Resources{
......@@ -296,10 +306,10 @@ func GenerateDynamoNIMDeployments(parentDynamoDeployment *v1alpha1.DynamoDeploym
if dependencyDeployment == nil {
return nil, fmt.Errorf("dependency %s not found", dependentServiceName)
}
if novaService, ok := novaServices[dependentServiceName]; ok {
if dynamoService, ok := dynamoServices[dependentServiceName]; ok {
deployment.Spec.ExternalServices[dependentServiceName] = v1alpha1.ExternalService{
DeploymentSelectorKey: "nova",
DeploymentSelectorValue: novaService,
DeploymentSelectorKey: "dynamo",
DeploymentSelectorValue: dynamoService,
}
} else {
deployment.Spec.ExternalServices[dependentServiceName] = v1alpha1.ExternalService{
......
deploy/dynamo/operator/internal/nim/nim_test.go
View file @ 8621d914
......@@ -50,12 +50,13 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
},
config: &DynamoNIMConfig{
DynamoTag: "dynamonim:MyService1",
Services: []ServiceConfig{
{
Name: "service1",
Dependencies: []map[string]string{{"service": "service2"}},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service1",
......@@ -76,7 +77,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Name: "service2",
Dependencies: []map[string]string{},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: false,
},
},
......@@ -92,6 +93,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService1",
ServiceName: "service1",
Resources: &compounaiCommon.Resources{
Requests: &compounaiCommon.ResourceItem{
......@@ -125,8 +127,9 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
ServiceName: "service2",
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService1",
ServiceName: "service2",
},
},
},
......@@ -145,16 +148,13 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
},
config: &DynamoNIMConfig{
DynamoTag: "dynamonim:MyService2",
EntryService: "service1",
Services: []ServiceConfig{
{
Name: "service1",
Dependencies: []map[string]string{{"service": "service2"}},
Config: Config{
Nova: &NovaConfig{
Enabled: true,
Namespace: "default",
Name: "service1",
},
Resources: &Resources{
CPU: "1",
Memory: "1Gi",
......@@ -171,7 +171,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Name: "service2",
Dependencies: []map[string]string{},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service2",
......@@ -189,6 +189,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService2",
ServiceName: "service1",
Resources: &compounaiCommon.Resources{
Requests: &compounaiCommon.ResourceItem{
......@@ -210,10 +211,14 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
ExternalServices: map[string]v1alpha1.ExternalService{
"service2": {
DeploymentSelectorKey: "nova",
DeploymentSelectorKey: "dynamo",
DeploymentSelectorValue: "service2/default",
},
},
Ingress: v1alpha1.IngressSpec{
Enabled: true,
UseVirtualService: &[]bool{true}[0],
},
},
},
"service2": {
......@@ -223,6 +228,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService2",
ServiceName: "service2",
},
},
......@@ -242,12 +248,13 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
},
config: &DynamoNIMConfig{
DynamoTag: "dynamonim:MyService3",
Services: []ServiceConfig{
{
Name: "service1",
Dependencies: []map[string]string{{"service": "service2"}},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service1",
......@@ -268,7 +275,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Name: "service3",
Dependencies: []map[string]string{},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service3",
......
deploy/dynamo/sdk/src/dynamo/sdk/cli/cli.py
View file @ 8621d914
......@@ -24,17 +24,21 @@ def create_bentoml_cli() -> click.Command:
from bentoml._internal.context import server_context
from bentoml_cli.bentos import bento_command
# from bentoml_cli.cloud import cloud_command
# from bentoml_cli.containerize import containerize_command
from bentoml_cli.utils import get_entry_points
from dynamo.sdk.cli.deployment import deployment_command
# from dynamo.sdk.cli.deploy import deploy_command
from dynamo.sdk.cli.run import run_command
from dynamo.sdk.cli.serve import serve_command
# from dynamo.sdk.cli.server import cloud_command
from dynamo.sdk.cli.server import cloud_command
from dynamo.sdk.cli.start import start_command
from dynamo.sdk.cli.utils import DynamoCommandGroup
# from dynamo.sdk.cli.cloud import cloud_command
server_context.service_type = "cli"
CONTEXT_SETTINGS = {"help_option_names": ("-h", "--help")}
......@@ -52,14 +56,15 @@ def create_bentoml_cli() -> click.Command:
"""
# Add top-level CLI commands
# bentoml_cli.add_command(cloud_command)
bentoml_cli.add_command(cloud_command)
bentoml_cli.add_single_command(bento_command, "build")
bentoml_cli.add_subcommands(start_command)
bentoml_cli.add_single_command(bento_command, "get")
bentoml_cli.add_subcommands(serve_command)
bentoml_cli.add_subcommands(run_command)
# bentoml_cli.add_command(containerize_command)
# bentoml_cli.add_command(deploy_command)
# bentoml_cli.add_command(containerize_command)
bentoml_cli.add_command(deployment_command)
# Load commands from extensions
for ep in get_entry_points("bentoml.commands"):
bentoml_cli.add_command(ep.load())
......
deploy/dynamo/sdk/src/dynamo/sdk/cli/server.py
View file @ 8621d914
......@@ -26,17 +26,19 @@ from bentoml._internal.cloud.config import (
CloudClientContext,
)
from bentoml._internal.configuration.containers import BentoMLContainer
from bentoml._internal.utils import add_experimental_docstring
from bentoml._internal.utils.cattr import bentoml_cattr
from bentoml.exceptions import CLIException, CloudRESTApiClientError
@click.group(name="server")
def cloud_command():
def build_cloud_command() -> click.Group:
@click.group(name="server")
@add_experimental_docstring
def cloud_command():
"""Interact with your Dynamo Server"""
@cloud_command.command()
@click.option(
@cloud_command.command()
@click.option(
"--endpoint",
type=click.STRING,
help="Dynamo Server endpoint",
......@@ -45,16 +47,16 @@ def cloud_command():
show_default=True,
show_envvar=True,
required=True,
)
@click.option(
)
@click.option(
"--api-token",
type=click.STRING,
help="Dynamo Server user API token",
envvar="DYNAMO_SERVER_API_KEY",
show_envvar=True,
required=True,
)
def login(endpoint: str, api_token: str) -> None: # type: ignore
)
def login(endpoint: str, api_token: str) -> None: # type: ignore
"""Connect to your Dynamo Server. You can find deployment instructions for this in our docs"""
try:
cloud_rest_client = RestApiClient(endpoint, api_token)
......@@ -72,7 +74,9 @@ def login(endpoint: str, api_token: str) -> None: # type: ignore
cloud_context = BentoMLContainer.cloud_context.get()
ctx = CloudClientContext(
name=cloud_context if cloud_context is not None else current_context_name,
name=cloud_context
if cloud_context is not None
else current_context_name,
endpoint=endpoint,
api_token=api_token,
email=user.email,
......@@ -80,7 +84,7 @@ def login(endpoint: str, api_token: str) -> None: # type: ignore
ctx.save()
rich.print(
f":white_check_mark: Configured BentoCloud credentials (current-context: {ctx.name})"
f":white_check_mark: Configured Dynamo Cloud credentials (current-context: {ctx.name})"
)
rich.print(
f":white_check_mark: Logged in as [blue]{user.email}[/] at [blue]{org.name}[/] organization"
......@@ -97,25 +101,29 @@ def login(endpoint: str, api_token: str) -> None: # type: ignore
file=sys.stderr,
)
@cloud_command.command()
def current_context() -> None: # type: ignore
@cloud_command.command()
def current_context() -> None: # type: ignore
"""Get current cloud context."""
rich.print_json(
data=bentoml_cattr.unstructure(CloudClientConfig.get_config().get_context())
)
@cloud_command.command()
def list_context() -> None: # type: ignore
@cloud_command.command()
def list_context() -> None: # type: ignore
"""List all available context."""
config = CloudClientConfig.get_config()
rich.print_json(data=bentoml_cattr.unstructure([i.name for i in config.contexts]))
rich.print_json(
data=bentoml_cattr.unstructure([i.name for i in config.contexts])
)
@cloud_command.command()
@click.argument("context_name", type=click.STRING)
def update_current_context(context_name: str) -> None: # type: ignore
@cloud_command.command()
@click.argument("context_name", type=click.STRING)
def update_current_context(context_name: str) -> None: # type: ignore
"""Update current context"""
ctx = CloudClientConfig.get_config().set_current_context(context_name)
rich.print(f"Successfully switched to context: {ctx.name}")
return cloud_command
cloud_command = build_cloud_command()
deploy/dynamo/tests/test_deployment.sh 0 → 100755
View file @ 8621d914
#!/bin/bash
#!/bin/bash -e
# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -euo pipefail
export DYNAMO_SEREVR="${DYNAMO_SEREVR:-http://dynamo-server}"
export DYNAMO_IMAGE="${DYNAMO_IMAGE:-dynamo-base:latest}"
export DEPLOYMENT_NAME="${DEPLOYMENT_NAME:-ci-hw}"
cd /workspace/examples/hello_world
# Step.1: Login to dynamo server
dynamo server login --api-token TEST-TOKEN --endpoint $DYNAMO_SEREVR
# Step.2: build a dynamo nim with framework-less base
DYNAMO_TAG=$(dynamo build hello_world:Frontend | grep "Successfully built" | awk -F"\"" '{ print $2 }')
# Step.3: Deploy!
echo $DYNAMO_TAG
dynamo deployment create $DYNAMO_TAG --no-wait -n $DEPLOYMENT_NAME
lib/bindings/python/pyproject.toml
View file @ 8621d914
......@@ -26,7 +26,7 @@ license = { text = "Apache-2.0" }
license-files = ["LICENSE"]
requires-python = ">=3.10"
dependencies = [
"pydantic>=2.10.6",
"pydantic>=2.10.6,<2.11.0",
"uvloop>=0.21.0",
"nats-py>=2.6.0",
]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment