Skip to content

GitLab

Commit 8621d914 authored by Biswa Panda's avatar Biswa Panda Committed by GitHub
Browse files

feat: dynamo deploy hello world example to k8s (#205)

parent 988378ab
Show whitespace changes
Inline Side-by-side
Showing with 998 additions and 249 deletions
deploy/dynamo/operator/internal/controller/dynamonimdeployment_controller.go
View file @ 8621d914
......@@ -1163,7 +1163,7 @@ func (r *DynamoNimDeploymentReconciler) createOrUpdateVirtualService(ctx context
Route: []*istioNetworking.HTTPRouteDestination{
{
Destination: &istioNetworking.Destination{
Host: fmt.Sprintf("%s.yatai.svc.cluster.local", dynamoNimDeployment.Name),
Host: dynamoNimDeployment.Name,
Port: &istioNetworking.PortSelector{
Number: 3000,
},
......@@ -1186,6 +1186,11 @@ func (r *DynamoNimDeploymentReconciler) createOrUpdateVirtualService(ctx context
vsEnabled := dynamoNimDeployment.Spec.Ingress.Enabled && dynamoNimDeployment.Spec.Ingress.UseVirtualService != nil && *dynamoNimDeployment.Spec.Ingress.UseVirtualService
if err := ctrl.SetControllerReference(dynamoNimDeployment, vs, r.Scheme); err != nil {
log.Error(err, "Failed to set controller reference for the VirtualService")
return false, err
}
if err != nil {
if vsEnabled {
log.Info("VirtualService not found, creating new one")
......@@ -1209,7 +1214,7 @@ func (r *DynamoNimDeploymentReconciler) createOrUpdateVirtualService(ctx context
}
log.Info("VirtualService found, updating", "OldVirtualService", oldVS)
vs.ObjectMeta.ResourceVersion = oldVS.ObjectMeta.ResourceVersion
if err := r.Update(ctx, vs); err != nil {
log.Error(err, "Failed to update VirtualService")
return false, err
......@@ -1764,34 +1769,12 @@ monitoring.options.insecure=true`
// do nothing
}
livenessProbe := &corev1.Probe{
InitialDelaySeconds: 10,
TimeoutSeconds: 20,
FailureThreshold: 6,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/livez",
Port: intstr.FromString(commonconsts.BentoContainerPortName),
},
},
}
var livenessProbe *corev1.Probe
if opt.dynamoNimDeployment.Spec.LivenessProbe != nil {
livenessProbe = opt.dynamoNimDeployment.Spec.LivenessProbe
}
readinessProbe := &corev1.Probe{
InitialDelaySeconds: 5,
TimeoutSeconds: 5,
FailureThreshold: 12,
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/readyz",
Port: intstr.FromString(commonconsts.BentoContainerPortName),
},
},
}
var readinessProbe *corev1.Probe
if opt.dynamoNimDeployment.Spec.ReadinessProbe != nil {
readinessProbe = opt.dynamoNimDeployment.Spec.ReadinessProbe
}
......@@ -1803,11 +1786,9 @@ monitoring.options.insecure=true`
args = append(args, "uv", "run", "dynamo", "start")
if opt.dynamoNimDeployment.Spec.ServiceName != "" {
args = append(args, []string{"--service-name", opt.dynamoNimDeployment.Spec.ServiceName}...)
}
if len(opt.dynamoNimDeployment.Spec.ExternalServices) > 0 {
// todo : remove this line when https://github.com/ai-dynamo/dynamo/issues/345 is fixed
enableDependsOption := false
if len(opt.dynamoNimDeployment.Spec.ExternalServices) > 0 && enableDependsOption {
serviceSuffix := fmt.Sprintf("%s.svc.cluster.local:3000", opt.dynamoNimDeployment.Namespace)
keys := make([]string, 0, len(opt.dynamoNimDeployment.Spec.ExternalServices))
......@@ -1823,13 +1804,18 @@ monitoring.options.insecure=true`
if service.DeploymentSelectorKey == "name" {
dependsFlag := fmt.Sprintf("--depends \"%s=http://%s.%s\"", key, service.DeploymentSelectorValue, serviceSuffix)
args = append(args, dependsFlag)
} else if service.DeploymentSelectorKey == "nova" {
dependsFlag := fmt.Sprintf("--depends \"%s=nova://%s\"", key, service.DeploymentSelectorValue)
} else if service.DeploymentSelectorKey == "dynamo" {
dependsFlag := fmt.Sprintf("--depends \"%s=dynamo://%s\"", key, service.DeploymentSelectorValue)
args = append(args, dependsFlag)
} else {
return nil, errors.Errorf("DeploymentSelectorKey '%s' not supported. Only 'name' and 'nova' are supported", service.DeploymentSelectorKey)
return nil, errors.Errorf("DeploymentSelectorKey '%s' not supported. Only 'name' and 'dynamo' are supported", service.DeploymentSelectorKey)
}
}
}
if opt.dynamoNimDeployment.Spec.ServiceName != "" {
args = append(args, []string{"--service-name", opt.dynamoNimDeployment.Spec.ServiceName}...)
args = append(args, "src."+opt.dynamoNimDeployment.Spec.DynamoTag)
}
yataiResources := opt.dynamoNimDeployment.Spec.Resources
......
deploy/dynamo/operator/internal/nim/nim.go
View file @ 8621d914
......@@ -44,7 +44,7 @@ import (
)
// ServiceConfig represents the YAML configuration structure for a service
type NovaConfig struct {
type DynamoConfig struct {
Enabled bool `yaml:"enabled"`
Namespace string `yaml:"namespace"`
Name string `yaml:"name"`
......@@ -67,7 +67,7 @@ type Autoscaling struct {
}
type Config struct {
Nova *NovaConfig `yaml:"nova,omitempty"`
Dynamo *DynamoConfig `yaml:"dynamo,omitempty"`
Resources *Resources `yaml:"resources,omitempty"`
Traffic *Traffic `yaml:"traffic,omitempty"`
Autoscaling *Autoscaling `yaml:"autoscaling,omitempty"`
......@@ -131,7 +131,9 @@ func RetrieveDynamoNimDownloadURL(ctx context.Context, dynamoDeployment *v1alpha
// ServicesConfig represents the top-level YAML structure of a dynamoNim yaml file stored in a dynamoNim tar file
type DynamoNIMConfig struct {
DynamoTag string `yaml:"service"`
Services []ServiceConfig `yaml:"services"`
EntryService string `yaml:"entry_service"`
}
type EventRecorder interface {
......@@ -249,16 +251,24 @@ func GetDynamoNIMConfig(ctx context.Context, dynamoDeployment *v1alpha1.DynamoDe
// generate DynamoNIMDeployment from config
func GenerateDynamoNIMDeployments(parentDynamoDeployment *v1alpha1.DynamoDeployment, config *DynamoNIMConfig) (map[string]*v1alpha1.DynamoNimDeployment, error) {
novaServices := make(map[string]string)
dynamoServices := make(map[string]string)
deployments := make(map[string]*v1alpha1.DynamoNimDeployment)
for _, service := range config.Services {
deployment := &v1alpha1.DynamoNimDeployment{}
deployment.Name = fmt.Sprintf("%s-%s", parentDynamoDeployment.Name, strings.ToLower(service.Name))
deployment.Namespace = parentDynamoDeployment.Namespace
deployment.Spec.DynamoTag = config.DynamoTag
deployment.Spec.DynamoNim = strings.Split(parentDynamoDeployment.Spec.DynamoNim, ":")[0]
deployment.Spec.ServiceName = service.Name
if service.Config.Nova != nil && service.Config.Nova.Enabled {
novaServices[service.Name] = fmt.Sprintf("%s/%s", service.Config.Nova.Name, service.Config.Nova.Namespace)
if service.Config.Dynamo != nil && service.Config.Dynamo.Enabled {
dynamoServices[service.Name] = fmt.Sprintf("%s/%s", service.Config.Dynamo.Name, service.Config.Dynamo.Namespace)
} else {
// dynamo is not enabled
if config.EntryService == service.Name {
// enable virtual service for the entry service
deployment.Spec.Ingress.Enabled = true
deployment.Spec.Ingress.UseVirtualService = &deployment.Spec.Ingress.Enabled
}
}
if service.Config.Resources != nil {
deployment.Spec.Resources = &compounaiCommon.Resources{
......@@ -296,10 +306,10 @@ func GenerateDynamoNIMDeployments(parentDynamoDeployment *v1alpha1.DynamoDeploym
if dependencyDeployment == nil {
return nil, fmt.Errorf("dependency %s not found", dependentServiceName)
}
if novaService, ok := novaServices[dependentServiceName]; ok {
if dynamoService, ok := dynamoServices[dependentServiceName]; ok {
deployment.Spec.ExternalServices[dependentServiceName] = v1alpha1.ExternalService{
DeploymentSelectorKey: "nova",
DeploymentSelectorValue: novaService,
DeploymentSelectorKey: "dynamo",
DeploymentSelectorValue: dynamoService,
}
} else {
deployment.Spec.ExternalServices[dependentServiceName] = v1alpha1.ExternalService{
......
deploy/dynamo/operator/internal/nim/nim_test.go
View file @ 8621d914
......@@ -50,12 +50,13 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
},
config: &DynamoNIMConfig{
DynamoTag: "dynamonim:MyService1",
Services: []ServiceConfig{
{
Name: "service1",
Dependencies: []map[string]string{{"service": "service2"}},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service1",
......@@ -76,7 +77,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Name: "service2",
Dependencies: []map[string]string{},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: false,
},
},
......@@ -92,6 +93,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService1",
ServiceName: "service1",
Resources: &compounaiCommon.Resources{
Requests: &compounaiCommon.ResourceItem{
......@@ -125,8 +127,9 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
ServiceName: "service2",
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService1",
ServiceName: "service2",
},
},
},
......@@ -145,16 +148,13 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
},
config: &DynamoNIMConfig{
DynamoTag: "dynamonim:MyService2",
EntryService: "service1",
Services: []ServiceConfig{
{
Name: "service1",
Dependencies: []map[string]string{{"service": "service2"}},
Config: Config{
Nova: &NovaConfig{
Enabled: true,
Namespace: "default",
Name: "service1",
},
Resources: &Resources{
CPU: "1",
Memory: "1Gi",
......@@ -171,7 +171,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Name: "service2",
Dependencies: []map[string]string{},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service2",
......@@ -189,6 +189,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService2",
ServiceName: "service1",
Resources: &compounaiCommon.Resources{
Requests: &compounaiCommon.ResourceItem{
......@@ -210,10 +211,14 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
ExternalServices: map[string]v1alpha1.ExternalService{
"service2": {
DeploymentSelectorKey: "nova",
DeploymentSelectorKey: "dynamo",
DeploymentSelectorValue: "service2/default",
},
},
Ingress: v1alpha1.IngressSpec{
Enabled: true,
UseVirtualService: &[]bool{true}[0],
},
},
},
"service2": {
......@@ -223,6 +228,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
Spec: v1alpha1.DynamoNimDeploymentSpec{
DynamoNim: "dynamonim",
DynamoTag: "dynamonim:MyService2",
ServiceName: "service2",
},
},
......@@ -242,12 +248,13 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
},
},
config: &DynamoNIMConfig{
DynamoTag: "dynamonim:MyService3",
Services: []ServiceConfig{
{
Name: "service1",
Dependencies: []map[string]string{{"service": "service2"}},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service1",
......@@ -268,7 +275,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
Name: "service3",
Dependencies: []map[string]string{},
Config: Config{
Nova: &NovaConfig{
Dynamo: &DynamoConfig{
Enabled: true,
Namespace: "default",
Name: "service3",
......
deploy/dynamo/sdk/src/dynamo/sdk/cli/cli.py
View file @ 8621d914
......@@ -24,17 +24,21 @@ def create_bentoml_cli() -> click.Command:
from bentoml._internal.context import server_context
from bentoml_cli.bentos import bento_command
# from bentoml_cli.cloud import cloud_command
# from bentoml_cli.containerize import containerize_command
from bentoml_cli.utils import get_entry_points
from dynamo.sdk.cli.deployment import deployment_command
# from dynamo.sdk.cli.deploy import deploy_command
from dynamo.sdk.cli.run import run_command
from dynamo.sdk.cli.serve import serve_command
# from dynamo.sdk.cli.server import cloud_command
from dynamo.sdk.cli.server import cloud_command
from dynamo.sdk.cli.start import start_command
from dynamo.sdk.cli.utils import DynamoCommandGroup
# from dynamo.sdk.cli.cloud import cloud_command
server_context.service_type = "cli"
CONTEXT_SETTINGS = {"help_option_names": ("-h", "--help")}
......@@ -52,14 +56,15 @@ def create_bentoml_cli() -> click.Command:
"""
# Add top-level CLI commands
# bentoml_cli.add_command(cloud_command)
bentoml_cli.add_command(cloud_command)
bentoml_cli.add_single_command(bento_command, "build")
bentoml_cli.add_subcommands(start_command)
bentoml_cli.add_single_command(bento_command, "get")
bentoml_cli.add_subcommands(serve_command)
bentoml_cli.add_subcommands(run_command)
# bentoml_cli.add_command(containerize_command)
# bentoml_cli.add_command(deploy_command)
# bentoml_cli.add_command(containerize_command)
bentoml_cli.add_command(deployment_command)
# Load commands from extensions
for ep in get_entry_points("bentoml.commands"):
bentoml_cli.add_command(ep.load())
......
deploy/dynamo/sdk/src/dynamo/sdk/cli/deployment.py
View file @ 8621d914
......@@ -12,119 +12,817 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, Optional
import json
import logging
import os
import typing as t
from http import HTTPStatus
from kubernetes import config as k8s_config
import bentoml.deployment
import click
import rich
import rich.style
import yaml
from bentoml._internal.cloud.base import Spinner
from bentoml._internal.cloud.deployment import Deployment, DeploymentConfigParameters
from bentoml._internal.cloud.schemas.modelschemas import DeploymentStrategy
from bentoml._internal.configuration.containers import BentoMLContainer
from bentoml._internal.utils import add_experimental_docstring
from bentoml.exceptions import BentoMLException, CLIException
from bentoml_cli.utils import BentoMLCommandGroup
from rich.console import Console
from rich.syntax import Syntax
from rich.table import Table
from simple_di import Provide, inject
logger = logging.getLogger("dynamo.deployment")
class DynamoDeployment:
def __init__(
self,
if t.TYPE_CHECKING:
from bentoml._internal.cloud import BentoCloudClient
TupleStrAny = tuple[str, ...]
else:
TupleStrAny = tuple
def raise_deployment_config_error(err: BentoMLException, action: str) -> t.NoReturn:
if err.error_code == HTTPStatus.UNAUTHORIZED:
raise BentoMLException(
f"{err}\n* BentoCloud API token is required for authorization. Run `bentoml cloud login` command to login"
) from None
raise BentoMLException(
f"Failed to {action} deployment due to invalid configuration: {err}"
) from None
def convert_env_to_dict(env: tuple[str] | None) -> list[dict[str, str]] | None:
if env is None:
return None
collected_envs: list[dict[str, str]] = []
if env:
for item in env:
if "=" in item:
name, value = item.split("=", 1)
else:
name = item
if name not in os.environ:
raise CLIException(f"Environment variable {name} not found")
value = os.environ[name]
collected_envs.append({"name": name, "value": value})
return collected_envs
@click.command(name="deploy")
@click.argument(
"bento",
type=click.STRING,
required=False,
)
@click.option(
"-n",
"--name",
type=click.STRING,
help="Deployment name",
)
@click.option(
"--cluster",
type=click.STRING,
help="Name of the cluster",
)
@click.option(
"--access-authorization",
type=click.BOOL,
help="Enable access authorization",
)
@click.option(
"--scaling-min",
type=click.INT,
help="Minimum scaling value",
)
@click.option(
"--scaling-max",
type=click.INT,
help="Maximum scaling value",
)
@click.option(
"--instance-type",
type=click.STRING,
help="Type of instance",
)
@click.option(
"--strategy",
type=click.Choice(
[deployment_strategy.value for deployment_strategy in DeploymentStrategy]
),
help="Deployment strategy",
)
@click.option(
"--env",
type=click.STRING,
help="List of environment variables pass by --env key[=value] --env ...",
multiple=True,
)
@click.option(
"--secret",
type=click.STRING,
help="List of secret names pass by --secret name1, --secret name2, ...",
multiple=True,
)
@click.option(
"-f",
"--config-file",
type=click.File(),
help="Configuration file path",
default=None,
)
@click.option(
"--config-dict",
type=click.STRING,
help="Configuration json string",
default=None,
)
@click.option(
"--wait/--no-wait",
type=click.BOOL,
is_flag=True,
help="Do not wait for deployment to be ready",
default=True,
)
@click.option(
"--timeout",
type=click.INT,
default=3600,
help="Timeout for deployment to be ready in seconds",
)
@add_experimental_docstring
def deploy_command(
bento: str | None,
name: str | None,
cluster: str | None,
access_authorization: bool | None,
scaling_min: int | None,
scaling_max: int | None,
instance_type: str | None,
strategy: str | None,
env: tuple[str] | None,
secret: tuple[str] | None,
config_file: str | t.TextIO | None,
config_dict: str | None,
wait: bool,
timeout: int,
) -> None:
"""Create a deployment on BentoCloud.
\b
Create a deployment using parameters, or using config yaml file.
"""
create_deployment(
bento=bento,
name=name,
cluster=cluster,
access_authorization=access_authorization,
scaling_min=scaling_min,
scaling_max=scaling_max,
instance_type=instance_type,
strategy=strategy,
env=env,
secret=secret,
config_file=config_file,
config_dict=config_dict,
wait=wait,
timeout=timeout,
)
output_option = click.option(
"-o",
"--output",
type=click.Choice(["yaml", "json"]),
default="yaml",
help="Display the output of this command.",
)
def shared_decorator(
f: t.Callable[..., t.Any] | None = None,
) -> t.Callable[..., t.Any]:
def decorate(f: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]:
options = [
click.option(
"--cluster",
type=click.STRING,
default=None,
help="Name of the cluster.",
),
]
for opt in reversed(options):
f = opt(f)
return f
if f:
return decorate(f)
else:
return decorate
def build_deployment_command() -> click.Group:
@click.group(name="deployment", cls=BentoMLCommandGroup)
@add_experimental_docstring
def deployment_command():
"""Deploy Dynamo applications to Kubernetes cluster"""
@deployment_command.command()
@shared_decorator()
@click.argument(
"name",
type=click.STRING,
required=False,
)
@click.option(
"--bento",
type=click.STRING,
help="Bento name or path to Bento project directory",
)
@click.option(
"--access-authorization",
type=click.BOOL,
help="Enable access authorization",
)
@click.option(
"--scaling-min",
type=click.INT,
help="Minimum scaling value",
)
@click.option(
"--scaling-max",
type=click.INT,
help="Maximum scaling value",
)
@click.option(
"--instance-type",
type=click.STRING,
help="Type of instance",
)
@click.option(
"--strategy",
type=click.Choice(
[deployment_strategy.value for deployment_strategy in DeploymentStrategy]
),
help="Deployment strategy",
)
@click.option(
"--env",
type=click.STRING,
help="List of environment variables pass by --env key[=value] --env ...",
multiple=True,
)
@click.option(
"--secret",
type=click.STRING,
help="List of secret names pass by --secret name1, --secret name2, ...",
multiple=True,
)
@click.option(
"-f",
"--config-file",
type=click.File(),
help="Configuration file path, mututally exclusive with other config options",
default=None,
)
@click.option(
"--config-dict",
type=click.STRING,
help="Configuration json string",
default=None,
)
@inject
def update( # type: ignore
name: str | None,
cluster: str | None,
bento: str | None,
access_authorization: bool | None,
scaling_min: int | None,
scaling_max: int | None,
instance_type: str | None,
strategy: str | None,
env: tuple[str] | None,
secret: tuple[str] | None,
config_file: t.TextIO | None,
config_dict: str | None,
_cloud_client: BentoCloudClient = Provide[BentoMLContainer.bentocloud_client],
) -> None:
"""Update a deployment on BentoCloud.
\b
A deployment can be updated using parameters, or using config yaml file.
You can also update bento by providing a project path or existing bento.
"""
cfg_dict = None
if config_dict is not None and config_dict != "":
cfg_dict = json.loads(config_dict)
config_params = DeploymentConfigParameters(
name=name,
bento=bento,
cluster=cluster,
access_authorization=access_authorization,
scaling_max=scaling_max,
scaling_min=scaling_min,
instance_type=instance_type,
strategy=strategy,
envs=convert_env_to_dict(env),
secrets=list(secret) if secret is not None else None,
config_file=config_file,
config_dict=cfg_dict,
cli=True,
)
try:
config_params.verify(create=False)
except BentoMLException as e:
raise_deployment_config_error(e, "update")
deployment_info = _cloud_client.deployment.update(
deployment_config_params=config_params
)
rich.print(
f"Deployment [green]'{deployment_info.name}'[/] updated successfully."
)
@deployment_command.command()
@click.argument(
"bento",
type=click.STRING,
required=False,
)
@click.option(
"-n",
"--name",
type=click.STRING,
help="Deployment name",
)
@click.option(
"--cluster",
type=click.STRING,
help="Name of the cluster",
)
@click.option(
"--access-authorization",
type=click.BOOL,
help="Enable access authorization",
)
@click.option(
"--scaling-min",
type=click.INT,
help="Minimum scaling value",
)
@click.option(
"--scaling-max",
type=click.INT,
help="Maximum scaling value",
)
@click.option(
"--instance-type",
type=click.STRING,
help="Type of instance",
)
@click.option(
"--strategy",
type=click.Choice(
[deployment_strategy.value for deployment_strategy in DeploymentStrategy]
),
help="Deployment strategy",
)
@click.option(
"--env",
type=click.STRING,
help="List of environment variables pass by --env key[=value] --env ...",
multiple=True,
)
@click.option(
"--secret",
type=click.STRING,
help="List of secret names pass by --secret name1, --secret name2, ...",
multiple=True,
)
@click.option(
"-f",
"--config-file",
type=click.File(),
help="Configuration file path",
default=None,
)
@click.option(
"-f",
"--config-file",
help="Configuration file path, mututally exclusive with other config options",
default=None,
)
@click.option(
"--config-dict",
type=click.STRING,
help="Configuration json string",
default=None,
)
@inject
def apply( # type: ignore
bento: str | None,
name: str | None,
cluster: str | None,
access_authorization: bool | None,
scaling_min: int | None,
scaling_max: int | None,
instance_type: str | None,
strategy: str | None,
env: tuple[str] | None,
secret: tuple[str] | None,
config_file: str | t.TextIO | None,
config_dict: str | None,
_cloud_client: BentoCloudClient = Provide[BentoMLContainer.bentocloud_client],
) -> None:
"""Apply a deployment on BentoCloud.
\b
A deployment can be applied using config yaml file.
"""
cfg_dict = None
if config_dict is not None and config_dict != "":
cfg_dict = json.loads(config_dict)
config_params = DeploymentConfigParameters(
name=name,
bento=bento,
cluster=cluster,
access_authorization=access_authorization,
scaling_max=scaling_max,
scaling_min=scaling_min,
instance_type=instance_type,
strategy=strategy,
envs=convert_env_to_dict(env),
secrets=list(secret) if secret is not None else None,
config_file=config_file,
config_dict=cfg_dict,
cli=True,
)
try:
config_params.verify(create=False)
except BentoMLException as e:
raise_deployment_config_error(e, "apply")
deployment_info = _cloud_client.deployment.apply(
deployment_config_params=config_params
)
rich.print(
f"Deployment [green]'{deployment_info.name}'[/] applied successfully."
)
@deployment_command.command()
@click.argument(
"bento",
type=click.STRING,
required=False,
)
@click.option(
"-n",
"--name",
type=click.STRING,
help="Deployment name",
)
@click.option(
"--cluster",
type=click.STRING,
help="Name of the cluster",
)
@click.option(
"--access-authorization",
type=click.BOOL,
help="Enable access authorization",
)
@click.option(
"--scaling-min",
type=click.INT,
help="Minimum scaling value",
)
@click.option(
"--scaling-max",
type=click.INT,
help="Maximum scaling value",
)
@click.option(
"--instance-type",
type=click.STRING,
help="Type of instance",
)
@click.option(
"--strategy",
type=click.Choice(
[deployment_strategy.value for deployment_strategy in DeploymentStrategy]
),
help="Deployment strategy",
)
@click.option(
"--env",
type=click.STRING,
help="List of environment variables pass by --env key[=value] --env ...",
multiple=True,
)
@click.option(
"--secret",
type=click.STRING,
help="List of secret names pass by --secret name1, --secret name2, ...",
multiple=True,
)
@click.option(
"-f",
"--config-file",
type=click.File(),
help="Configuration file path",
default=None,
)
@click.option(
"--config-dict",
type=click.STRING,
help="Configuration json string",
default=None,
)
@click.option(
"--wait/--no-wait",
type=click.BOOL,
is_flag=True,
help="Do not wait for deployment to be ready",
default=True,
)
@click.option(
"--timeout",
type=click.INT,
default=3600,
help="Timeout for deployment to be ready in seconds",
)
def create(
bento: str | None,
name: str | None,
cluster: str | None,
access_authorization: bool | None,
scaling_min: int | None,
scaling_max: int | None,
instance_type: str | None,
strategy: str | None,
env: tuple[str] | None,
secret: tuple[str] | None,
config_file: str | t.TextIO | None,
config_dict: str | None,
wait: bool,
timeout: int,
) -> None:
"""Create a deployment on BentoCloud.
\b
Create a deployment using parameters, or using config yaml file.
"""
create_deployment(
bento=bento,
name=name,
cluster=cluster,
access_authorization=access_authorization,
scaling_min=scaling_min,
scaling_max=scaling_max,
instance_type=instance_type,
strategy=strategy,
env=env,
secret=secret,
config_file=config_file,
config_dict=config_dict,
wait=wait,
timeout=timeout,
)
@deployment_command.command()
@shared_decorator
@click.argument(
"name",
type=click.STRING,
required=True,
)
@output_option
def get( # type: ignore
name: str,
cluster: str,
admin_console: str,
created_at: str,
created_by: str,
_schema: str = "v1",
ingress_base_url: Optional[str] = None,
):
self.name = name
self.cluster = cluster
self.admin_console = admin_console
self.created_at = created_at
self.created_by = created_by
self._schema = _schema
self.ingress_url = (
f"{ingress_base_url}/api/v2/deployments/{name}?cluster={cluster}"
if ingress_base_url
else None
)
@classmethod
def create_deployment(
cls, deployment_name: str, namespace: str, config: Any
) -> "DynamoDeployment":
# Load kube config and get username
k8s_config.load_kube_config()
username = (
k8s_config.list_kube_config_contexts()[1]
.get("context", {})
.get("user", "unknown")
)
return cls(
name=deployment_name,
cluster=namespace,
admin_console=f"kubectl get dynamodeployment {deployment_name} -n {namespace}",
created_at=datetime.now().isoformat(),
created_by=username,
_schema="v1alpha1",
ingress_base_url=config.get("ingress_base_url"),
)
def get_crd_payload(
self,
bento: str,
scaling_min: int,
scaling_max: int,
instance_type: str,
env_vars: list,
secret: list,
) -> Dict[str, Any]:
# Ensure bento is in name:tag format
if ":" not in bento:
bento = f"{bento}:latest"
payload = {
"apiVersion": "nvidia.com/v1alpha1",
"kind": "DynamoDeployment",
"metadata": {
"name": self.name,
"namespace": self.cluster,
"labels": {
"app.kubernetes.io/name": "dynamo-kubernetes-operator",
"app.kubernetes.io/managed-by": "dynamo-cli",
},
},
"spec": {
"dynamoNim": bento,
"services": {
"main": {
"spec": {
"dynamoNim": bento,
"serviceName": self.name,
"autoscaling": {
"minReplicas": scaling_min or 1,
"maxReplicas": scaling_max or 5,
},
"resources": {
"requests": {
"cpu": "4",
"memory": "16Gi",
"gpu": instance_type or "1",
},
"limits": {
"cpu": "8",
"memory": "32Gi",
"gpu": instance_type or "1",
},
},
"envs": env_vars,
"ingress": {
"enabled": True,
"hostPrefix": self.name,
},
"readinessProbe": {
"httpGet": {"path": "/healthz", "port": 8080},
"initialDelaySeconds": 5,
"periodSeconds": 10,
},
}
}
},
},
}
return payload
cluster: str | None,
output: t.Literal["json", "default"],
) -> None:
"""Get a deployment on BentoCloud."""
d = bentoml.deployment.get(name, cluster=cluster)
if output == "json":
info = json.dumps(d.to_dict(), indent=2, default=str)
rich.print_json(info)
else:
info = yaml.dump(d.to_dict(), indent=2, sort_keys=False)
rich.print(Syntax(info, "yaml", background_color="default"))
@deployment_command.command()
@shared_decorator
@click.argument(
"name",
type=click.STRING,
required=True,
)
@click.option(
"--wait", is_flag=True, help="Wait for the deployment to be terminated"
)
def terminate(name: str, cluster: str | None, wait: bool) -> None: # type: ignore
"""Terminate a deployment on BentoCloud."""
bentoml.deployment.terminate(name, cluster=cluster, wait=wait)
rich.print(f"Deployment [green]'{name}'[/] terminated successfully.")
@deployment_command.command()
@click.argument(
"name",
type=click.STRING,
required=True,
)
@shared_decorator
def delete(name: str, cluster: str | None) -> None: # type: ignore
"""Delete a deployment on BentoCloud."""
bentoml.deployment.delete(name, cluster=cluster)
rich.print(f"Deployment [green]'{name}'[/] deleted successfully.")
@deployment_command.command(name="list")
@click.option(
"--cluster", type=click.STRING, default=None, help="Name of the cluster."
)
@click.option(
"--search", type=click.STRING, default=None, help="Search for list request."
)
@click.option(
"-o",
"--output",
help="Display the output of this command.",
type=click.Choice(["json", "yaml", "table"]),
default="table",
)
@click.option(
"--label",
"labels",
type=click.STRING,
multiple=True,
default=None,
help="Filter deployments by label(s).",
metavar="KEY=VALUE",
)
def list_command( # type: ignore
cluster: str | None,
search: str | None,
labels: tuple[str, ...] | None,
output: t.Literal["json", "yaml", "table"],
) -> None:
"""List existing deployments on BentoCloud."""
if labels is not None:
# For labels like ["env=prod", "team=infra"]
# This will output: "label:env=prod label:team=infra"
labels_query = " ".join(f"label:{label}" for label in labels)
try:
d_list = bentoml.deployment.list(
cluster=cluster, search=search, q=labels_query
)
except BentoMLException as e:
raise_deployment_config_error(e, "list")
res: list[dict[str, t.Any]] = [d.to_dict() for d in d_list]
if output == "table":
table = Table(box=None, expand=True)
table.add_column("Deployment", overflow="fold")
table.add_column("created_at", overflow="fold")
table.add_column("Bento", overflow="fold")
table.add_column("Status", overflow="fold")
table.add_column("Region", overflow="fold")
for info in d_list:
table.add_row(
info.name,
info.created_at,
info.get_bento(refetch=False),
info.get_status(refetch=False).status,
info.cluster,
)
rich.print(table)
elif output == "json":
info = json.dumps(res, indent=2, default=str)
rich.print_json(info)
else:
info = yaml.dump(res, indent=2, sort_keys=False)
rich.print(Syntax(info, "yaml", background_color="default"))
@deployment_command.command()
@click.option(
"--cluster", type=click.STRING, default=None, help="Name of the cluster."
)
@click.option(
"-o",
"--output",
help="Display the output of this command.",
type=click.Choice(["json", "yaml", "table"]),
default="table",
)
@inject
def list_instance_types( # type: ignore
cluster: str | None,
output: t.Literal["json", "yaml", "table"],
_cloud_client: BentoCloudClient = Provide[BentoMLContainer.bentocloud_client],
) -> None:
"""List existing instance types in cluster on BentoCloud."""
try:
d_list = _cloud_client.deployment.list_instance_types(cluster=cluster)
except BentoMLException as e:
raise_deployment_config_error(e, "list_instance_types")
res: list[dict[str, t.Any]] = [d.to_dict() for d in d_list]
if output == "table":
table = Table(box=None, expand=True)
table.add_column("Name", overflow="fold")
table.add_column("Price", overflow="fold")
table.add_column("CPU", overflow="fold")
table.add_column("Memory", overflow="fold")
table.add_column("GPU", overflow="fold")
table.add_column("GPU Type", overflow="fold")
for info in d_list:
table.add_row(
info.name,
info.price,
info.cpu,
info.memory,
info.gpu,
info.gpu_type,
)
rich.print(table)
elif output == "json":
info = json.dumps(res, indent=2, default=str)
rich.print_json(info)
else:
info = yaml.dump(res, indent=2, sort_keys=False)
rich.print(Syntax(info, "yaml", background_color="default"))
return deployment_command
deployment_command = build_deployment_command()
@inject
def create_deployment(
bento: str | None = None,
name: str | None = None,
cluster: str | None = None,
access_authorization: bool | None = None,
scaling_min: int | None = None,
scaling_max: int | None = None,
instance_type: str | None = None,
strategy: str | None = None,
env: tuple[str] | None = None,
secret: tuple[str] | None = None,
config_file: str | t.TextIO | None = None,
config_dict: str | None = None,
wait: bool = True,
timeout: int = 3600,
dev: bool = False,
_cloud_client: BentoCloudClient = Provide[BentoMLContainer.bentocloud_client],
) -> Deployment:
cfg_dict = None
if config_dict is not None and config_dict != "":
cfg_dict = json.loads(config_dict)
config_params = DeploymentConfigParameters(
name=name,
bento=bento,
cluster=cluster,
access_authorization=access_authorization,
scaling_max=scaling_max,
scaling_min=scaling_min,
instance_type=instance_type,
strategy=strategy,
envs=convert_env_to_dict(env),
secrets=list(secret) if secret is not None else None,
config_file=config_file,
config_dict=cfg_dict,
cli=True,
dev=dev,
)
try:
config_params.verify()
except BentoMLException as e:
raise_deployment_config_error(e, "create")
console = Console(highlight=False)
with Spinner(console=console) as spinner:
spinner.update("Creating deployment on BentoCloud")
deployment = _cloud_client.deployment.create(
deployment_config_params=config_params
)
spinner.log(
f':white_check_mark: Created deployment "{deployment.name}" in cluster "{deployment.cluster}"'
)
spinner.log(f":laptop_computer: View Dashboard: {deployment.admin_console}")
if wait:
spinner.update(
"[bold blue]Waiting for deployment to be ready, you can use --no-wait to skip this process[/]",
)
retcode = deployment.wait_until_ready(timeout=timeout, spinner=spinner)
if retcode != 0:
raise SystemExit(retcode)
return deployment
deploy/dynamo/sdk/src/dynamo/sdk/cli/server.py
View file @ 8621d914
......@@ -26,17 +26,19 @@ from bentoml._internal.cloud.config import (
CloudClientContext,
)
from bentoml._internal.configuration.containers import BentoMLContainer
from bentoml._internal.utils import add_experimental_docstring
from bentoml._internal.utils.cattr import bentoml_cattr
from bentoml.exceptions import CLIException, CloudRESTApiClientError
@click.group(name="server")
def cloud_command():
def build_cloud_command() -> click.Group:
@click.group(name="server")
@add_experimental_docstring
def cloud_command():
"""Interact with your Dynamo Server"""
@cloud_command.command()
@click.option(
@cloud_command.command()
@click.option(
"--endpoint",
type=click.STRING,
help="Dynamo Server endpoint",
......@@ -45,16 +47,16 @@ def cloud_command():
show_default=True,
show_envvar=True,
required=True,
)
@click.option(
)
@click.option(
"--api-token",
type=click.STRING,
help="Dynamo Server user API token",
envvar="DYNAMO_SERVER_API_KEY",
show_envvar=True,
required=True,
)
def login(endpoint: str, api_token: str) -> None: # type: ignore
)
def login(endpoint: str, api_token: str) -> None: # type: ignore
"""Connect to your Dynamo Server. You can find deployment instructions for this in our docs"""
try:
cloud_rest_client = RestApiClient(endpoint, api_token)
......@@ -72,7 +74,9 @@ def login(endpoint: str, api_token: str) -> None: # type: ignore
cloud_context = BentoMLContainer.cloud_context.get()
ctx = CloudClientContext(
name=cloud_context if cloud_context is not None else current_context_name,
name=cloud_context
if cloud_context is not None
else current_context_name,
endpoint=endpoint,
api_token=api_token,
email=user.email,
......@@ -80,7 +84,7 @@ def login(endpoint: str, api_token: str) -> None: # type: ignore
ctx.save()
rich.print(
f":white_check_mark: Configured BentoCloud credentials (current-context: {ctx.name})"
f":white_check_mark: Configured Dynamo Cloud credentials (current-context: {ctx.name})"
)
rich.print(
f":white_check_mark: Logged in as [blue]{user.email}[/] at [blue]{org.name}[/] organization"
......@@ -97,25 +101,29 @@ def login(endpoint: str, api_token: str) -> None: # type: ignore
file=sys.stderr,
)
@cloud_command.command()
def current_context() -> None: # type: ignore
@cloud_command.command()
def current_context() -> None: # type: ignore
"""Get current cloud context."""
rich.print_json(
data=bentoml_cattr.unstructure(CloudClientConfig.get_config().get_context())
)
@cloud_command.command()
def list_context() -> None: # type: ignore
@cloud_command.command()
def list_context() -> None: # type: ignore
"""List all available context."""
config = CloudClientConfig.get_config()
rich.print_json(data=bentoml_cattr.unstructure([i.name for i in config.contexts]))
rich.print_json(
data=bentoml_cattr.unstructure([i.name for i in config.contexts])
)
@cloud_command.command()
@click.argument("context_name", type=click.STRING)
def update_current_context(context_name: str) -> None: # type: ignore
@cloud_command.command()
@click.argument("context_name", type=click.STRING)
def update_current_context(context_name: str) -> None: # type: ignore
"""Update current context"""
ctx = CloudClientConfig.get_config().set_current_context(context_name)
rich.print(f"Successfully switched to context: {ctx.name}")
return cloud_command
cloud_command = build_cloud_command()
deploy/dynamo/tests/test_deployment.sh 0 → 100755
View file @ 8621d914
#!/bin/bash
#!/bin/bash -e
# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -euo pipefail
export DYNAMO_SEREVR="${DYNAMO_SEREVR:-http://dynamo-server}"
export DYNAMO_IMAGE="${DYNAMO_IMAGE:-dynamo-base:latest}"
export DEPLOYMENT_NAME="${DEPLOYMENT_NAME:-ci-hw}"
cd /workspace/examples/hello_world
# Step.1: Login to dynamo server
dynamo server login --api-token TEST-TOKEN --endpoint $DYNAMO_SEREVR
# Step.2: build a dynamo nim with framework-less base
DYNAMO_TAG=$(dynamo build hello_world:Frontend | grep "Successfully built" | awk -F"\"" '{ print $2 }')
# Step.3: Deploy!
echo $DYNAMO_TAG
dynamo deployment create $DYNAMO_TAG --no-wait -n $DEPLOYMENT_NAME
lib/bindings/python/pyproject.toml
View file @ 8621d914
......@@ -26,7 +26,7 @@ license = { text = "Apache-2.0" }
license-files = ["LICENSE"]
requires-python = ">=3.10"
dependencies = [
"pydantic>=2.10.6",
"pydantic>=2.10.6,<2.11.0",
"uvloop>=0.21.0",
"nats-py>=2.6.0",
]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment