Skip to content

GitLab

Unverified Commit ba16ed52 authored by hhzhang16's avatar hhzhang16 Committed by GitHub
Browse files

feat: set env variables in Dynamo deployments from secrets (#1325) 


Signed-off-by: default avatarhhzhang16 <54051230+hhzhang16@users.noreply.github.com>
Co-authored-by: default avatarcoderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
parent d9f6d7a5
Hide whitespace changes
Inline Side-by-side
Showing with 201 additions and 72 deletions
deploy/cloud/api-store/ai_dynamo_store/models/schemas.py
View file @ ba16ed52
......@@ -15,7 +15,7 @@
# limitations under the License.
from datetime import datetime
from typing import Dict, List, Optional
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
......@@ -49,7 +49,7 @@ class ClusterSchema(ResourceSchema):
class DeploymentConfigSchema(BaseModel):
access_authorization: bool = False
envs: Optional[List[Dict[str, str]]] = None
envs: Optional[List[Dict[str, Any]]] = None
labels: Optional[List[Dict[str, str]]] = None
secrets: Optional[List[str]] = None
services: Dict[str, Dict] = Field(default_factory=dict)
......
deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller.go
View file @ ba16ed52
......@@ -1378,10 +1378,16 @@ func (r *DynamoComponentDeploymentReconciler) generatePodTemplateSpec(ctx contex
}
}
envsSeen[env.Name] = struct{}{}
envs = append(envs, corev1.EnvVar{
Name: env.Name,
Value: env.Value,
})
envVar := corev1.EnvVar{
Name: env.Name,
}
if env.Value != "" {
envVar.Value = env.Value
}
if env.ValueFrom != nil {
envVar.ValueFrom = env.ValueFrom
}
envs = append(envs, envVar)
}
}
......
deploy/sdk/src/dynamo/sdk/cli/deployment.py
View file @ ba16ed52
......@@ -29,13 +29,14 @@ from dynamo.sdk.core.deploy.consts import DeploymentTargetType
from dynamo.sdk.core.deploy.kubernetes import KubernetesDeploymentManager
from dynamo.sdk.core.protocol.deployment import (
Deployment,
DeploymentConfig,
DeploymentManager,
DeploymentResponse,
)
from dynamo.sdk.core.runner import TargetEnum
app = typer.Typer(
help="Deploy Dynamo applications to Dynamo Cloud Kubernetes Platform",
help="Deploy Dynamo applications to Dynamo Cloud Platform",
add_completion=True,
no_args_is_help=True,
)
......@@ -88,66 +89,55 @@ def _build_env_dicts(
config_file: t.Optional[t.TextIO] = None,
args: t.Optional[t.List[str]] = None,
envs: t.Optional[t.List[str]] = None,
) -> t.List[dict]:
envs_from_secret: t.Optional[t.List[str]] = None,
env_secrets_name: t.Optional[str] = "dynamo-env-secrets",
) -> t.List[t.Dict[str, t.Any]]:
"""
Build a list of environment variable dicts from config file, args, and env strings.
Args:
config_file: Optional configuration file
args: Optional list of extra arguments
envs: Optional list of environment variable strings (KEY=VALUE)
Returns:
List of dicts suitable for use as envs
Build a list of environment variable dicts.
"""
service_configs = resolve_service_config(config_file=config_file, args=args)
env_dicts = []
if service_configs:
env_dicts: t.List[t.Dict[str, t.Any]] = []
if config_file or args:
service_configs = resolve_service_config(config_file=config_file, args=args)
config_json = json.dumps(service_configs)
env_dicts.append({"name": "DYN_DEPLOYMENT_CONFIG", "value": config_json})
if envs:
for env in envs:
if "=" not in env:
if "=" in env:
key, value = env.split("=", 1)
env_dicts.append({"name": key, "value": value})
else:
raise RuntimeError(f"Invalid env format: {env}. Use KEY=VALUE.")
key, value = env.split("=", 1)
env_dicts.append({"name": key, "value": value})
if envs_from_secret:
for env in envs_from_secret:
if "=" in env:
key, secret_key = env.split("=", 1)
env_dicts.append(
{
"name": key,
"valueFrom": {
"secretKeyRef": {
"name": env_secrets_name,
"key": secret_key,
}
},
}
)
else:
raise RuntimeError(
f"Invalid env-from-secret format: {env}. Use KEY=SECRET_KEY."
)
return env_dicts
def _handle_deploy_create(
ctx: typer.Context,
pipeline: str = typer.Argument(..., help="Dynamo pipeline to deploy"),
name: t.Optional[str] = typer.Option(None, "--name", "-n", help="Deployment name"),
config_file: t.Optional[typer.FileText] = typer.Option(
None, "--config-file", "-f", help="Configuration file path"
),
wait: bool = typer.Option(
True, "--wait/--no-wait", help="Do not wait for deployment to be ready"
),
timeout: int = typer.Option(
3600, "--timeout", help="Timeout for deployment to be ready in seconds"
),
endpoint: str = typer.Option(
..., "--endpoint", "-e", help="Dynamo Cloud endpoint", envvar="DYNAMO_CLOUD"
),
envs: t.Optional[t.List[str]] = typer.Option(
None,
"--env",
help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
),
target: str = typer.Option(
DeploymentTargetType.KUBERNETES.value,
"--target",
"-t",
help="Deployment target",
),
dev: bool = typer.Option(False, "--dev", help="Development mode for deployment"),
config: DeploymentConfig,
) -> DeploymentResponse:
"""Handle deployment creation. This is a helper function for the create and deploy commands.
Args:
ctx: typer context
pipeline: pipeline to deploy
name: name of the deployment
config: DeploymentConfig object
"""
from dynamo.sdk.cli.utils import configure_target_environment
......@@ -156,14 +146,21 @@ def _handle_deploy_create(
# TODO: hardcoding this is a hack to get the services for the deployment
# we should find a better way to do this once build is finished/generic
configure_target_environment(TargetEnum.DYNAMO)
entry_service = load_entry_service(pipeline)
deployment_manager = get_deployment_manager(target, endpoint)
env_dicts = _build_env_dicts(config_file=config_file, args=ctx.args, envs=envs)
entry_service = load_entry_service(config.pipeline)
deployment_manager = get_deployment_manager(config.target, config.endpoint)
env_dicts = _build_env_dicts(
config_file=config.config_file,
args=ctx.args,
envs=config.envs,
envs_from_secret=config.envs_from_secret,
env_secrets_name=config.env_secrets_name,
)
deployment = Deployment(
name=name or (pipeline if pipeline else "unnamed-deployment"),
name=config.name
or (config.pipeline if config.pipeline else "unnamed-deployment"),
namespace="default",
pipeline=pipeline,
pipeline=config.pipeline,
entry_service=entry_service,
envs=env_dicts,
)
......@@ -171,24 +168,24 @@ def _handle_deploy_create(
console.print("[bold green]Creating deployment...")
deployment = deployment_manager.create_deployment(
deployment,
dev=dev,
dev=config.dev,
)
console.print(f"[bold green]Deployment '{name}' created.")
if wait:
console.print(f"[bold green]Deployment '{config.name}' created.")
if config.wait:
deployment, ready = deployment_manager.wait_until_ready(
name, timeout=timeout
config.name, timeout=config.timeout
)
if ready:
console.print(
Panel(
f"Deployment [bold]{name}[/] is [green]ready[/]",
f"Deployment [bold]{config.name}[/] is [green]ready[/]",
title="Status",
)
)
else:
console.print(
Panel(
f"Deployment [bold]{name}[/] did not become ready in time.",
f"Deployment [bold]{config.name}[/] did not become ready in time.",
title="Status",
style="red",
)
......@@ -201,7 +198,7 @@ def _handle_deploy_create(
if status == 409:
console.print(
Panel(
f"Cannot create deployment because deployment with name '{name}' already exists.",
f"Cannot create deployment because deployment with name '{config.name}' already exists.",
title="Error",
style="red",
)
......@@ -253,6 +250,11 @@ def create(
"--env",
help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
),
envs_from_secret: t.Optional[t.List[str]] = typer.Option(
None,
"--env-from-secret",
help="Environment variable(s) from secret (format: KEY=SECRET_KEY). These will be set from your Dynamo secrets.",
),
target: str = typer.Option(
DeploymentTargetType.KUBERNETES.value,
"--target",
......@@ -260,11 +262,28 @@ def create(
help="Deployment target",
),
dev: bool = typer.Option(False, "--dev", help="Development mode for deployment"),
env_secrets_name: t.Optional[str] = typer.Option(
"dynamo-env-secrets",
"--env-secrets-name",
help="Environment secrets name",
envvar="DYNAMO_ENV_SECRETS",
),
) -> DeploymentResponse:
"""Create a deployment on Dynamo Cloud."""
return _handle_deploy_create(
ctx, pipeline, name, config_file, wait, timeout, endpoint, envs, target, dev
config = DeploymentConfig(
pipeline=pipeline,
endpoint=endpoint,
name=name,
config_file=config_file,
wait=wait,
timeout=timeout,
envs=envs,
envs_from_secret=envs_from_secret,
target=target,
dev=dev,
env_secrets_name=env_secrets_name,
)
return _handle_deploy_create(ctx, config)
@app.command()
......@@ -374,9 +393,20 @@ def update(
"--env",
help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
),
envs_from_secret: t.Optional[t.List[str]] = typer.Option(
None,
"--env-from-secret",
help="Environment variable(s) from secret (format: KEY=SECRET_KEY). These will be set from your Dynamo secrets.",
),
endpoint: str = typer.Option(
..., "--endpoint", "-e", help="Dynamo Cloud endpoint", envvar="DYNAMO_CLOUD"
),
env_secrets_name: t.Optional[str] = typer.Option(
"dynamo-env-secrets",
"--env-secrets-name",
help="Environment secrets name",
envvar="DYNAMO_ENV_SECRETS",
),
) -> None:
"""Update an existing deployment on Dynamo Cloud.
......@@ -386,7 +416,11 @@ def update(
try:
with console.status(f"[bold green]Updating deployment '{name}'..."):
env_dicts = _build_env_dicts(
config_file=config_file, args=ctx.args, envs=envs
config_file=config_file,
args=ctx.args,
envs=envs,
envs_from_secret=envs_from_secret,
env_secrets_name=env_secrets_name,
)
deployment = Deployment(
name=name,
......@@ -449,7 +483,7 @@ def delete(
)
except Exception as e:
if isinstance(e, RuntimeError) and isinstance(e.args[0], tuple):
status, msg, url = e.args[0]
status, msg, _ = e.args[0]
if status == 404:
console.print(
Panel(
......@@ -492,6 +526,11 @@ def deploy(
"--env",
help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
),
envs_from_secret: t.Optional[t.List[str]] = typer.Option(
None,
"--env-from-secret",
help="Environment variable(s) from secret (format: KEY=SECRET_KEY). These will be set from your Dynamo secrets.",
),
target: str = typer.Option(
DeploymentTargetType.KUBERNETES.value,
"--target",
......@@ -499,8 +538,25 @@ def deploy(
help="Deployment target",
),
dev: bool = typer.Option(False, "--dev", help="Development mode for deployment"),
env_secrets_name: t.Optional[str] = typer.Option(
"dynamo-env-secrets",
"--env-secrets-name",
help="Environment secrets name",
envvar="DYNAMO_ENV_SECRETS",
),
) -> DeploymentResponse:
"""Deploy a Dynamo pipeline (same as deployment create)."""
return _handle_deploy_create(
ctx, pipeline, name, config_file, wait, timeout, endpoint, envs, target, dev
config = DeploymentConfig(
pipeline=pipeline,
endpoint=endpoint,
name=name,
config_file=config_file,
wait=wait,
timeout=timeout,
envs=envs,
envs_from_secret=envs_from_secret,
target=target,
dev=dev,
env_secrets_name=env_secrets_name,
)
return _handle_deploy_create(ctx, config)
deploy/sdk/src/dynamo/sdk/core/protocol/deployment.py
View file @ ba16ed52
......@@ -18,6 +18,8 @@ from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from enum import Enum
import typer
@dataclass
class Resources:
......@@ -137,13 +139,39 @@ class Deployment:
namespace: str
pipeline: t.Optional[str] = None
entry_service: t.Optional[Service] = None
envs: t.Optional[t.List[dict]] = None
envs: t.Optional[t.List[t.Dict[str, t.Any]]] = None
# Type alias for deployment responses (e.g., from backend APIs)
DeploymentResponse = t.Dict[str, t.Any]
@dataclass
class DeploymentConfig:
"""Configuration object for deployment operations.
Consolidates all deployment parameters including pipeline configuration,
environment variables, and deployment settings.
"""
# Core deployment settings
pipeline: str
endpoint: str
name: t.Optional[str] = None
target: str = "kubernetes"
dev: bool = False
# Configuration and timing
config_file: t.Optional[typer.FileText] = None
wait: bool = True
timeout: int = 3600
# Environment variables
envs: t.Optional[t.List[str]] = None
envs_from_secret: t.Optional[t.List[str]] = None
env_secrets_name: t.Optional[str] = "dynamo-env-secrets"
class DeploymentManager(ABC):
"""Interface for managing dynamo graph deployments."""
......
docs/guides/dynamo_deploy/operator_deployment.md
View file @ ba16ed52
......@@ -188,3 +188,40 @@ This demonstrates the service pipeline:
1. The Frontend receives "test"
2. The Middle service adds "-mid" to create "test-mid"
3. The Backend service adds "-back" to create "test-mid-back"
## Using Kubernetes Secrets for Environment Variables
Dynamo supports securely injecting environment variables from Kubernetes secrets into your deployment. This is only supported when deploying with `--target kubernetes`.
### Creating a Secret
First, create a Kubernetes secret containing your sensitive values:
```bash
export HF_TOKEN=your_hf_token
kubectl create secret generic dynamo-env-secrets \
--from-literal=huggingface.token=$HF_TOKEN \
--from-literal=another_secret.key=value \
-n $KUBE_NS
```
### Referencing Secrets in Your Deployment
You can reference secret keys in your deployment using the `--env-from-secret` flag:
- `--env-from-secret HF_TOKEN=huggingface.token` will set the `HF_TOKEN` environment variable from the `huggingface.token` key in the secret.
- `--env-from-secret ANOTHER_SECRET=another_secret.key` will set the `ANOTHER_SECRET` environment variable from the same-named key in the secret.
- You can also mix normal envs: `--env NORMAL_ENV_KEY=value`.
By default, Dynamo will look for a secret named `dynamo-env-secrets`. You can override this with the `--env-secrets-name` flag or the `DYNAMO_ENV_SECRETS` environment variable.
### Example Full Command
```bash
dynamo deploy $DYNAMO_TAG -n $DEPLOYMENT_NAME -f ./configs/agg.yaml \
--env NORMAL_ENV_KEY=value \
--env-from-secret HF_TOKEN=huggingface.token \
--env-from-secret ANOTHER_SECRET=another_secret.key \
--target kubernetes
```
examples/llm/README.md
View file @ ba16ed52
......@@ -219,6 +219,8 @@ export DEPLOYMENT_NAME=llm-agg
dynamo deployment create $DYNAMO_TAG -n $DEPLOYMENT_NAME -f ./configs/agg.yaml
```
**Note**: To avoid rate limiting from unauthenticated requests to HuggingFace (HF), you can provide your `HF_TOKEN` as a secret in your deployment. See the [operator deployment guide](../../docs/guides/dynamo_deploy/operator_deployment.md#referencing-secrets-in-your-deployment) for instructions on referencing secrets like `HF_TOKEN` in your deployment configuration.
**Note**: Optionally add `--Planner.no-operation=false` at the end of the deployment command to enable the planner component to take scaling actions on your deployment.
### Testing the Deployment
......@@ -248,4 +250,4 @@ curl localhost:8000/v1/chat/completions \
}'
```
For more details on managing deployments, testing, and troubleshooting, please refer to the [Operator Deployment Guide](../../docs/guides/dynamo_deploy/operator_deployment.md).
\ No newline at end of file
For more details on managing deployments, testing, and troubleshooting, please refer to the [Operator Deployment Guide](../../docs/guides/dynamo_deploy/operator_deployment.md).
examples/llm/configs/disagg.yaml
View file @ ba16ed52
......@@ -48,4 +48,4 @@ PrefillWorker:
Planner:
environment: local
no-operation: true
\ No newline at end of file
no-operation: true
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment