feat: set env variables in Dynamo deployments from secrets (#1325)

Signed-off-by: hhzhang16 <54051230+hhzhang16@users.noreply.github.com>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

deploy/cloud/api-store/ai_dynamo_store/models/schemas.py

@@ -15,7 +15,7 @@
 #  limitations under the License.
 
 from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel, Field
 
@@ -49,7 +49,7 @@ class ClusterSchema(ResourceSchema):
 
 class DeploymentConfigSchema(BaseModel):
     access_authorization: bool = False
-    envs: Optional[List[Dict[str, str]]] = None
+    envs: Optional[List[Dict[str, Any]]] = None
     labels: Optional[List[Dict[str, str]]] = None
     secrets: Optional[List[str]] = None
     services: Dict[str, Dict] = Field(default_factory=dict)

deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller.go

@@ -1378,10 +1378,16 @@ func (r *DynamoComponentDeploymentReconciler) generatePodTemplateSpec(ctx contex
 				}
 			}
 			envsSeen[env.Name] = struct{}{}
-			envs = append(envs, corev1.EnvVar{
-				Name:  env.Name,
-				Value: env.Value,
-			})
+			envVar := corev1.EnvVar{
+				Name: env.Name,
+			}
+			if env.Value != "" {
+				envVar.Value = env.Value
+			}
+			if env.ValueFrom != nil {
+				envVar.ValueFrom = env.ValueFrom
+			}
+			envs = append(envs, envVar)
 		}
 	}
 

deploy/sdk/src/dynamo/sdk/cli/deployment.py

@@ -29,13 +29,14 @@ from dynamo.sdk.core.deploy.consts import DeploymentTargetType
 from dynamo.sdk.core.deploy.kubernetes import KubernetesDeploymentManager
 from dynamo.sdk.core.protocol.deployment import (
     Deployment,
+    DeploymentConfig,
     DeploymentManager,
     DeploymentResponse,
 )
 from dynamo.sdk.core.runner import TargetEnum
 
 app = typer.Typer(
-    help="Deploy Dynamo applications to Dynamo Cloud Kubernetes Platform",
+    help="Deploy Dynamo applications to Dynamo Cloud Platform",
     add_completion=True,
     no_args_is_help=True,
 )
@@ -88,66 +89,55 @@ def _build_env_dicts(
     config_file: t.Optional[t.TextIO] = None,
     args: t.Optional[t.List[str]] = None,
     envs: t.Optional[t.List[str]] = None,
-) -> t.List[dict]:
+    envs_from_secret: t.Optional[t.List[str]] = None,
+    env_secrets_name: t.Optional[str] = "dynamo-env-secrets",
+) -> t.List[t.Dict[str, t.Any]]:
     """
-    Build a list of environment variable dicts from config file, args, and env strings.
-
-    Args:
-        config_file: Optional configuration file
-        args: Optional list of extra arguments
-        envs: Optional list of environment variable strings (KEY=VALUE)
-    Returns:
-        List of dicts suitable for use as envs
+    Build a list of environment variable dicts.
     """
-    service_configs = resolve_service_config(config_file=config_file, args=args)
-    env_dicts = []
-    if service_configs:
+    env_dicts: t.List[t.Dict[str, t.Any]] = []
+    if config_file or args:
+        service_configs = resolve_service_config(config_file=config_file, args=args)
         config_json = json.dumps(service_configs)
         env_dicts.append({"name": "DYN_DEPLOYMENT_CONFIG", "value": config_json})
     if envs:
         for env in envs:
-            if "=" not in env:
+            if "=" in env:
+                key, value = env.split("=", 1)
+                env_dicts.append({"name": key, "value": value})
+            else:
                 raise RuntimeError(f"Invalid env format: {env}. Use KEY=VALUE.")
-            key, value = env.split("=", 1)
-            env_dicts.append({"name": key, "value": value})
+    if envs_from_secret:
+        for env in envs_from_secret:
+            if "=" in env:
+                key, secret_key = env.split("=", 1)
+                env_dicts.append(
+                    {
+                        "name": key,
+                        "valueFrom": {
+                            "secretKeyRef": {
+                                "name": env_secrets_name,
+                                "key": secret_key,
+                            }
+                        },
+                    }
+                )
+            else:
+                raise RuntimeError(
+                    f"Invalid env-from-secret format: {env}. Use KEY=SECRET_KEY."
+                )
     return env_dicts
 
 
 def _handle_deploy_create(
     ctx: typer.Context,
-    pipeline: str = typer.Argument(..., help="Dynamo pipeline to deploy"),
-    name: t.Optional[str] = typer.Option(None, "--name", "-n", help="Deployment name"),
-    config_file: t.Optional[typer.FileText] = typer.Option(
-        None, "--config-file", "-f", help="Configuration file path"
-    ),
-    wait: bool = typer.Option(
-        True, "--wait/--no-wait", help="Do not wait for deployment to be ready"
-    ),
-    timeout: int = typer.Option(
-        3600, "--timeout", help="Timeout for deployment to be ready in seconds"
-    ),
-    endpoint: str = typer.Option(
-        ..., "--endpoint", "-e", help="Dynamo Cloud endpoint", envvar="DYNAMO_CLOUD"
-    ),
-    envs: t.Optional[t.List[str]] = typer.Option(
-        None,
-        "--env",
-        help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
-    ),
-    target: str = typer.Option(
-        DeploymentTargetType.KUBERNETES.value,
-        "--target",
-        "-t",
-        help="Deployment target",
-    ),
-    dev: bool = typer.Option(False, "--dev", help="Development mode for deployment"),
+    config: DeploymentConfig,
 ) -> DeploymentResponse:
     """Handle deployment creation. This is a helper function for the create and deploy commands.
 
     Args:
         ctx: typer context
-        pipeline: pipeline to deploy
-        name: name of the deployment
+        config: DeploymentConfig object
     """
 
     from dynamo.sdk.cli.utils import configure_target_environment
@@ -156,14 +146,21 @@ def _handle_deploy_create(
     # TODO: hardcoding this is a hack to get the services for the deployment
     # we should find a better way to do this once build is finished/generic
     configure_target_environment(TargetEnum.DYNAMO)
-    entry_service = load_entry_service(pipeline)
-
-    deployment_manager = get_deployment_manager(target, endpoint)
-    env_dicts = _build_env_dicts(config_file=config_file, args=ctx.args, envs=envs)
+    entry_service = load_entry_service(config.pipeline)
+
+    deployment_manager = get_deployment_manager(config.target, config.endpoint)
+    env_dicts = _build_env_dicts(
+        config_file=config.config_file,
+        args=ctx.args,
+        envs=config.envs,
+        envs_from_secret=config.envs_from_secret,
+        env_secrets_name=config.env_secrets_name,
+    )
     deployment = Deployment(
-        name=name or (pipeline if pipeline else "unnamed-deployment"),
+        name=config.name
+        or (config.pipeline if config.pipeline else "unnamed-deployment"),
         namespace="default",
-        pipeline=pipeline,
+        pipeline=config.pipeline,
         entry_service=entry_service,
         envs=env_dicts,
     )
@@ -171,24 +168,24 @@ def _handle_deploy_create(
         console.print("[bold green]Creating deployment...")
         deployment = deployment_manager.create_deployment(
             deployment,
-            dev=dev,
+            dev=config.dev,
         )
-        console.print(f"[bold green]Deployment '{name}' created.")
-        if wait:
+        console.print(f"[bold green]Deployment '{config.name}' created.")
+        if config.wait:
             deployment, ready = deployment_manager.wait_until_ready(
-                name, timeout=timeout
+                config.name, timeout=config.timeout
             )
             if ready:
                 console.print(
                     Panel(
-                        f"Deployment [bold]{name}[/] is [green]ready[/]",
+                        f"Deployment [bold]{config.name}[/] is [green]ready[/]",
                         title="Status",
                     )
                 )
             else:
                 console.print(
                     Panel(
-                        f"Deployment [bold]{name}[/] did not become ready in time.",
+                        f"Deployment [bold]{config.name}[/] did not become ready in time.",
                         title="Status",
                         style="red",
                     )
@@ -201,7 +198,7 @@ def _handle_deploy_create(
             if status == 409:
                 console.print(
                     Panel(
-                        f"Cannot create deployment because deployment with name '{name}' already exists.",
+                        f"Cannot create deployment because deployment with name '{config.name}' already exists.",
                         title="Error",
                         style="red",
                     )
@@ -253,6 +250,11 @@ def create(
         "--env",
         help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
     ),
+    envs_from_secret: t.Optional[t.List[str]] = typer.Option(
+        None,
+        "--env-from-secret",
+        help="Environment variable(s) from secret (format: KEY=SECRET_KEY). These will be set from your Dynamo secrets.",
+    ),
     target: str = typer.Option(
         DeploymentTargetType.KUBERNETES.value,
         "--target",
@@ -260,11 +262,28 @@ def create(
         help="Deployment target",
     ),
     dev: bool = typer.Option(False, "--dev", help="Development mode for deployment"),
+    env_secrets_name: t.Optional[str] = typer.Option(
+        "dynamo-env-secrets",
+        "--env-secrets-name",
+        help="Environment secrets name",
+        envvar="DYNAMO_ENV_SECRETS",
+    ),
 ) -> DeploymentResponse:
     """Create a deployment on Dynamo Cloud."""
-    return _handle_deploy_create(
-        ctx, pipeline, name, config_file, wait, timeout, endpoint, envs, target, dev
+    config = DeploymentConfig(
+        pipeline=pipeline,
+        endpoint=endpoint,
+        name=name,
+        config_file=config_file,
+        wait=wait,
+        timeout=timeout,
+        envs=envs,
+        envs_from_secret=envs_from_secret,
+        target=target,
+        dev=dev,
+        env_secrets_name=env_secrets_name,
     )
+    return _handle_deploy_create(ctx, config)
 
 
 @app.command()
@@ -374,9 +393,20 @@ def update(
         "--env",
         help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
     ),
+    envs_from_secret: t.Optional[t.List[str]] = typer.Option(
+        None,
+        "--env-from-secret",
+        help="Environment variable(s) from secret (format: KEY=SECRET_KEY). These will be set from your Dynamo secrets.",
+    ),
     endpoint: str = typer.Option(
         ..., "--endpoint", "-e", help="Dynamo Cloud endpoint", envvar="DYNAMO_CLOUD"
     ),
+    env_secrets_name: t.Optional[str] = typer.Option(
+        "dynamo-env-secrets",
+        "--env-secrets-name",
+        help="Environment secrets name",
+        envvar="DYNAMO_ENV_SECRETS",
+    ),
 ) -> None:
     """Update an existing deployment on Dynamo Cloud.
 
@@ -386,7 +416,11 @@ def update(
     try:
         with console.status(f"[bold green]Updating deployment '{name}'..."):
             env_dicts = _build_env_dicts(
-                config_file=config_file, args=ctx.args, envs=envs
+                config_file=config_file,
+                args=ctx.args,
+                envs=envs,
+                envs_from_secret=envs_from_secret,
+                env_secrets_name=env_secrets_name,
             )
             deployment = Deployment(
                 name=name,
@@ -449,7 +483,7 @@ def delete(
             )
     except Exception as e:
         if isinstance(e, RuntimeError) and isinstance(e.args[0], tuple):
-            status, msg, url = e.args[0]
+            status, msg, _ = e.args[0]
             if status == 404:
                 console.print(
                     Panel(
@@ -492,6 +526,11 @@ def deploy(
         "--env",
         help="Environment variable(s) to set (format: KEY=VALUE). Note: These environment variables will be set on ALL services in your Dynamo pipeline.",
     ),
+    envs_from_secret: t.Optional[t.List[str]] = typer.Option(
+        None,
+        "--env-from-secret",
+        help="Environment variable(s) from secret (format: KEY=SECRET_KEY). These will be set from your Dynamo secrets.",
+    ),
     target: str = typer.Option(
         DeploymentTargetType.KUBERNETES.value,
         "--target",
@@ -499,8 +538,25 @@ def deploy(
         help="Deployment target",
     ),
     dev: bool = typer.Option(False, "--dev", help="Development mode for deployment"),
+    env_secrets_name: t.Optional[str] = typer.Option(
+        "dynamo-env-secrets",
+        "--env-secrets-name",
+        help="Environment secrets name",
+        envvar="DYNAMO_ENV_SECRETS",
+    ),
 ) -> DeploymentResponse:
     """Deploy a Dynamo pipeline (same as deployment create)."""
-    return _handle_deploy_create(
-        ctx, pipeline, name, config_file, wait, timeout, endpoint, envs, target, dev
+    config = DeploymentConfig(
+        pipeline=pipeline,
+        endpoint=endpoint,
+        name=name,
+        config_file=config_file,
+        wait=wait,
+        timeout=timeout,
+        envs=envs,
+        envs_from_secret=envs_from_secret,
+        target=target,
+        dev=dev,
+        env_secrets_name=env_secrets_name,
     )
+    return _handle_deploy_create(ctx, config)

deploy/sdk/src/dynamo/sdk/core/protocol/deployment.py

@@ -18,6 +18,8 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from enum import Enum
 
+import typer
+
 
 @dataclass
 class Resources:
@@ -137,13 +139,39 @@ class Deployment:
     namespace: str
     pipeline: t.Optional[str] = None
     entry_service: t.Optional[Service] = None
-    envs: t.Optional[t.List[dict]] = None
+    envs: t.Optional[t.List[t.Dict[str, t.Any]]] = None
 
 
 # Type alias for deployment responses (e.g., from backend APIs)
 DeploymentResponse = t.Dict[str, t.Any]
 
 
+@dataclass
+class DeploymentConfig:
+    """Configuration object for deployment operations.
+
+    Consolidates all deployment parameters including pipeline configuration,
+    environment variables, and deployment settings.
+    """
+
+    # Core deployment settings
+    pipeline: str
+    endpoint: str
+    name: t.Optional[str] = None
+    target: str = "kubernetes"
+    dev: bool = False
+
+    # Configuration and timing
+    config_file: t.Optional[typer.FileText] = None
+    wait: bool = True
+    timeout: int = 3600
+
+    # Environment variables
+    envs: t.Optional[t.List[str]] = None
+    envs_from_secret: t.Optional[t.List[str]] = None
+    env_secrets_name: t.Optional[str] = "dynamo-env-secrets"
+
+
 class DeploymentManager(ABC):
     """Interface for managing dynamo graph deployments."""
 

docs/guides/dynamo_deploy/operator_deployment.md

@@ -188,3 +188,40 @@ This demonstrates the service pipeline:
 1. The Frontend receives "test"
 2. The Middle service adds "-mid" to create "test-mid"
 3. The Backend service adds "-back" to create "test-mid-back"
+
+## Using Kubernetes Secrets for Environment Variables
+
+Dynamo supports securely injecting environment variables from Kubernetes secrets into your deployment. This is only supported when deploying with `--target kubernetes`.
+
+### Creating a Secret
+
+First, create a Kubernetes secret containing your sensitive values:
+
+```bash
+export HF_TOKEN=your_hf_token
+kubectl create secret generic dynamo-env-secrets \
+  --from-literal=huggingface.token=$HF_TOKEN \
+  --from-literal=another_secret.key=value \
+  -n $KUBE_NS
+```
+
+### Referencing Secrets in Your Deployment
+
+You can reference secret keys in your deployment using the `--env-from-secret` flag:
+
+- `--env-from-secret HF_TOKEN=huggingface.token` will set the `HF_TOKEN` environment variable from the `huggingface.token` key in the secret.
+- `--env-from-secret ANOTHER_SECRET=another_secret.key` will set the `ANOTHER_SECRET` environment variable from the same-named key in the secret.
+- You can also mix normal envs: `--env NORMAL_ENV_KEY=value`.
+
+By default, Dynamo will look for a secret named `dynamo-env-secrets`. You can override this with the `--env-secrets-name` flag or the `DYNAMO_ENV_SECRETS` environment variable.
+
+### Example Full Command
+
+```bash
+dynamo deploy $DYNAMO_TAG -n $DEPLOYMENT_NAME -f ./configs/agg.yaml \
+  --env NORMAL_ENV_KEY=value \
+  --env-from-secret HF_TOKEN=huggingface.token \
+  --env-from-secret ANOTHER_SECRET=another_secret.key \
+  --target kubernetes
+```
+

examples/llm/README.md

@@ -219,6 +219,8 @@ export DEPLOYMENT_NAME=llm-agg
 dynamo deployment create $DYNAMO_TAG -n $DEPLOYMENT_NAME -f ./configs/agg.yaml
 ```
 
+**Note**: To avoid rate limiting from unauthenticated requests to HuggingFace (HF), you can provide your `HF_TOKEN` as a secret in your deployment. See the [operator deployment guide](../../docs/guides/dynamo_deploy/operator_deployment.md#referencing-secrets-in-your-deployment) for instructions on referencing secrets like `HF_TOKEN` in your deployment configuration.
+
 **Note**: Optionally add `--Planner.no-operation=false` at the end of the deployment command to enable the planner component to take scaling actions on your deployment.
 
 ### Testing the Deployment
@@ -248,4 +250,4 @@ curl localhost:8000/v1/chat/completions \
   }'
 ```
 
-For more details on managing deployments, testing, and troubleshooting, please refer to the [Operator Deployment Guide](../../docs/guides/dynamo_deploy/operator_deployment.md).
+For more details on managing deployments, testing, and troubleshooting, please refer to the [Operator Deployment Guide](../../docs/guides/dynamo_deploy/operator_deployment.md).
\ No newline at end of file

examples/llm/configs/disagg.yaml

@@ -48,4 +48,4 @@ PrefillWorker:
 
 Planner:
   environment: local
-  no-operation: true
+  no-operation: true
\ No newline at end of file