Skip to content

GitLab

"deploy/operator/cmd/main.go" did not exist on "9b893c9340daa1e6b55c5dae34153964ecd4316e"
Unverified Commit 60975b51 authored by Julien Mancuso's avatar Julien Mancuso Committed by GitHub
Browse files

feat: improve CRD documentation (#3504) 


Signed-off-by: default avatarJulien Mancuso <jmancuso@nvidia.com>
parent c3fcfdd6
Show whitespace changes
Inline Side-by-side
Showing with 1332 additions and 1063 deletions
deploy/cloud/operator/Makefile
View file @ 60975b51
......@@ -287,10 +287,10 @@ generate-api-docs: crd-ref-docs ## Generate API reference documentation from CRD
--renderer=markdown \
--output-path=./docs/api_reference.md
@echo "✅ Generated API reference at ./docs/api_reference.md"
# concatenate header.md and api_reference.md
cat docs/header.md ./docs/api_reference.md > ../../../docs/kubernetes/api_reference.md
# concatenate header.md, api_reference.md, and footer.md
cat docs/header.md ./docs/api_reference.md docs/footer.md > ../../../docs/kubernetes/api_reference.md
rm ./docs/api_reference.md
@echo "✅ Concatenated header.md and api_reference.md"
@echo "✅ Concatenated header.md, api_reference.md, and footer.md"
.PHONY: coverage
coverage: test
......
deploy/cloud/operator/api/v1alpha1/dynamocomponentdeployment_types.go
View file @ 60975b51
......@@ -20,8 +20,6 @@
package v1alpha1
import (
"strings"
dynamoCommon "github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/common"
commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
corev1 "k8s.io/api/core/v1"
......@@ -38,12 +36,6 @@ const (
// DynamoComponentDeploymentSpec defines the desired state of DynamoComponentDeployment
type DynamoComponentDeploymentSpec struct {
// DynamoComponent selects the Dynamo component from the archive to deploy.
// Typically corresponds to a component defined in the packaged Dynamo artifacts.
DynamoComponent string `json:"dynamoComponent,omitempty"`
// contains the tag of the DynamoComponent: for example, "my_package:MyService"
DynamoTag string `json:"dynamoTag,omitempty"`
// BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm")
// +kubebuilder:validation:Enum=sglang;vllm;trtllm
BackendFramework string `json:"backendFramework,omitempty"`
......@@ -53,10 +45,6 @@ type DynamoComponentDeploymentSpec struct {
DynamoComponentDeploymentSharedSpec `json:",inline"`
}
type DynamoComponentDeploymentOverridesSpec struct {
DynamoComponentDeploymentSharedSpec `json:",inline"`
}
type DynamoComponentDeploymentSharedSpec struct {
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
// Important: Run "make" to regenerate code after modifying this file
......@@ -229,7 +217,7 @@ func (s *DynamoComponentDeployment) SetSpec(spec any) {
}
func (s *DynamoComponentDeployment) IsFrontendComponent() bool {
return strings.HasSuffix(s.Spec.DynamoTag, s.Spec.ServiceName) || s.Spec.ComponentType == commonconsts.ComponentTypeFrontend
return s.Spec.ComponentType == commonconsts.ComponentTypeFrontend
}
func (s *DynamoComponentDeployment) GetDynamoDeploymentConfig() []byte {
......
deploy/cloud/operator/api/v1alpha1/dynamocomponentdeployment_types_test.go
View file @ 60975b51
......@@ -46,9 +46,8 @@ func TestDynamoComponentDeployment_IsFrontendComponent(t *testing.T) {
name: "main component",
fields: fields{
Spec: DynamoComponentDeploymentSpec{
DynamoTag: "dynamo-component:main",
DynamoComponentDeploymentSharedSpec: DynamoComponentDeploymentSharedSpec{
ServiceName: "main",
ComponentType: commonconsts.ComponentTypeFrontend,
},
},
},
......@@ -58,9 +57,8 @@ func TestDynamoComponentDeployment_IsFrontendComponent(t *testing.T) {
name: "not main component",
fields: fields{
Spec: DynamoComponentDeploymentSpec{
DynamoTag: "dynamo-component:main",
DynamoComponentDeploymentSharedSpec: DynamoComponentDeploymentSharedSpec{
ServiceName: "not-main",
ComponentType: commonconsts.ComponentTypeWorker,
},
},
},
......
deploy/cloud/operator/api/v1alpha1/dynamographdeployment_types.go
View file @ 60975b51
......@@ -29,20 +29,14 @@ import (
// DynamoGraphDeploymentSpec defines the desired state of DynamoGraphDeployment.
type DynamoGraphDeploymentSpec struct {
// DynamoGraph selects the graph (workflow/topology) to deploy. This must match
// a graph name packaged with the Dynamo archive.
DynamoGraph string `json:"dynamoGraph,omitempty"`
// PVCs defines a list of persistent volume claims that can be referenced by components.
// Each PVC must have a unique name that can be referenced in component specifications.
// +kubebuilder:validation:Optional
PVCs []PVC `json:"pvcs,omitempty"`
// Services allows per-service overrides of the component deployment settings.
// - key: name of the service defined by the DynamoComponent
// - value: overrides for that service
// If not set for a service, the default DynamoComponentDeployment values are used.
// Services are the services to deploy as part of this deployment.
// +kubebuilder:validation:Optional
Services map[string]*DynamoComponentDeploymentOverridesSpec `json:"services,omitempty"`
// Envs are environment variables applied to all services in the graph unless
Services map[string]*DynamoComponentDeploymentSharedSpec `json:"services,omitempty"`
// Envs are environment variables applied to all services in the deployment unless
// overridden by service-specific configuration.
// +kubebuilder:validation:Optional
Envs []corev1.EnvVar `json:"envs,omitempty"`
......
deploy/cloud/operator/api/v1alpha1/zz_generated.deepcopy.go
View file @ 60975b51
......@@ -173,22 +173,6 @@ func (in *DynamoComponentDeploymentList) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DynamoComponentDeploymentOverridesSpec) DeepCopyInto(out *DynamoComponentDeploymentOverridesSpec) {
*out = *in
in.DynamoComponentDeploymentSharedSpec.DeepCopyInto(&out.DynamoComponentDeploymentSharedSpec)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoComponentDeploymentOverridesSpec.
func (in *DynamoComponentDeploymentOverridesSpec) DeepCopy() *DynamoComponentDeploymentOverridesSpec {
if in == nil {
return nil
}
out := new(DynamoComponentDeploymentOverridesSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DynamoComponentDeploymentSharedSpec) DeepCopyInto(out *DynamoComponentDeploymentSharedSpec) {
*out = *in
......@@ -406,15 +390,15 @@ func (in *DynamoGraphDeploymentSpec) DeepCopyInto(out *DynamoGraphDeploymentSpec
}
if in.Services != nil {
in, out := &in.Services, &out.Services
*out = make(map[string]*DynamoComponentDeploymentOverridesSpec, len(*in))
*out = make(map[string]*DynamoComponentDeploymentSharedSpec, len(*in))
for key, val := range *in {
var outVal *DynamoComponentDeploymentOverridesSpec
var outVal *DynamoComponentDeploymentSharedSpec
if val == nil {
(*out)[key] = nil
} else {
inVal := (*in)[key]
in, out := &inVal, &outVal
*out = new(DynamoComponentDeploymentOverridesSpec)
*out = new(DynamoComponentDeploymentSharedSpec)
(*in).DeepCopyInto(*out)
}
(*out)[key] = outVal
......
deploy/cloud/operator/docs/crd-ref-docs-config.yaml
View file @ 60975b51
......@@ -32,7 +32,7 @@ processor:
- "ParseError$"
# Ignore only the override wrapper type to reduce repetition
# Keep SharedSpec so embedded fields are documented once
- "DynamoComponentDeploymentOverridesSpec$"
# - "DynamoComponentDeploymentOverridesSpec$"
- "DynamoComponentDeploymentStatus$"
- "BaseStatus$"
......
deploy/cloud/operator/docs/footer.md 0 → 100644
View file @ 60975b51
# Operator Default Values Injection
The Dynamo operator automatically applies default values to various fields when they are not explicitly specified in your deployments. These defaults include:
- **Health Probes**: Startup, liveness, and readiness probes are configured differently for frontend, worker, and planner components. For example, worker components receive a startup probe with a 2-hour timeout (720 failures × 10 seconds) to accommodate long model loading times.
- **Shared Memory**: All components receive an 8Gi shared memory volume mounted at `/dev/shm` by default (can be disabled or resized via the `sharedMemory` field).
- **Environment Variables**: Components automatically receive environment variables like `DYN_NAMESPACE`, `DYN_PARENT_DGD_K8S_NAME`, `DYNAMO_PORT`, and backend-specific variables.
- **Pod Configuration**: Default `terminationGracePeriodSeconds` of 60 seconds and `restartPolicy: Always`.
- **Autoscaling**: When enabled without explicit metrics, defaults to CPU-based autoscaling with 80% target utilization.
- **Backend-Specific Behavior**: For multinode deployments, probes are automatically modified or removed for worker nodes depending on the backend framework (VLLM, SGLang, or TensorRT-LLM).
## Pod Specification Defaults
All components receive the following pod-level defaults unless overridden:
- **`terminationGracePeriodSeconds`**: `60` seconds
- **`restartPolicy`**: `Always`
## Shared Memory Configuration
Shared memory is enabled by default for all components:
- **Enabled**: `true` (unless explicitly disabled via `sharedMemory.disabled`)
- **Size**: `8Gi`
- **Mount Path**: `/dev/shm`
- **Volume Type**: `emptyDir` with `memory` medium
To disable shared memory or customize the size, use the `sharedMemory` field in your component specification.
## Health Probes by Component Type
The operator applies different default health probes based on the component type.
### Frontend Components
Frontend components receive the following probe configurations:
**Liveness Probe:**
- **Type**: HTTP GET
- **Path**: `/health`
- **Port**: `http` (8000)
- **Initial Delay**: 60 seconds
- **Period**: 60 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 10
**Readiness Probe:**
- **Type**: Exec command
- **Command**: `curl -s http://localhost:${DYNAMO_PORT}/health | jq -e ".status == \"healthy\""`
- **Initial Delay**: 60 seconds
- **Period**: 60 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 10
### Worker Components
Worker components receive the following probe configurations:
**Liveness Probe:**
- **Type**: HTTP GET
- **Path**: `/live`
- **Port**: `system` (9090)
- **Period**: 5 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 1
**Readiness Probe:**
- **Type**: HTTP GET
- **Path**: `/health`
- **Port**: `system` (9090)
- **Period**: 10 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 60
**Startup Probe:**
- **Type**: HTTP GET
- **Path**: `/live`
- **Port**: `system` (9090)
- **Period**: 10 seconds
- **Timeout**: 5 seconds
- **Failure Threshold**: 720 (allows up to 2 hours for startup: 10s × 720 = 7200s)
:::{note}
For larger models (typically >70B parameters) or slower storage systems, you may need to increase the `failureThreshold` to allow more time for model loading. Calculate the required threshold based on your expected startup time: `failureThreshold = (expected_startup_seconds / period)`. Override the startup probe in your component specification if the default 2-hour window is insufficient.
:::
### Multinode Deployment Probe Modifications
For multinode deployments, the operator modifies probes based on the backend framework and node role:
#### VLLM Backend
- **Worker nodes**: All probes (liveness, readiness, startup) are removed
#### SGLang Backend
- **Worker nodes**: All probes (liveness, readiness, startup) are removed
#### TensorRT-LLM Backend
- **Leader nodes**: All probes remain unchanged
- **Worker nodes**:
- Liveness and startup probes are removed
- Readiness probe is replaced with a TCP socket check on SSH port (2222):
- **Initial Delay**: 20 seconds
- **Period**: 20 seconds
- **Timeout**: 5 seconds
- **Failure Threshold**: 10
## Environment Variables
The operator automatically injects environment variables based on component type and configuration:
### All Components
- **`DYN_NAMESPACE`**: The Dynamo namespace for the component
- **`DYN_PARENT_DGD_K8S_NAME`**: The parent DynamoGraphDeployment Kubernetes resource name
- **`DYN_PARENT_DGD_K8S_NAMESPACE`**: The parent DynamoGraphDeployment Kubernetes namespace
### Frontend Components
- **`DYNAMO_PORT`**: `8000`
- **`DYN_HTTP_PORT`**: `8000`
### Worker Components
- **`DYN_SYSTEM_ENABLED`**: `true`
- **`DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS`**: `["generate"]`
- **`DYN_SYSTEM_PORT`**: `9090`
### Planner Components
- **`PLANNER_PROMETHEUS_PORT`**: `9085`
### VLLM Backend (with compilation cache)
When a volume mount is configured with `useAsCompilationCache: true`:
- **`VLLM_CACHE_ROOT`**: Set to the mount point of the cache volume
## Service Account
Planner components automatically receive the following service account:
- **`serviceAccountName`**: `planner-serviceaccount`
## Image Pull Secrets
The operator automatically discovers and injects image pull secrets for container images. When a component specifies a container image, the operator:
1. Scans all Kubernetes secrets of type `kubernetes.io/dockerconfigjson` in the component's namespace
2. Extracts the docker registry server URLs from each secret's authentication configuration
3. Matches the container image's registry host against the discovered registry URLs
4. Automatically injects matching secrets as `imagePullSecrets` in the pod specification
This eliminates the need to manually specify image pull secrets for each component. The operator maintains an internal index of docker secrets and their associated registries, refreshing this index periodically.
**To disable automatic image pull secret discovery** for a specific component, add the following annotation:
```yaml
annotations:
nvidia.com/disable-image-pull-secret-discovery: "true"
```
## Autoscaling Defaults
When autoscaling is enabled but no metrics are specified, the operator applies:
- **Default Metric**: CPU utilization
- **Target Average Utilization**: `80%`
## Port Configurations
Default container ports are configured based on component type:
### Frontend Components
- **Port**: 8000
- **Protocol**: TCP
- **Name**: `http`
### Worker Components
- **Port**: 9090
- **Protocol**: TCP
- **Name**: `system`
### Planner Components
- **Port**: 9085
- **Protocol**: TCP
- **Name**: `metrics`
## Backend-Specific Configurations
### VLLM
- **Ray Head Port**: 6379 (for multinode deployments)
### SGLang
- **Distribution Init Port**: 29500 (for multinode deployments)
### TensorRT-LLM
- **SSH Port**: 2222 (for multinode MPI communication)
- **OpenMPI Environment**: `OMPI_MCA_orte_keep_fqdn_hostnames=1`
## Implementation Reference
For users who want to understand the implementation details or contribute to the operator, the default values described in this document are set in the following source files:
- **Health Probes & Pod Specifications**: [`internal/dynamo/graph.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/graph.go) - Contains the main logic for applying default probes, environment variables, shared memory, and pod configurations
- **Component-Specific Defaults**:
- [`internal/dynamo/component_frontend.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/component_frontend.go)
- [`internal/dynamo/component_worker.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/component_worker.go)
- [`internal/dynamo/component_planner.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/component_planner.go)
- **Image Pull Secrets**: [`internal/secrets/docker.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/secrets/docker.go) - Implements the docker secret indexer and automatic discovery
- **Backend-Specific Behavior**:
- [`internal/dynamo/backend_vllm.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/backend_vllm.go)
- [`internal/dynamo/backend_sglang.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/backend_sglang.go)
- [`internal/dynamo/backend_trtllm.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/backend_trtllm.go)
- **Constants & Annotations**: [`internal/consts/consts.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/consts/consts.go) - Defines annotation keys and other constants
## Notes
- All these defaults can be overridden by explicitly specifying values in your DynamoComponentDeployment or DynamoGraphDeployment resources
- User-specified probes (via `livenessProbe`, `readinessProbe`, or `startupProbe` fields) take precedence over operator defaults
- For multinode deployments, some defaults are modified or removed as described above to accommodate distributed execution patterns
- The `extraPodSpec.mainContainer` field can be used to override probe configurations set by the operator
\ No newline at end of file
deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller_test.go
View file @ 60975b51
......@@ -688,8 +688,6 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
},
},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
DynamoComponent: "test-lws-component",
DynamoTag: "test-tag",
BackendFramework: string(dynamo.BackendFrameworkVLLM),
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Envs: []corev1.EnvVar{
......@@ -990,7 +988,6 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{Name: "test-lws-nil-id", Namespace: "default"},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
DynamoComponent: "test-comp", DynamoTag: "test",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Multinode: &v1alpha1.MultinodeSpec{
NodeCount: 2,
......@@ -1034,7 +1031,6 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{Name: "test-lws-leader-err", Namespace: "default"},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
DynamoComponent: "test-comp", DynamoTag: "test",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Multinode: &v1alpha1.MultinodeSpec{
NodeCount: 2,
......
deploy/cloud/operator/internal/dynamo/backend_sglang.go
View file @ 60975b51
......@@ -27,7 +27,7 @@ func isPythonCommand(cmd string) bool {
return matched
}
func (b *SGLangBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
func (b *SGLangBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
// Check for volumeMounts with useAsCompilationCache=true
for _, volumeMount := range component.VolumeMounts {
if volumeMount.UseAsCompilationCache {
......@@ -112,7 +112,7 @@ func (b *SGLangBackend) UpdateContainer(container *corev1.Container, numberOfNod
}
}
func (b *SGLangBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string) {
func (b *SGLangBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string) {
// do nothing
}
......
deploy/cloud/operator/internal/dynamo/backend_sglang_test.go
View file @ 60975b51
......@@ -192,7 +192,7 @@ func TestSGLangBackend_PythonCommandInjection(t *testing.T) {
Args: append([]string{}, tt.initialArgs...),
}
backend.UpdateContainer(container, tt.numberOfNodes, tt.role, &v1alpha1.DynamoComponentDeploymentOverridesSpec{}, "test-service", tt.multinodeDeployer)
backend.UpdateContainer(container, tt.numberOfNodes, tt.role, &v1alpha1.DynamoComponentDeploymentSharedSpec{}, "test-service", tt.multinodeDeployer)
if !reflect.DeepEqual(container.Command, tt.expectedCommand) {
t.Errorf("UpdateContainer() command = %v, want %v", container.Command, tt.expectedCommand)
......@@ -317,7 +317,7 @@ func TestSGLangBackend_ShellCommandInjection(t *testing.T) {
Args: append([]string{}, tt.initialArgs...),
}
backend.UpdateContainer(container, tt.numberOfNodes, tt.role, &v1alpha1.DynamoComponentDeploymentOverridesSpec{}, "test-service", tt.multinodeDeployer)
backend.UpdateContainer(container, tt.numberOfNodes, tt.role, &v1alpha1.DynamoComponentDeploymentSharedSpec{}, "test-service", tt.multinodeDeployer)
if !reflect.DeepEqual(container.Args, tt.expectedArgs) {
t.Errorf("UpdateContainer() args = %v, want %v", container.Args, tt.expectedArgs)
......@@ -491,7 +491,7 @@ func TestSGLangBackend_ProbeRemoval(t *testing.T) {
StartupProbe: startupProbe,
}
backend.UpdateContainer(container, tt.numberOfNodes, tt.role, &v1alpha1.DynamoComponentDeploymentOverridesSpec{}, "test-service", tt.multinodeDeployer)
backend.UpdateContainer(container, tt.numberOfNodes, tt.role, &v1alpha1.DynamoComponentDeploymentSharedSpec{}, "test-service", tt.multinodeDeployer)
if tt.expectProbesRemoved {
if container.LivenessProbe != nil {
......@@ -523,15 +523,14 @@ func TestSGLangBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
volumeMounts []corev1.VolumeMount
expectNoEnvVarChanges bool
expectLoggedPartialSupport bool
}{
{
name: "SGLang backend with useAsCompilationCache volume mount",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "sglang-cache",
......@@ -540,15 +539,13 @@ func TestSGLangBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true, // SGLang doesn't set env vars yet
expectLoggedPartialSupport: true,
},
{
name: "SGLang backend with useAsCompilationCache at custom volume mount",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "custom-cache",
......@@ -557,15 +554,13 @@ func TestSGLangBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true, // SGLang doesn't set env vars yet
expectLoggedPartialSupport: true,
},
{
name: "SGLang backend without useAsCompilationCache volume mount",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "regular-volume",
......@@ -573,18 +568,15 @@ func TestSGLangBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true,
expectLoggedPartialSupport: false,
},
{
name: "SGLang backend with no volume mounts",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: nil,
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true,
expectLoggedPartialSupport: false,
......
deploy/cloud/operator/internal/dynamo/backend_trtllm.go
View file @ 60975b51
......@@ -18,7 +18,7 @@ type TRTLLMBackend struct {
MpiRunSecretName string
}
func (b *TRTLLMBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
func (b *TRTLLMBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
// Check for volumeMounts with useAsCompilationCache=true
for _, volumeMount := range component.VolumeMounts {
if volumeMount.UseAsCompilationCache {
......@@ -75,7 +75,7 @@ func (b *TRTLLMBackend) UpdateContainer(container *corev1.Container, numberOfNod
}
}
func (b *TRTLLMBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string) {
func (b *TRTLLMBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string) {
// Add SSH keypair volume for TRTLLM multinode deployments
if numberOfNodes > 1 {
sshVolume := corev1.Volume{
......@@ -102,7 +102,7 @@ func (b *TRTLLMBackend) addSSHVolumeMount(container *corev1.Container) {
}
// setupLeaderContainer configures the leader node with SSH setup and mpirun command
func (b *TRTLLMBackend) setupLeaderContainer(container *corev1.Container, numberOfNodes int32, serviceName string, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, multinodeDeployer MultinodeDeployer) {
func (b *TRTLLMBackend) setupLeaderContainer(container *corev1.Container, numberOfNodes int32, serviceName string, component *v1alpha1.DynamoComponentDeploymentSharedSpec, multinodeDeployer MultinodeDeployer) {
// Generate the list of worker hostnames
workerHosts := b.generateWorkerHostnames(numberOfNodes, serviceName, multinodeDeployer)
......
deploy/cloud/operator/internal/dynamo/backend_trtllm_test.go
View file @ 60975b51
......@@ -21,7 +21,7 @@ func TestTRTLLMBackend_UpdateContainer(t *testing.T) {
numberOfNodes int32
role Role
multinodeDeployer MultinodeDeployer
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
expectedVolumeMounts []corev1.VolumeMount
expectedCommand []string
expectedArgs []string
......@@ -36,7 +36,7 @@ func TestTRTLLMBackend_UpdateContainer(t *testing.T) {
numberOfNodes: 1,
role: RoleMain,
multinodeDeployer: &GroveMultinodeDeployer{},
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
expectedVolumeMounts: []corev1.VolumeMount{},
expectedCommand: []string{},
expectedArgs: []string{"python3", "--model", "test"},
......@@ -51,15 +51,13 @@ func TestTRTLLMBackend_UpdateContainer(t *testing.T) {
numberOfNodes: 3,
role: RoleLeader,
multinodeDeployer: &GroveMultinodeDeployer{},
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Requests: &common.ResourceItem{
GPU: "2",
},
},
},
},
expectedVolumeMounts: []corev1.VolumeMount{
{Name: mpiRunSecretName, MountPath: "/ssh-pk", ReadOnly: true},
},
......@@ -78,7 +76,7 @@ func TestTRTLLMBackend_UpdateContainer(t *testing.T) {
numberOfNodes: 3,
role: RoleWorker,
multinodeDeployer: &GroveMultinodeDeployer{},
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
expectedVolumeMounts: []corev1.VolumeMount{
{Name: mpiRunSecretName, MountPath: "/ssh-pk", ReadOnly: true},
},
......@@ -107,15 +105,13 @@ func TestTRTLLMBackend_UpdateContainer(t *testing.T) {
numberOfNodes: 2,
role: RoleLeader,
multinodeDeployer: &LWSMultinodeDeployer{},
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Limits: &common.ResourceItem{
GPU: "1",
},
},
},
},
expectedVolumeMounts: []corev1.VolumeMount{
{Name: mpiRunSecretName, MountPath: "/ssh-pk", ReadOnly: true},
},
......@@ -350,7 +346,7 @@ func TestTRTLLMBackend_UpdatePodSpec(t *testing.T) {
},
},
}
component := &v1alpha1.DynamoComponentDeploymentOverridesSpec{}
component := &v1alpha1.DynamoComponentDeploymentSharedSpec{}
// Call UpdatePodSpec
backend.UpdatePodSpec(podSpec, tt.numberOfNodes, tt.role, component, "test-service")
......@@ -550,7 +546,7 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes int32
multinodeDeployer MultinodeDeployer
serviceName string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
initialArgs []string
initialCommand []string
expected string
......@@ -560,15 +556,13 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 3,
multinodeDeployer: &GroveMultinodeDeployer{},
serviceName: "test-service",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Requests: &common.ResourceItem{
GPU: "2",
},
},
},
},
initialArgs: []string{"python3", "--model", "test"},
initialCommand: []string{},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 6 -H $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-service-ldr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-service-wkr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-service-wkr-1.$(GROVE_HEADLESS_SERVICE) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch python3 --model test'",
......@@ -578,7 +572,7 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 2,
multinodeDeployer: &LWSMultinodeDeployer{},
serviceName: "worker",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
initialArgs: []string{},
initialCommand: []string{"python", "-m", "worker"},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 0 -H $(LWS_LEADER_ADDRESS),$(LWS_WORKER_1_ADDRESS) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch python -m worker'",
......@@ -588,15 +582,13 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 2,
multinodeDeployer: &GroveMultinodeDeployer{},
serviceName: "test",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Limits: &common.ResourceItem{
GPU: "1",
},
},
},
},
initialArgs: []string{"launch", "--config", "test.yaml"},
initialCommand: []string{"sh", "-c"},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 2 -H $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-ldr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-wkr-0.$(GROVE_HEADLESS_SERVICE) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch launch --config test.yaml'",
......@@ -606,15 +598,13 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 2,
multinodeDeployer: &GroveMultinodeDeployer{},
serviceName: "test",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Limits: &common.ResourceItem{
GPU: "1",
},
},
},
},
initialArgs: []string{"-m", "dynamo.trtllm", "--model-path", "test"},
initialCommand: []string{"python3"},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 2 -H $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-ldr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-wkr-0.$(GROVE_HEADLESS_SERVICE) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch python3 -m dynamo.trtllm --model-path test'",
......@@ -624,15 +614,13 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 2,
multinodeDeployer: &GroveMultinodeDeployer{},
serviceName: "test",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Limits: &common.ResourceItem{
GPU: "1",
},
},
},
},
initialArgs: []string{"--model-path", "Qwen/Qwen3-0.6B", "--served-model-name", "Qwen/Qwen3-0.6B", "--disaggregation-mode", "prefill"},
initialCommand: []string{"python3", "-m", "dynamo.trtllm"},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 2 -H $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-ldr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-wkr-0.$(GROVE_HEADLESS_SERVICE) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch python3 -m dynamo.trtllm --model-path Qwen/Qwen3-0.6B --served-model-name Qwen/Qwen3-0.6B --disaggregation-mode prefill'",
......@@ -642,15 +630,13 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 2,
multinodeDeployer: &GroveMultinodeDeployer{},
serviceName: "test",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Limits: &common.ResourceItem{
GPU: "1",
},
},
},
},
initialArgs: []string{"-m", "dynamo.trtllm", "--model-path", "test"},
initialCommand: []string{"/usr/bin/python3.8"},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 2 -H $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-ldr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-wkr-0.$(GROVE_HEADLESS_SERVICE) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch /usr/bin/python3.8 -m dynamo.trtllm --model-path test'",
......@@ -660,15 +646,13 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 2,
multinodeDeployer: &GroveMultinodeDeployer{},
serviceName: "test",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Requests: &common.ResourceItem{
GPU: "1",
},
},
},
},
initialArgs: []string{"serve", "--model", "test"},
initialCommand: []string{},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 2 -H $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-ldr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-wkr-0.$(GROVE_HEADLESS_SERVICE) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch serve --model test'",
......@@ -678,15 +662,13 @@ func TestTRTLLMBackend_setupLeaderContainer(t *testing.T) {
numberOfNodes: 2,
multinodeDeployer: &GroveMultinodeDeployer{},
serviceName: "test",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Resources: &common.Resources{
Requests: &common.ResourceItem{
GPU: "1",
},
},
},
},
initialArgs: []string{"serve", "--model", "test"},
initialCommand: []string{},
expected: "mkdir -p ~/.ssh && ls -la /ssh-pk/ && cp /ssh-pk/private.key ~/.ssh/id_rsa && cp /ssh-pk/private.key.pub ~/.ssh/id_rsa.pub && cp /ssh-pk/private.key.pub ~/.ssh/authorized_keys && chmod 600 ~/.ssh/id_rsa ~/.ssh/authorized_keys && chmod 644 ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && printf 'Host *\\nIdentityFile ~/.ssh/id_rsa\\nStrictHostKeyChecking no\\nPort 2222\\n' > ~/.ssh/config && mpirun --oversubscribe -n 2 -H $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-ldr-0.$(GROVE_HEADLESS_SERVICE),$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-wkr-0.$(GROVE_HEADLESS_SERVICE) --mca pml ob1 --mca plm_rsh_args \"-p 2222 -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa\" -x CUDA_VISIBLE_DEVICES -x CUSTOM_VAR -x HF_DATASETS_CACHE -x HF_ENDPOINT -x HF_HOME -x HF_TOKEN -x HOME -x HUGGING_FACE_HUB_TOKEN -x LD_LIBRARY_PATH -x MODEL_PATH -x NCCL_DEBUG -x NCCL_IB_DISABLE -x NCCL_P2P_DISABLE -x PATH -x PYTHONPATH -x TENSORRT_LLM_CACHE_DIR -x TOKENIZERS_PARALLELISM -x TRANSFORMERS_CACHE -x USER bash -c 'source /opt/dynamo/venv/bin/activate && trtllm-llmapi-launch serve --model test'",
......@@ -874,15 +856,14 @@ func TestTRTLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
volumeMounts []corev1.VolumeMount
expectNoEnvVarChanges bool
expectLoggedPartialSupport bool
}{
{
name: "TensorRT-LLM backend with useAsCompilationCache volume mount",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "trtllm-cache",
......@@ -891,15 +872,13 @@ func TestTRTLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true, // TensorRT-LLM doesn't set env vars yet
expectLoggedPartialSupport: true,
},
{
name: "TensorRT-LLM backend with useAsCompilationCache at custom mount point",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "custom-cache",
......@@ -908,15 +887,13 @@ func TestTRTLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true, // TensorRT-LLM doesn't set env vars yet
expectLoggedPartialSupport: true,
},
{
name: "TensorRT-LLM backend without useAsCompilationCache",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "regular-volume",
......@@ -924,18 +901,15 @@ func TestTRTLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true,
expectLoggedPartialSupport: false,
},
{
name: "TensorRT-LLM backend with no volume mounts",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: nil,
},
},
volumeMounts: []corev1.VolumeMount{},
expectNoEnvVarChanges: true,
expectLoggedPartialSupport: false,
......
deploy/cloud/operator/internal/dynamo/backend_vllm.go
View file @ 60975b51
......@@ -15,7 +15,7 @@ const (
type VLLMBackend struct{}
func (b *VLLMBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
func (b *VLLMBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
isMultinode := numberOfNodes > 1
if isMultinode {
......@@ -60,7 +60,7 @@ func (b *VLLMBackend) UpdateContainer(container *corev1.Container, numberOfNodes
}
}
func (b *VLLMBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string) {
func (b *VLLMBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string) {
// do nothing
}
......
deploy/cloud/operator/internal/dynamo/backend_vllm_test.go
View file @ 60975b51
......@@ -16,7 +16,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
name string
numberOfNodes int32
role Role
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
multinodeDeployer MultinodeDeployer
initialArgs []string
initialLivenessProbe *corev1.Probe
......@@ -31,7 +31,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
name: "single node does not modify args",
numberOfNodes: 1,
role: RoleMain,
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialArgs: []string{"python3", "-m", "dynamo.vllm"},
expectNotModified: true,
......@@ -40,7 +40,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
name: "multinode leader prepends ray start --head",
numberOfNodes: 3,
role: RoleLeader,
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialArgs: []string{"python3", "-m", "dynamo.vllm", "--model", "test"},
expectContains: []string{"ray start --head --port=6379 &&", "python3", "-m", "dynamo.vllm", "--model", "test"},
......@@ -50,7 +50,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
name: "multinode worker replaces args with ray start --block",
numberOfNodes: 3,
role: RoleWorker,
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialArgs: []string{"python3", "-m", "dynamo.vllm", "--model", "test"},
expectedArgs: []string{"ray start --address=$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-test-service-ldr-0.$(GROVE_HEADLESS_SERVICE):6379 --block"},
......@@ -60,7 +60,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
name: "multinode worker with LWS deployment type",
numberOfNodes: 2,
role: RoleWorker,
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &LWSMultinodeDeployer{},
initialArgs: []string{"python3", "-m", "dynamo.vllm"},
expectedArgs: []string{"ray start --address=$(LWS_LEADER_ADDRESS):6379 --block"},
......@@ -70,7 +70,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
name: "multinode leader with no initial args",
numberOfNodes: 2,
role: RoleLeader,
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialArgs: []string{},
expectNotModified: true, // Should not modify empty args
......@@ -79,7 +79,7 @@ func TestVLLMBackend_UpdateContainer(t *testing.T) {
name: "multinode main role (non-leader/worker) does not modify args",
numberOfNodes: 3,
role: RoleMain,
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{},
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{},
multinodeDeployer: &GroveMultinodeDeployer{},
initialArgs: []string{"python3", "-m", "dynamo.frontend"},
expectNotModified: true,
......@@ -131,7 +131,7 @@ func TestVLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
volumeMounts []corev1.VolumeMount
expectCacheEnvVar bool
expectCacheEnvVarName string
......@@ -139,8 +139,7 @@ func TestVLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
}{
{
name: "VLLM backend with useAsCompilationCache volume mount",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "vllm-cache",
......@@ -149,7 +148,6 @@ func TestVLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectCacheEnvVar: true,
expectCacheEnvVarName: "VLLM_CACHE_ROOT",
......@@ -157,8 +155,7 @@ func TestVLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
{
name: "VLLM backend with useAsCompilationCache at custom mount point",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "custom-cache",
......@@ -167,7 +164,6 @@ func TestVLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectCacheEnvVar: true,
expectCacheEnvVarName: "VLLM_CACHE_ROOT",
......@@ -175,8 +171,7 @@ func TestVLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
{
name: "VLLM backend without useAsCompilationCache",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: []v1alpha1.VolumeMount{
{
Name: "regular-volume",
......@@ -184,17 +179,14 @@ func TestVLLMBackend_UpdateContainer_UseAsCompilationCache(t *testing.T) {
},
},
},
},
volumeMounts: []corev1.VolumeMount{},
expectCacheEnvVar: false,
},
{
name: "VLLM backend with no volume mounts",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
VolumeMounts: nil,
},
},
volumeMounts: []corev1.VolumeMount{},
expectCacheEnvVar: false,
},
......
deploy/cloud/operator/internal/dynamo/graph.go
View file @ 60975b51
......@@ -125,7 +125,7 @@ func GenerateDynamoComponentsDeployments(ctx context.Context, parentDynamoGraphD
}
for componentName, component := range parentDynamoGraphDeployment.Spec.Services {
deployment := &v1alpha1.DynamoComponentDeployment{}
deployment.Spec.DynamoComponentDeploymentSharedSpec = component.DynamoComponentDeploymentSharedSpec
deployment.Spec.DynamoComponentDeploymentSharedSpec = *component
deployment.Name = GetDynamoComponentName(parentDynamoGraphDeployment, componentName)
deployment.Spec.BackendFramework = parentDynamoGraphDeployment.Spec.BackendFramework
deployment.Namespace = parentDynamoGraphDeployment.Namespace
......@@ -597,18 +597,18 @@ const (
// Backend interface for modular backend logic
// Each backend (SGLang, VLLM, etc.) implements this interface
type Backend interface {
UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string, multinodeDeployer MultinodeDeployer)
UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string)
UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string, multinodeDeployer MultinodeDeployer)
UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string)
}
// NoopBackend does no processing - used for non-worker components like frontend, planner, router
type NoopBackend struct{}
func (b *NoopBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
func (b *NoopBackend) UpdateContainer(container *corev1.Container, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string, multinodeDeployer MultinodeDeployer) {
// No-op: frontend, planner, router, etc. don't need backend-specific processing
}
func (b *NoopBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentOverridesSpec, serviceName string) {
func (b *NoopBackend) UpdatePodSpec(podSpec *corev1.PodSpec, numberOfNodes int32, role Role, component *v1alpha1.DynamoComponentDeploymentSharedSpec, serviceName string) {
// No-op: frontend, planner, router, etc. don't need backend-specific processing
}
......@@ -691,7 +691,7 @@ func addStandardEnvVars(container *corev1.Container, controllerConfig controller
//
//nolint:gocyclo
func GenerateBasePodSpec(
component *v1alpha1.DynamoComponentDeploymentOverridesSpec,
component *v1alpha1.DynamoComponentDeploymentSharedSpec,
backendFramework BackendFramework,
secretsRetriever SecretsRetriever,
parentGraphDeploymentName string,
......@@ -861,7 +861,7 @@ func setMetricsLabels(labels map[string]string, dynamoGraphDeployment *v1alpha1.
labels[commonconsts.KubeLabelMetricsEnabled] = commonconsts.KubeLabelValueTrue
}
func generateComponentContext(component *v1alpha1.DynamoComponentDeploymentOverridesSpec, parentGraphDeploymentName string, namespace string, numberOfNodes int32) ComponentContext {
func generateComponentContext(component *v1alpha1.DynamoComponentDeploymentSharedSpec, parentGraphDeploymentName string, namespace string, numberOfNodes int32) ComponentContext {
componentContext := ComponentContext{
numberOfNodes: numberOfNodes,
ParentGraphDeploymentName: parentGraphDeploymentName,
......@@ -875,7 +875,7 @@ func generateComponentContext(component *v1alpha1.DynamoComponentDeploymentOverr
// GeneratePodSpecForComponent creates a PodSpec for Grove deployments (simplified wrapper)
func GeneratePodSpecForComponent(
component *v1alpha1.DynamoComponentDeploymentOverridesSpec,
component *v1alpha1.DynamoComponentDeploymentSharedSpec,
backendFramework BackendFramework,
secretsRetriever SecretsRetriever,
dynamoDeployment *v1alpha1.DynamoGraphDeployment,
......@@ -1004,7 +1004,7 @@ func GenerateGrovePodCliqueSet(
return controller_common.CanonicalizePodCliqueSet(gangSet), nil
}
func generateLabels(component *v1alpha1.DynamoComponentDeploymentOverridesSpec, dynamoDeployment *v1alpha1.DynamoGraphDeployment, componentName string) (map[string]string, error) {
func generateLabels(component *v1alpha1.DynamoComponentDeploymentSharedSpec, dynamoDeployment *v1alpha1.DynamoGraphDeployment, componentName string) (map[string]string, error) {
labels := make(map[string]string)
labels[commonconsts.KubeLabelDynamoSelector] = GetDynamoComponentName(dynamoDeployment, componentName)
labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dynamoDeployment.Name
......@@ -1033,7 +1033,7 @@ func generateLabels(component *v1alpha1.DynamoComponentDeploymentOverridesSpec,
return labels, nil
}
func generateAnnotations(component *v1alpha1.DynamoComponentDeploymentOverridesSpec) (map[string]string, error) {
func generateAnnotations(component *v1alpha1.DynamoComponentDeploymentSharedSpec) (map[string]string, error) {
annotations := make(map[string]string)
if component.Annotations != nil {
err := mergo.Merge(&annotations, component.Annotations, mergo.WithOverride)
......@@ -1147,7 +1147,7 @@ func determineBackendFramework(
// 3. Return error if worker has neither detection nor explicit config
// Also validates consistency between detected and explicit if both exist
func getBackendFrameworkFromComponent(
component *v1alpha1.DynamoComponentDeploymentOverridesSpec,
component *v1alpha1.DynamoComponentDeploymentSharedSpec,
dynamoDeployment *v1alpha1.DynamoGraphDeployment,
) (BackendFramework, error) {
// Extract command/args from component
......@@ -1170,10 +1170,8 @@ func getBackendFrameworkFromComponent(
// ConvertDynamoComponentDeploymentToSpec converts a DynamoComponentDeployment to our component spec interface
// This is a helper for the controller to use our backend logic
func ConvertDynamoComponentDeploymentToSpec(dynComponent *v1alpha1.DynamoComponentDeployment) *v1alpha1.DynamoComponentDeploymentOverridesSpec {
return &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: *dynComponent.Spec.DynamoComponentDeploymentSharedSpec.DeepCopy(),
}
func ConvertDynamoComponentDeploymentToSpec(dynComponent *v1alpha1.DynamoComponentDeployment) *v1alpha1.DynamoComponentDeploymentSharedSpec {
return dynComponent.Spec.DynamoComponentDeploymentSharedSpec.DeepCopy()
}
// GetBackendFrameworkFromDynamoComponent determines backend framework for a DynamoComponentDeployment
......
deploy/cloud/operator/internal/dynamo/graph_test.go
View file @ 60975b51
......@@ -59,9 +59,8 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha1.DynamoGraphDeploymentSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"service1": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: &[]string{"default"}[0],
ComponentType: "frontend",
SubComponentType: "test-sub-component",
......@@ -75,9 +74,7 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
"service2": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: &[]string{"default"}[0],
Replicas: &[]int32{3}[0],
Resources: &common.Resources{
......@@ -93,7 +90,6 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
want: map[string]*v1alpha1.DynamoComponentDeployment{
"service1": {
ObjectMeta: metav1.ObjectMeta{
......@@ -173,9 +169,8 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha1.DynamoGraphDeploymentSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"service1": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: nil,
ComponentType: "frontend",
Replicas: &[]int32{3}[0],
......@@ -188,9 +183,7 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
"service2": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: nil,
Replicas: &[]int32{3}[0],
Resources: &common.Resources{
......@@ -206,7 +199,6 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
want: map[string]*v1alpha1.DynamoComponentDeployment{
"service1": {
ObjectMeta: metav1.ObjectMeta{
......@@ -285,9 +277,8 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha1.DynamoGraphDeploymentSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"service1": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: &[]string{"default"}[0],
ComponentType: "frontend",
Replicas: &[]int32{3}[0],
......@@ -300,9 +291,7 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
"service2": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: &[]string{"another"}[0],
Replicas: &[]int32{3}[0],
Resources: &common.Resources{
......@@ -318,7 +307,6 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
want: nil,
wantErr: true,
},
......@@ -331,9 +319,8 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
Namespace: "default",
},
Spec: v1alpha1.DynamoGraphDeploymentSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"service1": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: nil,
ComponentType: "frontend",
Replicas: &[]int32{3}[0],
......@@ -346,9 +333,7 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
"service2": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: nil,
Replicas: &[]int32{3}[0],
Resources: &common.Resources{
......@@ -363,7 +348,6 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
ingressSpec: &v1alpha1.IngressSpec{
Enabled: true,
Host: "test-dynamographdeployment",
......@@ -457,9 +441,8 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
Value: `{"service1":{"port":8080,"ServiceArgs":{"Workers":2, "Resources":{"CPU":"2", "Memory":"2Gi", "GPU":"2"}}}}`,
},
},
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"service1": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: nil,
ComponentType: "frontend",
Replicas: &[]int32{3}[0],
......@@ -472,9 +455,7 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
"service2": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: nil,
Replicas: &[]int32{3}[0],
Resources: &common.Resources{
......@@ -490,7 +471,6 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
want: map[string]*v1alpha1.DynamoComponentDeployment{
"service1": {
ObjectMeta: metav1.ObjectMeta{
......@@ -588,9 +568,8 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
Spec: v1alpha1.DynamoGraphDeploymentSpec{
BackendFramework: string(BackendFrameworkSGLang),
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"service1": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: &[]string{"default"}[0],
ComponentType: "frontend",
Replicas: &[]int32{3}[0],
......@@ -609,9 +588,7 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
"service2": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
DynamoNamespace: &[]string{"default"}[0],
Replicas: &[]int32{3}[0],
Resources: &common.Resources{
......@@ -624,7 +601,6 @@ func TestGenerateDynamoComponentsDeployments(t *testing.T) {
},
},
},
},
Envs: []corev1.EnvVar{
{
Name: "TEST_ENV",
......@@ -754,9 +730,9 @@ func Test_updateDynDeploymentConfig(t *testing.T) {
args: args{
dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
Spec: v1alpha1.DynamoComponentDeploymentSpec{
DynamoTag: "graphs.agg:Frontend",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "Frontend",
ComponentType: commonconsts.ComponentTypeFrontend,
Envs: []corev1.EnvVar{
{
Name: "OTHER",
......@@ -780,7 +756,6 @@ func Test_updateDynDeploymentConfig(t *testing.T) {
args: args{
dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
Spec: v1alpha1.DynamoComponentDeploymentSpec{
DynamoTag: "graphs.agg:Frontend",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "Other",
Envs: []corev1.EnvVar{
......@@ -802,7 +777,6 @@ func Test_updateDynDeploymentConfig(t *testing.T) {
args: args{
dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
Spec: v1alpha1.DynamoComponentDeploymentSpec{
DynamoTag: "graphs.agg:Frontend",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "Frontend",
Envs: []corev1.EnvVar{
......@@ -1105,9 +1079,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
Value: "1",
},
},
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"Frontend": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "frontend", // Frontend component
SubComponentType: "test-sub-component",
ExtraPodMetadata: &common.ExtraPodMetadata{
......@@ -1178,9 +1151,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
"Planner": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Replicas: &[]int32{2}[0],
ComponentType: commonconsts.ComponentTypePlanner,
Resources: &common.Resources{
......@@ -1242,7 +1213,6 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
want: &grovev1alpha1.PodCliqueSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dynamo-graph-deployment",
......@@ -1590,9 +1560,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
BackendFramework: string(BackendFrameworkSGLang),
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"Frontend": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Replicas: &[]int32{1}[0],
Resources: &common.Resources{
Requests: &common.ResourceItem{
......@@ -1652,9 +1621,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
"worker": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Multinode: &v1alpha1.MultinodeSpec{
NodeCount: 3,
},
......@@ -1701,9 +1668,7 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
"Planner": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypePlanner,
Replicas: &[]int32{2}[0],
Resources: &common.Resources{
......@@ -1765,7 +1730,6 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
want: &grovev1alpha1.PodCliqueSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dynamo-graph-deployment",
......@@ -2371,9 +2335,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
BackendFramework: string(BackendFrameworkVLLM),
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"Frontend": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Replicas: &[]int32{1}[0],
ComponentType: commonconsts.ComponentTypeFrontend,
Resources: &common.Resources{
......@@ -2433,9 +2396,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
"worker": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Multinode: &v1alpha1.MultinodeSpec{
NodeCount: 3,
},
......@@ -2505,9 +2467,8 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
"Planner": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypePlanner,
Replicas: &[]int32{2}[0],
Resources: &common.Resources{
......@@ -2569,7 +2530,6 @@ func TestGenerateGrovePodCliqueSet(t *testing.T) {
},
},
},
},
want: &grovev1alpha1.PodCliqueSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dynamo-graph-deployment",
......@@ -3208,7 +3168,7 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
backendFramework BackendFramework
role Role
numberOfNodes int32
......@@ -3218,8 +3178,7 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
}{
{
name: "SGLang single node worker",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3227,7 +3186,6 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkSGLang,
role: RoleMain,
numberOfNodes: 1,
......@@ -3237,8 +3195,7 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
},
{
name: "SGLang multinode leader",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3246,7 +3203,6 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkSGLang,
role: RoleLeader,
numberOfNodes: 3,
......@@ -3255,8 +3211,7 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
},
{
name: "SGLang multinode worker",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3264,7 +3219,6 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkSGLang,
role: RoleWorker,
numberOfNodes: 3,
......@@ -3273,8 +3227,7 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
},
{
name: "SGLang with user command override",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
ExtraPodSpec: &common.ExtraPodSpec{
......@@ -3283,7 +3236,6 @@ func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkSGLang,
role: RoleMain,
numberOfNodes: 1,
......@@ -3362,7 +3314,7 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
backendFramework BackendFramework
role Role
numberOfNodes int32
......@@ -3372,8 +3324,7 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
}{
{
name: "VLLM single node worker",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3381,7 +3332,6 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkVLLM,
role: RoleMain,
numberOfNodes: 1,
......@@ -3391,8 +3341,7 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
},
{
name: "VLLM multinode leader",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3400,7 +3349,6 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkVLLM,
role: RoleLeader,
numberOfNodes: 3,
......@@ -3409,11 +3357,9 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
},
{
name: "VLLM multinode worker",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
},
},
backendFramework: BackendFrameworkVLLM,
role: RoleWorker,
numberOfNodes: 3,
......@@ -3423,8 +3369,7 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
},
{
name: "VLLM worker single node",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3432,7 +3377,6 @@ func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkVLLM,
role: RoleMain,
numberOfNodes: 1,
......@@ -3505,10 +3449,8 @@ func TestGeneratePodSpecForComponent_UnsupportedBackend(t *testing.T) {
}
controllerConfig := controller_common.Config{}
component := &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component := &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
},
}
tests := []struct {
......@@ -3855,7 +3797,7 @@ func TestDetermineBackendFramework(t *testing.T) {
func TestGetBackendFrameworkFromComponent(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
deployment *v1alpha1.DynamoGraphDeployment
expected BackendFramework
expectError bool
......@@ -3863,8 +3805,7 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
}{
{
name: "detect from args - VLLM",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "worker", // Worker component
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3872,17 +3813,14 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
},
},
},
},
deployment: &v1alpha1.DynamoGraphDeployment{},
expected: BackendFrameworkVLLM,
},
{
name: "explicit framework only",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "worker", // Worker component
},
},
deployment: &v1alpha1.DynamoGraphDeployment{
Spec: v1alpha1.DynamoGraphDeploymentSpec{
BackendFramework: "sglang",
......@@ -3892,8 +3830,7 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
},
{
name: "detected matches explicit",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "worker", // Worker component
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3901,7 +3838,6 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
},
},
},
},
deployment: &v1alpha1.DynamoGraphDeployment{
Spec: v1alpha1.DynamoGraphDeploymentSpec{
BackendFramework: "sglang",
......@@ -3911,8 +3847,7 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
},
{
name: "detected conflicts with explicit",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "worker", // Worker component
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3920,7 +3855,6 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
},
},
},
},
deployment: &v1alpha1.DynamoGraphDeployment{
Spec: v1alpha1.DynamoGraphDeploymentSpec{
BackendFramework: "sglang",
......@@ -3931,29 +3865,24 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
},
{
name: "non-worker component returns noop",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "frontend", // Frontend component
},
},
deployment: &v1alpha1.DynamoGraphDeployment{},
expected: BackendFrameworkNoop,
},
{
name: "worker with no detection, no explicit - returns noop",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "worker", // Worker component
},
},
deployment: &v1alpha1.DynamoGraphDeployment{},
expected: BackendFrameworkNoop,
expectError: false,
},
{
name: "worker with detection failure, no explicit - returns noop",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: "worker", // Worker component
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -3961,7 +3890,6 @@ func TestGetBackendFrameworkFromComponent(t *testing.T) {
},
},
},
},
deployment: &v1alpha1.DynamoGraphDeployment{},
expected: BackendFrameworkNoop,
expectError: false,
......@@ -4211,9 +4139,8 @@ func TestGenerateGrovePodCliqueSet_StartsAfterDependencies(t *testing.T) {
},
Spec: v1alpha1.DynamoGraphDeploymentSpec{
BackendFramework: tt.backendFramework,
Services: map[string]*v1alpha1.DynamoComponentDeploymentOverridesSpec{
Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"main": {
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
Multinode: &v1alpha1.MultinodeSpec{
NodeCount: 2,
},
......@@ -4227,7 +4154,6 @@ func TestGenerateGrovePodCliqueSet_StartsAfterDependencies(t *testing.T) {
},
},
},
},
}
controllerConfig := controller_common.Config{
......@@ -4298,18 +4224,16 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
backendFramework BackendFramework
wantEnvVars map[string]string
wantErr bool
}{
{
name: "frontend with default command",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
},
},
backendFramework: BackendFrameworkVLLM,
wantEnvVars: map[string]string{
"DYN_HTTP_PORT": fmt.Sprintf("%d", commonconsts.DynamoServicePort),
......@@ -4317,8 +4241,7 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) {
},
{
name: "frontend with overriding env var",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
Envs: []corev1.EnvVar{
{
......@@ -4327,7 +4250,6 @@ func TestGenerateBasePodSpec_Frontend(t *testing.T) {
},
},
},
},
backendFramework: BackendFrameworkVLLM,
wantEnvVars: map[string]string{
"DYN_HTTP_PORT": "3000",
......@@ -4391,25 +4313,21 @@ func TestGenerateBasePodSpec_PlannerServiceAccount(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
expectedServiceAcc string
}{
{
name: "Planner component should have planner service account",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypePlanner,
},
},
expectedServiceAcc: commonconsts.PlannerServiceAccountName,
},
{
name: "Planner service account should not be set for non-planner components",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeWorker,
},
},
expectedServiceAcc: "",
},
}
......@@ -4445,14 +4363,13 @@ func TestGenerateBasePodSpec_PlannerServiceAccount(t *testing.T) {
func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
secretsRetriever SecretsRetriever
expectedImagePullSecrets []corev1.LocalObjectReference
}{
{
name: "disable docker secrets annotation set to true",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
Annotations: map[string]string{
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueTrue,
......@@ -4463,14 +4380,12 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
},
},
},
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
expectedImagePullSecrets: nil, // Should be nil when disabled
},
{
name: "disable docker secrets annotation set to false",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
Annotations: map[string]string{
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueFalse,
......@@ -4481,7 +4396,6 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
},
},
},
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
expectedImagePullSecrets: []corev1.LocalObjectReference{
{Name: "test-docker-secret"},
......@@ -4489,8 +4403,7 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
{
name: "disable docker secrets annotation not set (default behavior)",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
ExtraPodSpec: &common.ExtraPodSpec{
MainContainer: &corev1.Container{
......@@ -4498,7 +4411,6 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
},
},
},
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
expectedImagePullSecrets: []corev1.LocalObjectReference{
{Name: "test-docker-secret"},
......@@ -4506,8 +4418,7 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
{
name: "disable docker secrets annotation set to invalid value",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
Annotations: map[string]string{
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: "invalid",
......@@ -4518,7 +4429,6 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
},
},
},
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
expectedImagePullSecrets: []corev1.LocalObjectReference{
{Name: "test-docker-secret"},
......@@ -4526,8 +4436,7 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
{
name: "disable docker secrets but no secrets retriever",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
Annotations: map[string]string{
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueFalse,
......@@ -4538,7 +4447,6 @@ func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
},
},
},
},
secretsRetriever: nil,
expectedImagePullSecrets: nil, // Should be nil when no retriever
},
......@@ -4580,13 +4488,12 @@ func TestGenerateBasePodSpec_Worker(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
expectedPodSpec *corev1.PodSpec
}{
{
name: "Planner component should have planner service account",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
Envs: []corev1.EnvVar{
{Name: "ANOTHER_COMPONENTENV", Value: "true"},
},
......@@ -4601,7 +4508,6 @@ func TestGenerateBasePodSpec_Worker(t *testing.T) {
},
},
},
},
expectedPodSpec: &corev1.PodSpec{
Containers: []corev1.Container{
{
......@@ -4717,15 +4623,14 @@ func TestGenerateBasePodSpec_VolumeMounts(t *testing.T) {
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
expectError bool
expectedPVCs []string
expectedMounts []corev1.VolumeMount
}{
{
name: "valid volumeMounts",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4734,7 +4639,6 @@ func TestGenerateBasePodSpec_VolumeMounts(t *testing.T) {
},
},
},
},
expectError: false,
expectedPVCs: []string{"test-pvc"},
expectedMounts: []corev1.VolumeMount{
......@@ -4744,15 +4648,13 @@ func TestGenerateBasePodSpec_VolumeMounts(t *testing.T) {
},
{
name: "multiple volumeMounts",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{Name: "pvc1", MountPoint: "/data1"},
{Name: "pvc2", MountPoint: "/data2"},
},
},
},
expectError: false,
expectedPVCs: []string{"pvc1", "pvc2"},
expectedMounts: []corev1.VolumeMount{
......@@ -4763,26 +4665,22 @@ func TestGenerateBasePodSpec_VolumeMounts(t *testing.T) {
},
{
name: "empty volumeMount name",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{Name: "", MountPoint: "/data"},
},
},
},
expectError: true,
},
{
name: "empty volumeMount mountPoint",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{Name: "test-pvc", MountPoint: ""},
},
},
},
expectError: true,
},
}
......@@ -4859,15 +4757,14 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
tests := []struct {
name string
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
component *v1alpha1.DynamoComponentDeploymentSharedSpec
backendFramework BackendFramework
expectError bool
expectedMount *corev1.VolumeMount
}{
{
name: "useAsCompilationCache with custom mount point",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4877,15 +4774,13 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
},
},
},
},
backendFramework: BackendFrameworkVLLM,
expectError: false,
expectedMount: &corev1.VolumeMount{Name: "cache-pvc", MountPath: "/custom/cache"},
},
{
name: "useAsCompilationCache with default mount point for VLLM",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4894,15 +4789,13 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
},
},
},
},
backendFramework: BackendFrameworkVLLM,
expectError: false,
expectedMount: &corev1.VolumeMount{Name: "cache-pvc", MountPath: commonconsts.DefaultVLLMCacheMountPoint},
},
{
name: "useAsCompilationCache without mount point for SGLang - should error",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4911,15 +4804,13 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
},
},
},
},
backendFramework: BackendFrameworkSGLang,
expectError: true, // SGLang doesn't support compilation cache, requires explicit mount point
expectedMount: nil,
},
{
name: "useAsCompilationCache with explicit mount point for SGLang - should work",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4929,15 +4820,13 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
},
},
},
},
backendFramework: BackendFrameworkSGLang,
expectError: false,
expectedMount: &corev1.VolumeMount{Name: "cache-pvc", MountPath: "/custom/sglang/cache"},
},
{
name: "useAsCompilationCache without mount point for TensorRT-LLM - should error",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4946,15 +4835,13 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
},
},
},
},
backendFramework: BackendFrameworkTRTLLM,
expectError: true, // TensorRT-LLM doesn't support compilation cache, requires explicit mount point
expectedMount: nil,
},
{
name: "useAsCompilationCache with explicit mount point for TensorRT-LLM - should work",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4964,15 +4851,13 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
},
},
},
},
backendFramework: BackendFrameworkTRTLLM,
expectError: false,
expectedMount: &corev1.VolumeMount{Name: "cache-pvc", MountPath: "/custom/trtllm/cache"},
},
{
name: "no useAsCompilationCache volumes - should be ignored",
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
ComponentType: commonconsts.ComponentTypeFrontend,
VolumeMounts: []v1alpha1.VolumeMount{
{
......@@ -4981,7 +4866,6 @@ func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T)
},
},
},
},
backendFramework: BackendFrameworkVLLM,
expectError: false,
expectedMount: nil, // Should be ignored, not error
......
docs/kubernetes/api_reference.md
View file @ 60975b51
......@@ -85,6 +85,7 @@ DynamoComponentDeployment is the Schema for the dynamocomponentdeployments API
_Appears in:_
- [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec)
- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
......@@ -92,12 +93,13 @@ _Appears in:_
| `labels` _object (keys:string, values:string)_ | Labels to add to generated Kubernetes resources for this component. | | |
| `serviceName` _string_ | The name of the component | | |
| `componentType` _string_ | ComponentType indicates the role of this component (for example, "main"). | | |
| `subComponentType` _string_ | SubComponentType indicates the sub-role of this component (for example, "prefill"). | | |
| `dynamoNamespace` _string_ | Dynamo namespace of the service (allows to override the Dynamo namespace of the service defined in annotations inside the Dynamo archive) | | |
| `resources` _[Resources](#resources)_ | Resources requested and limits for this component, including CPU, memory,<br />GPUs/devices, and any runtime-specific resources. | | |
| `autoscaling` _[Autoscaling](#autoscaling)_ | Autoscaling config for this component (replica range, target utilization, etc.). | | |
| `envs` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array_ | Envs defines additional environment variables to inject into the component containers. | | |
| `envFromSecret` _string_ | EnvFromSecret references a Secret whose key/value pairs will be exposed as<br />environment variables in the component containers. | | |
| `pvc` _[PVC](#pvc)_ | PVC config describing volumes to be mounted by the component. | | |
| `volumeMounts` _[VolumeMount](#volumemount) array_ | VolumeMounts references PVCs defined at the top level for volumes to be mounted by the component. | | |
| `ingress` _[IngressSpec](#ingressspec)_ | Ingress config to expose the component outside the cluster (or through a service mesh). | | |
| `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | |
| `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | |
......@@ -121,19 +123,18 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `dynamoComponent` _string_ | DynamoComponent selects the Dynamo component from the archive to deploy.<br />Typically corresponds to a component defined in the packaged Dynamo artifacts. | | |
| `dynamoTag` _string_ | contains the tag of the DynamoComponent: for example, "my_package:MyService" | | |
| `backendFramework` _string_ | BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm") | | Enum: [sglang vllm trtllm] <br /> |
| `annotations` _object (keys:string, values:string)_ | Annotations to add to generated Kubernetes resources for this component<br />(such as Pod, Service, and Ingress when applicable). | | |
| `labels` _object (keys:string, values:string)_ | Labels to add to generated Kubernetes resources for this component. | | |
| `serviceName` _string_ | The name of the component | | |
| `componentType` _string_ | ComponentType indicates the role of this component (for example, "main"). | | |
| `subComponentType` _string_ | SubComponentType indicates the sub-role of this component (for example, "prefill"). | | |
| `dynamoNamespace` _string_ | Dynamo namespace of the service (allows to override the Dynamo namespace of the service defined in annotations inside the Dynamo archive) | | |
| `resources` _[Resources](#resources)_ | Resources requested and limits for this component, including CPU, memory,<br />GPUs/devices, and any runtime-specific resources. | | |
| `autoscaling` _[Autoscaling](#autoscaling)_ | Autoscaling config for this component (replica range, target utilization, etc.). | | |
| `envs` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array_ | Envs defines additional environment variables to inject into the component containers. | | |
| `envFromSecret` _string_ | EnvFromSecret references a Secret whose key/value pairs will be exposed as<br />environment variables in the component containers. | | |
| `pvc` _[PVC](#pvc)_ | PVC config describing volumes to be mounted by the component. | | |
| `volumeMounts` _[VolumeMount](#volumemount) array_ | VolumeMounts references PVCs defined at the top level for volumes to be mounted by the component. | | |
| `ingress` _[IngressSpec](#ingressspec)_ | Ingress config to expose the component outside the cluster (or through a service mesh). | | |
| `sharedMemory` _[SharedMemorySpec](#sharedmemoryspec)_ | SharedMemory controls the tmpfs mounted at /dev/shm (enable/disable and size). | | |
| `extraPodMetadata` _[ExtraPodMetadata](#extrapodmetadata)_ | ExtraPodMetadata adds labels/annotations to the created Pods. | | |
......@@ -176,8 +177,9 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `dynamoGraph` _string_ | DynamoGraph selects the graph (workflow/topology) to deploy. This must match<br />a graph name packaged with the Dynamo archive. | | |
| `envs` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array_ | Envs are environment variables applied to all services in the graph unless<br />overridden by service-specific configuration. | | Optional: {} <br /> |
| `pvcs` _[PVC](#pvc) array_ | PVCs defines a list of persistent volume claims that can be referenced by components.<br />Each PVC must have a unique name that can be referenced in component specifications. | | Optional: {} <br /> |
| `services` _object (keys:string, values:[DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec))_ | Services are the services to deploy as part of this deployment. | | Optional: {} <br /> |
| `envs` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array_ | Envs are environment variables applied to all services in the deployment unless<br />overridden by service-specific configuration. | | Optional: {} <br /> |
| `backendFramework` _string_ | BackendFramework specifies the backend framework (e.g., "sglang", "vllm", "trtllm"). | | Enum: [sglang vllm trtllm] <br /> |
......@@ -266,17 +268,15 @@ _Appears in:_
_Appears in:_
- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec)
- [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec)
- [DynamoGraphDeploymentSpec](#dynamographdeploymentspec)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `create` _boolean_ | Create indicates to create a new PVC | | |
| `name` _string_ | Name is the name of the PVC | | |
| `storageClass` _string_ | StorageClass to be used for PVC creation. Leave it as empty if the PVC is already created. | | |
| `size` _[Quantity](#quantity)_ | Size of the NIM cache in Gi, used during PVC creation | | |
| `volumeAccessMode` _[PersistentVolumeAccessMode](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#persistentvolumeaccessmode-v1-core)_ | VolumeAccessMode is the volume access mode of the PVC | | |
| `mountPoint` _string_ | | | |
| `name` _string_ | Name is the name of the PVC | | Required: {} <br /> |
| `storageClass` _string_ | StorageClass to be used for PVC creation. Required when create is true. | | |
| `size` _[Quantity](#quantity)_ | Size of the volume in Gi, used during PVC creation. Required when create is true. | | |
| `volumeAccessMode` _[PersistentVolumeAccessMode](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#persistentvolumeaccessmode-v1-core)_ | VolumeAccessMode is the volume access mode of the PVC. Required when create is true. | | |
#### SharedMemorySpec
......@@ -297,3 +297,247 @@ _Appears in:_
| `size` _[Quantity](#quantity)_ | | | |
#### VolumeMount
VolumeMount references a PVC defined at the top level for volumes to be mounted by the component
_Appears in:_
- [DynamoComponentDeploymentSharedSpec](#dynamocomponentdeploymentsharedspec)
- [DynamoComponentDeploymentSpec](#dynamocomponentdeploymentspec)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `name` _string_ | Name references a PVC name defined in the top-level PVCs map | | Required: {} <br /> |
| `mountPoint` _string_ | MountPoint specifies where to mount the volume.<br />If useAsCompilationCache is true and mountPoint is not specified,<br />a backend-specific default will be used. | | |
| `useAsCompilationCache` _boolean_ | UseAsCompilationCache indicates this volume should be used as a compilation cache.<br />When true, backend-specific environment variables will be set and default mount points may be used. | false | |
# Operator Default Values Injection
The Dynamo operator automatically applies default values to various fields when they are not explicitly specified in your deployments. These defaults include:
- **Health Probes**: Startup, liveness, and readiness probes are configured differently for frontend, worker, and planner components. For example, worker components receive a startup probe with a 2-hour timeout (720 failures × 10 seconds) to accommodate long model loading times.
- **Shared Memory**: All components receive an 8Gi shared memory volume mounted at `/dev/shm` by default (can be disabled or resized via the `sharedMemory` field).
- **Environment Variables**: Components automatically receive environment variables like `DYN_NAMESPACE`, `DYN_PARENT_DGD_K8S_NAME`, `DYNAMO_PORT`, and backend-specific variables.
- **Pod Configuration**: Default `terminationGracePeriodSeconds` of 60 seconds and `restartPolicy: Always`.
- **Autoscaling**: When enabled without explicit metrics, defaults to CPU-based autoscaling with 80% target utilization.
- **Backend-Specific Behavior**: For multinode deployments, probes are automatically modified or removed for worker nodes depending on the backend framework (VLLM, SGLang, or TensorRT-LLM).
## Pod Specification Defaults
All components receive the following pod-level defaults unless overridden:
- **`terminationGracePeriodSeconds`**: `60` seconds
- **`restartPolicy`**: `Always`
## Shared Memory Configuration
Shared memory is enabled by default for all components:
- **Enabled**: `true` (unless explicitly disabled via `sharedMemory.disabled`)
- **Size**: `8Gi`
- **Mount Path**: `/dev/shm`
- **Volume Type**: `emptyDir` with `memory` medium
To disable shared memory or customize the size, use the `sharedMemory` field in your component specification.
## Health Probes by Component Type
The operator applies different default health probes based on the component type.
### Frontend Components
Frontend components receive the following probe configurations:
**Liveness Probe:**
- **Type**: HTTP GET
- **Path**: `/health`
- **Port**: `http` (8000)
- **Initial Delay**: 60 seconds
- **Period**: 60 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 10
**Readiness Probe:**
- **Type**: Exec command
- **Command**: `curl -s http://localhost:${DYNAMO_PORT}/health | jq -e ".status == \"healthy\""`
- **Initial Delay**: 60 seconds
- **Period**: 60 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 10
### Worker Components
Worker components receive the following probe configurations:
**Liveness Probe:**
- **Type**: HTTP GET
- **Path**: `/live`
- **Port**: `system` (9090)
- **Period**: 5 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 1
**Readiness Probe:**
- **Type**: HTTP GET
- **Path**: `/health`
- **Port**: `system` (9090)
- **Period**: 10 seconds
- **Timeout**: 30 seconds
- **Failure Threshold**: 60
**Startup Probe:**
- **Type**: HTTP GET
- **Path**: `/live`
- **Port**: `system` (9090)
- **Period**: 10 seconds
- **Timeout**: 5 seconds
- **Failure Threshold**: 720 (allows up to 2 hours for startup: 10s × 720 = 7200s)
:::{note}
For larger models (typically >70B parameters) or slower storage systems, you may need to increase the `failureThreshold` to allow more time for model loading. Calculate the required threshold based on your expected startup time: `failureThreshold = (expected_startup_seconds / period)`. Override the startup probe in your component specification if the default 2-hour window is insufficient.
:::
### Multinode Deployment Probe Modifications
For multinode deployments, the operator modifies probes based on the backend framework and node role:
#### VLLM Backend
- **Worker nodes**: All probes (liveness, readiness, startup) are removed
#### SGLang Backend
- **Worker nodes**: All probes (liveness, readiness, startup) are removed
#### TensorRT-LLM Backend
- **Leader nodes**: All probes remain unchanged
- **Worker nodes**:
- Liveness and startup probes are removed
- Readiness probe is replaced with a TCP socket check on SSH port (2222):
- **Initial Delay**: 20 seconds
- **Period**: 20 seconds
- **Timeout**: 5 seconds
- **Failure Threshold**: 10
## Environment Variables
The operator automatically injects environment variables based on component type and configuration:
### All Components
- **`DYN_NAMESPACE`**: The Dynamo namespace for the component
- **`DYN_PARENT_DGD_K8S_NAME`**: The parent DynamoGraphDeployment Kubernetes resource name
- **`DYN_PARENT_DGD_K8S_NAMESPACE`**: The parent DynamoGraphDeployment Kubernetes namespace
### Frontend Components
- **`DYNAMO_PORT`**: `8000`
- **`DYN_HTTP_PORT`**: `8000`
### Worker Components
- **`DYN_SYSTEM_ENABLED`**: `true`
- **`DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS`**: `["generate"]`
- **`DYN_SYSTEM_PORT`**: `9090`
### Planner Components
- **`PLANNER_PROMETHEUS_PORT`**: `9085`
### VLLM Backend (with compilation cache)
When a volume mount is configured with `useAsCompilationCache: true`:
- **`VLLM_CACHE_ROOT`**: Set to the mount point of the cache volume
## Service Account
Planner components automatically receive the following service account:
- **`serviceAccountName`**: `planner-serviceaccount`
## Image Pull Secrets
The operator automatically discovers and injects image pull secrets for container images. When a component specifies a container image, the operator:
1. Scans all Kubernetes secrets of type `kubernetes.io/dockerconfigjson` in the component's namespace
2. Extracts the docker registry server URLs from each secret's authentication configuration
3. Matches the container image's registry host against the discovered registry URLs
4. Automatically injects matching secrets as `imagePullSecrets` in the pod specification
This eliminates the need to manually specify image pull secrets for each component. The operator maintains an internal index of docker secrets and their associated registries, refreshing this index periodically.
**To disable automatic image pull secret discovery** for a specific component, add the following annotation:
```yaml
annotations:
nvidia.com/disable-image-pull-secret-discovery: "true"
```
## Autoscaling Defaults
When autoscaling is enabled but no metrics are specified, the operator applies:
- **Default Metric**: CPU utilization
- **Target Average Utilization**: `80%`
## Port Configurations
Default container ports are configured based on component type:
### Frontend Components
- **Port**: 8000
- **Protocol**: TCP
- **Name**: `http`
### Worker Components
- **Port**: 9090
- **Protocol**: TCP
- **Name**: `system`
### Planner Components
- **Port**: 9085
- **Protocol**: TCP
- **Name**: `metrics`
## Backend-Specific Configurations
### VLLM
- **Ray Head Port**: 6379 (for multinode deployments)
### SGLang
- **Distribution Init Port**: 29500 (for multinode deployments)
### TensorRT-LLM
- **SSH Port**: 2222 (for multinode MPI communication)
- **OpenMPI Environment**: `OMPI_MCA_orte_keep_fqdn_hostnames=1`
## Implementation Reference
For users who want to understand the implementation details or contribute to the operator, the default values described in this document are set in the following source files:
- **Health Probes & Pod Specifications**: [`internal/dynamo/graph.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/graph.go) - Contains the main logic for applying default probes, environment variables, shared memory, and pod configurations
- **Component-Specific Defaults**:
- [`internal/dynamo/component_frontend.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/component_frontend.go)
- [`internal/dynamo/component_worker.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/component_worker.go)
- [`internal/dynamo/component_planner.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/component_planner.go)
- **Image Pull Secrets**: [`internal/secrets/docker.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/secrets/docker.go) - Implements the docker secret indexer and automatic discovery
- **Backend-Specific Behavior**:
- [`internal/dynamo/backend_vllm.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/backend_vllm.go)
- [`internal/dynamo/backend_sglang.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/backend_sglang.go)
- [`internal/dynamo/backend_trtllm.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/dynamo/backend_trtllm.go)
- **Constants & Annotations**: [`internal/consts/consts.go`](https://github.com/ai-dynamo/dynamo/blob/main/deploy/cloud/operator/internal/consts/consts.go) - Defines annotation keys and other constants
## Notes
- All these defaults can be overridden by explicitly specifying values in your DynamoComponentDeployment or DynamoGraphDeployment resources
- User-specified probes (via `livenessProbe`, `readinessProbe`, or `startupProbe` fields) take precedence over operator defaults
- For multinode deployments, some defaults are modified or removed as described above to accommodate distributed execution patterns
- The `extraPodSpec.mainContainer` field can be used to override probe configurations set by the operator
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment