/*
 * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dynamo

import (
	"context"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"testing"
	"time"

	configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1"
	"github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1"
	commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
	"github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common"
	grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1"
	"github.com/google/go-cmp/cmp"
	"github.com/stretchr/testify/assert"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	ptr "k8s.io/utils/ptr"
)

func TestGenerateDynamoComponentsDeployments(t *testing.T) {
	type args struct {
		parentDynamoGraphDeployment *v1alpha1.DynamoGraphDeployment
		ingressSpec                 *v1alpha1.IngressSpec
	}
	tests := []struct {
		name    string
		args    args
		want    map[string]*v1alpha1.DynamoComponentDeployment
		wantErr bool
	}{
		{
			name: "Test GenerateDynamoComponentsDeployments",
			args: args{
				parentDynamoGraphDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment",
						Namespace: "default",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"service1": {
								DynamoNamespace:  &[]string{"default"}[0],
								ComponentType:    "frontend",
								SubComponentType: "test-sub-component",
								Replicas:         &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
							"service2": {
								DynamoNamespace: &[]string{"default"}[0],
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
						},
					},
				},
			},
			want: map[string]*v1alpha1.DynamoComponentDeployment{
				"service1": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service1",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service1",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:      "service1",
							DynamoNamespace:  &[]string{"default-test-dynamographdeployment"}[0],
							ComponentType:    "frontend",
							SubComponentType: "test-sub-component",
							Replicas:         &[]int32{3}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service1",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
						},
					},
				},
				"service2": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service2",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service2",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service2",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							Replicas:        &[]int32{3}[0],
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service2",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "Test GenerateDynamoComponentsDeployments with default dynamo namespace",
			args: args{
				parentDynamoGraphDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment",
						Namespace: "default",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"service1": {
								DynamoNamespace: nil,
								ComponentType:   "frontend",
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
							"service2": {
								DynamoNamespace: nil,
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
						},
					},
				},
			},
			want: map[string]*v1alpha1.DynamoComponentDeployment{
				"service1": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service1",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service1",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service1",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							ComponentType:   "frontend",
							Replicas:        &[]int32{3}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service1",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
						},
					},
				},
				"service2": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service2",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service2",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service2",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							Replicas:        &[]int32{3}[0],
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service2",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "Test GenerateDynamoComponentsDeployments with ingress enabled",
			args: args{
				parentDynamoGraphDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment",
						Namespace: "default",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"service1": {
								DynamoNamespace: nil,
								ComponentType:   "frontend",
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
							"service2": {
								DynamoNamespace: nil,
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
						},
					},
				},
				ingressSpec: &v1alpha1.IngressSpec{
					Enabled: true,
					Host:    "test-dynamographdeployment",
				},
			},
			want: map[string]*v1alpha1.DynamoComponentDeployment{
				"service1": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service1",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service1",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service1",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							ComponentType:   "frontend",
							Replicas:        &[]int32{3}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service1",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							Ingress: &v1alpha1.IngressSpec{
								Enabled: true,
								Host:    "test-dynamographdeployment",
							},
						},
					},
				},
				"service2": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service2",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service2",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service2",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							Replicas:        &[]int32{3}[0],
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service2",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "Test GenerateDynamoComponentsDeployments with config from DYN_DEPLOYMENT_CONFIG env var",
			args: args{
				parentDynamoGraphDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment",
						Namespace: "default",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						Envs: []corev1.EnvVar{
							{
								Name:  "DYN_DEPLOYMENT_CONFIG",
								Value: `{"service1":{"port":8080,"ServiceArgs":{"Workers":2, "Resources":{"CPU":"2", "Memory":"2Gi", "GPU":"2"}}}}`,
							},
						},
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"service1": {
								DynamoNamespace: nil,
								ComponentType:   "frontend",
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
							"service2": {
								DynamoNamespace: nil,
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
						},
					},
				},
			},
			want: map[string]*v1alpha1.DynamoComponentDeployment{
				"service1": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service1",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service1",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service1",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							ComponentType:   "frontend",
							Replicas:        &[]int32{3}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "2",
									Memory: "2Gi",
									GPU:    "2",
									Custom: map[string]string{},
								},
								Limits: &v1alpha1.ResourceItem{
									CPU:    "2",
									Memory: "2Gi",
									GPU:    "2",
									Custom: nil,
								},
							},
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service1",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							Envs: []corev1.EnvVar{
								{
									Name:  "DYN_DEPLOYMENT_CONFIG",
									Value: fmt.Sprintf(`{"service1":{"ServiceArgs":{"Resources":{"CPU":"2","GPU":"2","Memory":"2Gi"},"Workers":2},"port":%d}}`, commonconsts.DynamoServicePort),
								},
							},
						},
					},
				},
				"service2": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service2",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service2",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service2",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							Replicas:        &[]int32{3}[0],
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service2",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
							Envs: []corev1.EnvVar{
								{
									Name:  "DYN_DEPLOYMENT_CONFIG",
									Value: `{"service1":{"port":8080,"ServiceArgs":{"Workers":2, "Resources":{"CPU":"2", "Memory":"2Gi", "GPU":"2"}}}}`,
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "Test GenerateDynamoComponentsDeployments with ExtraPodSpec.MainContainer Command and Args",
			args: args{
				parentDynamoGraphDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment",
						Namespace: "default",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						BackendFramework: string(BackendFrameworkSGLang),
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"service1": {
								DynamoNamespace: &[]string{"default-test-dynamographdeployment-44136fa3"}[0],
								ComponentType:   "frontend",
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									MainContainer: &corev1.Container{
										Command: []string{"sh", "-c"},
										Args:    []string{"echo hello world", "sleep 99999"},
									},
								},
							},
							"service2": {
								DynamoNamespace: &[]string{"default-test-dynamographdeployment-44136fa3"}[0],
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
						},
						Envs: []corev1.EnvVar{
							{
								Name:  "TEST_ENV",
								Value: "test-value",
							},
						},
					},
				},
			},
			want: map[string]*v1alpha1.DynamoComponentDeployment{
				"service1": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service1",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service1",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						BackendFramework: string(BackendFrameworkSGLang),
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service1",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							ComponentType:   "frontend",
							Replicas:        &[]int32{3}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service1",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							ExtraPodSpec: &v1alpha1.ExtraPodSpec{
								MainContainer: &corev1.Container{
									Command: []string{"sh", "-c"},
									Args:    []string{"echo hello world", "sleep 99999"},
								},
							},
							Envs: []corev1.EnvVar{
								{
									Name:  "TEST_ENV",
									Value: "test-value",
								},
							},
						},
					},
				},
				"service2": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service2",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service2",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						BackendFramework: string(BackendFrameworkSGLang),
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:     "service2",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							Replicas:        &[]int32{3}[0],
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service2",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
							Envs: []corev1.EnvVar{
								{
									Name:  "TEST_ENV",
									Value: "test-value",
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "Test GenerateDynamoComponentsDeployments with Discover Backend and Metrics Annotatitions",
			args: args{
				parentDynamoGraphDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment",
						Namespace: "default",
						Annotations: map[string]string{
							commonconsts.KubeAnnotationEnableMetrics:          "false",
							commonconsts.KubeAnnotationDynamoDiscoveryBackend: "test",
						},
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						BackendFramework: string(BackendFrameworkSGLang),
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"service1": {
								DynamoNamespace: &[]string{"default-test-dynamographdeployment-44136fa3"}[0],
								ComponentType:   "frontend",
								Replicas:        &[]int32{3}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "0",
										Custom: map[string]string{},
									},
								},
							},
						},
					},
				},
			},
			want: map[string]*v1alpha1.DynamoComponentDeployment{
				"service1": {
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamographdeployment-service1",
						Namespace: "default",
						Labels: map[string]string{
							commonconsts.KubeLabelDynamoComponent:           "service1",
							commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
							commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
						},
					},
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						BackendFramework: string(BackendFrameworkSGLang),
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							Annotations: map[string]string{
								commonconsts.KubeAnnotationEnableMetrics:          "false",
								commonconsts.KubeAnnotationDynamoDiscoveryBackend: "test",
							},
							ServiceName:     "service1",
							DynamoNamespace: &[]string{"default-test-dynamographdeployment"}[0],
							ComponentType:   "frontend",
							Replicas:        &[]int32{3}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "0",
									Custom: map[string]string{},
								},
							},
							Labels: map[string]string{
								commonconsts.KubeLabelDynamoComponent:           "service1",
								commonconsts.KubeLabelDynamoNamespace:           "default-test-dynamographdeployment",
								commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamographdeployment",
							},
						},
					},
				},
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := GenerateDynamoComponentsDeployments(context.Background(), tt.args.parentDynamoGraphDeployment, tt.args.ingressSpec, nil, nil, RollingUpdateContext{})
			if (err != nil) != tt.wantErr {
				t.Errorf("GenerateDynamoComponentsDeployments() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if diff := cmp.Diff(got, tt.want); diff != "" {
				t.Errorf("GenerateDynamoComponentsDeployments() mismatch (-want +got):\n%s", diff)
			}
		})
	}
}

func Test_GetDynamoComponentDeploymentsGlobalNamespace(t *testing.T) {
	dgd := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-dynamographdeployment",
			Namespace: "default",
		},
		Spec: v1alpha1.DynamoGraphDeploymentSpec{
			BackendFramework: string(BackendFrameworkSGLang),
			Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"service1": {
					ComponentType:         "frontend",
					GlobalDynamoNamespace: true,
					Replicas:              &[]int32{3}[0],
				},
				"service2": {
					ComponentType: "worker",
					Replicas:      &[]int32{3}[0],
				},
			},
		},
	}

	got, err := GenerateDynamoComponentsDeployments(context.Background(), dgd, nil, nil, nil, RollingUpdateContext{})
	if !assert.NoError(t, err) {
		return
	}

	if !assert.Len(t, got, 2) {
		return
	}

	for _, d := range got {
		switch d.Spec.ComponentType {
		case commonconsts.ComponentTypeFrontend:
			assert.Equal(t, commonconsts.GlobalDynamoNamespace, *d.Spec.DynamoNamespace)
			assert.Equal(t, commonconsts.GlobalDynamoNamespace, d.Labels[commonconsts.KubeLabelDynamoNamespace])
		case commonconsts.ComponentTypeWorker:
			expectedNamespace := fmt.Sprintf("%s-%s", dgd.Namespace, dgd.Name)
			assert.Equal(t, expectedNamespace, *d.Spec.DynamoNamespace)
			assert.Equal(t, expectedNamespace, d.Labels[commonconsts.KubeLabelDynamoNamespace])
		default:
			t.Errorf("unexpected component type: %s", d.Spec.ComponentType)
		}
	}
}

// TestGenerateComponentContext tests the generateComponentContext function
// to ensure it correctly computes the DynamoNamespace from authoritative sources
// (k8s namespace + DGD name), ignoring any deprecated dynamoNamespace field.
func TestGenerateComponentContext(t *testing.T) {
	tests := []struct {
		name                       string
		component                  *v1alpha1.DynamoComponentDeploymentSharedSpec
		parentGraphDeploymentName  string
		namespace                  string
		numberOfNodes              int32
		discoveryBackend           configv1alpha1.DiscoveryBackend
		expectedDynamoNamespace    string
		expectedComponentType      string
		expectedParentDGDName      string
		expectedParentDGDNamespace string
	}{
		{
			name: "namespace-scoped operator: computes correct dynamo namespace",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypePlanner,
				// Deprecated field set to incorrect value - should be ignored
				DynamoNamespace: ptr.To("old-incorrect-value"),
			},
			parentGraphDeploymentName:  "my-deployment",
			namespace:                  "my-namespace",
			numberOfNodes:              1,
			discoveryBackend:           configv1alpha1.DiscoveryBackendKubernetes,
			expectedDynamoNamespace:    "my-namespace-my-deployment",
			expectedComponentType:      commonconsts.ComponentTypePlanner,
			expectedParentDGDName:      "my-deployment",
			expectedParentDGDNamespace: "my-namespace",
		},
		{
			name: "deprecated dynamoNamespace field is ignored",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				// This is the bug case: profiler sets dynamoNamespace to just DGD name
				DynamoNamespace: ptr.To("vllm-disagg"),
			},
			parentGraphDeploymentName:  "vllm-disagg",
			namespace:                  "djangoz",
			numberOfNodes:              1,
			discoveryBackend:           configv1alpha1.DiscoveryBackendKubernetes,
			expectedDynamoNamespace:    "djangoz-vllm-disagg",
			expectedComponentType:      commonconsts.ComponentTypeFrontend,
			expectedParentDGDName:      "vllm-disagg",
			expectedParentDGDNamespace: "djangoz",
		},
		{
			name: "GlobalDynamoNamespace takes precedence",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType:         commonconsts.ComponentTypeWorker,
				GlobalDynamoNamespace: true,
				// Even with deprecated field set, GlobalDynamoNamespace should win
				DynamoNamespace: ptr.To("should-be-ignored"),
			},
			parentGraphDeploymentName:  "shared-frontend",
			namespace:                  "production",
			numberOfNodes:              2,
			discoveryBackend:           configv1alpha1.DiscoveryBackendEtcd,
			expectedDynamoNamespace:    commonconsts.GlobalDynamoNamespace,
			expectedComponentType:      commonconsts.ComponentTypeWorker,
			expectedParentDGDName:      "shared-frontend",
			expectedParentDGDNamespace: "production",
		},
		{
			name: "nil dynamoNamespace field still computes correctly",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType:   commonconsts.ComponentTypePlanner,
				DynamoNamespace: nil,
			},
			parentGraphDeploymentName:  "test-dgd",
			namespace:                  "default",
			numberOfNodes:              1,
			discoveryBackend:           configv1alpha1.DiscoveryBackendKubernetes,
			expectedDynamoNamespace:    "default-test-dgd",
			expectedComponentType:      commonconsts.ComponentTypePlanner,
			expectedParentDGDName:      "test-dgd",
			expectedParentDGDNamespace: "default",
		},
		{
			name: "different namespace and DGD name combinations",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
			},
			parentGraphDeploymentName:  "llama-70b-prod",
			namespace:                  "ml-inference",
			numberOfNodes:              4,
			discoveryBackend:           configv1alpha1.DiscoveryBackendEtcd,
			expectedDynamoNamespace:    "ml-inference-llama-70b-prod",
			expectedComponentType:      commonconsts.ComponentTypeFrontend,
			expectedParentDGDName:      "llama-70b-prod",
			expectedParentDGDNamespace: "ml-inference",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			ctx := generateComponentContext(
				tt.component,
				tt.parentGraphDeploymentName,
				tt.namespace,
				tt.numberOfNodes,
				tt.discoveryBackend,
			)

			assert.Equal(t, tt.expectedDynamoNamespace, ctx.DynamoNamespace,
				"DynamoNamespace should be computed from k8s namespace + DGD name")
			assert.Equal(t, tt.expectedComponentType, ctx.ComponentType)
			assert.Equal(t, tt.expectedParentDGDName, ctx.ParentGraphDeploymentName)
			assert.Equal(t, tt.expectedParentDGDNamespace, ctx.ParentGraphDeploymentNamespace)
			assert.Equal(t, tt.numberOfNodes, ctx.numberOfNodes)
			assert.Equal(t, tt.discoveryBackend, ctx.DiscoveryBackend)
		})
	}
}

func Test_updateDynDeploymentConfig(t *testing.T) {
	type args struct {
		dynamoDeploymentComponent *v1alpha1.DynamoComponentDeployment
		newPort                   int
	}
	tests := []struct {
		name    string
		args    args
		want    []byte
		wantErr bool
	}{
		{
			name: "main component",
			args: args{
				dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName:   "Frontend",
							ComponentType: commonconsts.ComponentTypeFrontend,
							Envs: []corev1.EnvVar{
								{
									Name:  "OTHER",
									Value: `value`,
								},
								{
									Name:  commonconsts.DynamoDeploymentConfigEnvVar,
									Value: `{"Frontend":{"port":8080},"Planner":{"environment":"kubernetes"}}`,
								},
							},
						},
					},
				},
				newPort: commonconsts.DynamoServicePort,
			},
			want:    []byte(fmt.Sprintf(`{"Frontend":{"port":%d},"Planner":{"environment":"kubernetes"}}`, commonconsts.DynamoServicePort)),
			wantErr: false,
		},
		{
			name: "not main component",
			args: args{
				dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName: "Other",
							Envs: []corev1.EnvVar{
								{
									Name:  commonconsts.DynamoDeploymentConfigEnvVar,
									Value: `{"Frontend":{"port":8000},"Planner":{"environment":"kubernetes"}}`,
								},
							},
						},
					},
				},
				newPort: commonconsts.DynamoServicePort,
			},
			want:    []byte(fmt.Sprintf(`{"Frontend":{"port":%d},"Planner":{"environment":"kubernetes"}}`, commonconsts.DynamoServicePort)),
			wantErr: false,
		},
		{
			name: "no config variable",
			args: args{
				dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName: "Frontend",
							Envs: []corev1.EnvVar{
								{
									Name:  "OTHER",
									Value: `value`,
								},
							},
						},
					},
				},
				newPort: 8080,
			},
			want:    nil,
			wantErr: false,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			err := updateDynDeploymentConfig(tt.args.dynamoDeploymentComponent, tt.args.newPort)
			if (err != nil) != tt.wantErr {
				t.Errorf("updateDynDeploymentConfig() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if diff := cmp.Diff(tt.args.dynamoDeploymentComponent.GetDynamoDeploymentConfig(), tt.want); diff != "" {
				t.Errorf("updateDynDeploymentConfig() mismatch (-want +got):\n%s", diff)
			}
		})
	}
}

func Test_overrideWithDynDeploymentConfig(t *testing.T) {
	type args struct {
		ctx                       context.Context
		dynamoDeploymentComponent *v1alpha1.DynamoComponentDeployment
	}
	tests := []struct {
		name     string
		args     args
		wantErr  bool
		expected *v1alpha1.DynamoComponentDeployment
	}{
		{
			name: "no env var",
			args: args{
				ctx: context.Background(),
				dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName: "Frontend",
							Replicas:    &[]int32{1}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "1",
								},
							},
						},
					},
				},
			},
			wantErr: false,
			expected: &v1alpha1.DynamoComponentDeployment{
				Spec: v1alpha1.DynamoComponentDeploymentSpec{
					DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
						ServiceName: "Frontend",
						Replicas:    &[]int32{1}[0],
						Resources: &v1alpha1.Resources{
							Requests: &v1alpha1.ResourceItem{
								CPU:    "1",
								Memory: "1Gi",
								GPU:    "1",
							},
						},
					},
				},
			},
		},
		{
			name: "override workers and resources",
			args: args{
				ctx: context.Background(),
				dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName: "Frontend",
							Replicas:    &[]int32{1}[0],
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "1",
								},
							},
							Envs: []corev1.EnvVar{
								{
									Name:  commonconsts.DynamoDeploymentConfigEnvVar,
									Value: `{"Frontend":{"port":8080,"ServiceArgs":{"Workers":3, "Resources":{"CPU":"2", "Memory":"2Gi", "GPU":"2"}}},"Planner":{"environment":"kubernetes"}}`,
								},
							},
						},
					},
				},
			},
			wantErr: false,
			expected: &v1alpha1.DynamoComponentDeployment{
				Spec: v1alpha1.DynamoComponentDeploymentSpec{
					DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
						ServiceName: "Frontend",
						Replicas:    &[]int32{3}[0],
						Resources: &v1alpha1.Resources{
							Requests: &v1alpha1.ResourceItem{
								CPU:    "2",
								Memory: "2Gi",
								GPU:    "2",
							},
							Limits: &v1alpha1.ResourceItem{
								CPU:    "2",
								Memory: "2Gi",
								GPU:    "2",
							},
						},
						Envs: []corev1.EnvVar{
							{
								Name:  commonconsts.DynamoDeploymentConfigEnvVar,
								Value: `{"Frontend":{"port":8080,"ServiceArgs":{"Workers":3, "Resources":{"CPU":"2", "Memory":"2Gi", "GPU":"2"}}},"Planner":{"environment":"kubernetes"}}`,
							},
						},
					},
				},
			},
		},
		{
			name: "override subset of resources",
			args: args{
				ctx: context.Background(),
				dynamoDeploymentComponent: &v1alpha1.DynamoComponentDeployment{
					Spec: v1alpha1.DynamoComponentDeploymentSpec{
						DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
							ServiceName: "Frontend",
							Replicas:    nil,
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									CPU:    "1",
									Memory: "1Gi",
									GPU:    "1",
								},
							},
							Envs: []corev1.EnvVar{
								{
									Name:  commonconsts.DynamoDeploymentConfigEnvVar,
									Value: `{"Frontend":{"port":8080,"ServiceArgs":{"Workers":3, "Resources":{"GPU":"2"}}},"Planner":{"environment":"kubernetes"}}`,
								},
							},
						},
					},
				},
			},
			wantErr: false,
			expected: &v1alpha1.DynamoComponentDeployment{
				Spec: v1alpha1.DynamoComponentDeploymentSpec{
					DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
						ServiceName: "Frontend",
						Replicas:    &[]int32{3}[0],
						Resources: &v1alpha1.Resources{
							Requests: &v1alpha1.ResourceItem{
								CPU:    "1",
								Memory: "1Gi",
								GPU:    "2",
							},
							Limits: &v1alpha1.ResourceItem{
								CPU:    "",
								Memory: "",
								GPU:    "2",
							},
						},
						Envs: []corev1.EnvVar{
							{
								Name:  commonconsts.DynamoDeploymentConfigEnvVar,
								Value: `{"Frontend":{"port":8080,"ServiceArgs":{"Workers":3, "Resources":{"GPU":"2"}}},"Planner":{"environment":"kubernetes"}}`,
							},
						},
					},
				},
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if err := overrideWithDynDeploymentConfig(tt.args.ctx, tt.args.dynamoDeploymentComponent); (err != nil) != tt.wantErr {
				t.Errorf("overrideWithDynDeploymentConfig() error = %v, wantErr %v", err, tt.wantErr)
			}
			if diff := cmp.Diff(tt.args.dynamoDeploymentComponent, tt.expected); diff != "" {
				t.Errorf("overrideWithDynDeploymentConfig() mismatch (-want +got):\n%s", diff)
			}
		})
	}
}

func Test_mergeEnvs(t *testing.T) {
	type args struct {
		common   []corev1.EnvVar
		specific []corev1.EnvVar
	}
	tests := []struct {
		name string
		args args
		want []corev1.EnvVar
	}{
		{
			name: "no_common_envs",
			args: args{
				common:   []corev1.EnvVar{},
				specific: []corev1.EnvVar{{Name: "FOO", Value: "BAR"}},
			},
			want: []corev1.EnvVar{{Name: "FOO", Value: "BAR"}},
		},
		{
			name: "no_specific_envs",
			args: args{
				common:   []corev1.EnvVar{{Name: "FOO", Value: "BAR"}},
				specific: []corev1.EnvVar{},
			},
			want: []corev1.EnvVar{{Name: "FOO", Value: "BAR"}},
		},
		{
			name: "common_and_specific_envs",
			args: args{
				specific: []corev1.EnvVar{{Name: "BAZ", Value: "QUX"}},
				common:   []corev1.EnvVar{{Name: "FOO", Value: "BAR"}},
			},
			want: []corev1.EnvVar{{Name: "BAZ", Value: "QUX"}, {Name: "FOO", Value: "BAR"}},
		},
		{
			name: "common_and_specific_envs_with_same_name",
			args: args{
				common:   []corev1.EnvVar{{Name: "FOO", Value: "BAR"}},
				specific: []corev1.EnvVar{{Name: "FOO", Value: "QUX"}},
			},
			want: []corev1.EnvVar{{Name: "FOO", Value: "QUX"}},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := MergeEnvs(tt.args.common, tt.args.specific)
			sort.Slice(got, func(i, j int) bool {
				return got[i].Name < got[j].Name
			})
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("mergeEnvs() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGenerateGrovePodCliqueSet(t *testing.T) {
	type args struct {
		ctx              context.Context
		dynamoDeployment *v1alpha1.DynamoGraphDeployment
		controllerConfig *configv1alpha1.OperatorConfiguration
	}
	tests := []struct {
		name    string
		args    args
		want    *grovev1alpha1.PodCliqueSet
		wantErr bool
	}{
		{
			name: "test_generate_grove_pod_clique_set_single_node",
			args: args{
				ctx: context.Background(),
				controllerConfig: &configv1alpha1.OperatorConfiguration{
					Infrastructure: configv1alpha1.InfrastructureConfiguration{
						ETCDAddress:        "etcd-address",
						NATSAddress:        "nats-address",
						ModelExpressURL:    "model-express-url",
						PrometheusEndpoint: "http://localhost:9090",
					},
					Orchestrators: configv1alpha1.OrchestratorConfiguration{
						Grove: configv1alpha1.GroveConfiguration{
							TerminationDelay: metav1.Duration{Duration: 15 * time.Minute},
						},
					},
				},
				dynamoDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamo-graph-deployment",
						Namespace: "test-namespace",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						Envs: []corev1.EnvVar{
							{
								Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
								Value: "1",
							},
						},
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"Frontend": {
								ComponentType:    "frontend", // Frontend component
								SubComponentType: "test-sub-component",
								ExtraPodMetadata: &v1alpha1.ExtraPodMetadata{
									Annotations: map[string]string{
										"nvidia.com/annotation1": "annotation1",
										"nvidia.com/annotation2": "annotation2",
									},
									Labels: map[string]string{
										"nvidia.com/label1": "label1",
										"nvidia.com/label2": "label2",
									},
								},
								Replicas: &[]int32{1}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "1",
									},
								},
								Envs: []corev1.EnvVar{
									{
										Name:  "FRONTEND_ENV_1",
										Value: "1",
									},
								},
								EnvFromSecret: &[]string{"frontend-secret"}[0],
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/health",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/ready",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									PodSpec: &corev1.PodSpec{
										TerminationGracePeriodSeconds: ptr.To(int64(10)),
										ImagePullSecrets: []corev1.LocalObjectReference{
											{
												Name: "frontend-secret",
											},
										},
									},
									MainContainer: &corev1.Container{
										Command: []string{
											"/bin/sh",
											"-c",
											"echo $FRONTEND_ENV_1",
										},
										Args: []string{
											"--frontend-env-1",
											"1",
										},
										Image: "frontend-image",
									},
								},
							},
							"Planner": {
								Replicas:      &[]int32{2}[0],
								ComponentType: commonconsts.ComponentTypePlanner,
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
										GPU:    "2",
									},
								},
								Envs: []corev1.EnvVar{
									{
										Name:  "PLANNER_ENV_1",
										Value: "2",
									},
								},
								VolumeMounts: []v1alpha1.VolumeMount{
									{
										Name:       "dynamo-pvc",
										MountPoint: "/planner",
									},
								},
								EnvFromSecret: &[]string{"planner-secret"}[0],
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/health",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/ready",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									MainContainer: &corev1.Container{
										Command: []string{
											"/bin/sh",
											"-c",
											"echo $PLANNER_ENV_1",
										},
										Args: []string{
											"--planner-env-1",
											"1",
										},
										Image: "planner-image",
									},
								},
							},
						},
					},
				},
			},
			want: &grovev1alpha1.PodCliqueSet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-dynamo-graph-deployment",
					Namespace: "test-namespace",
				},
				Spec: grovev1alpha1.PodCliqueSetSpec{
					Replicas: 1,
					Template: grovev1alpha1.PodCliqueSetTemplateSpec{
						StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder),
						HeadlessServiceConfig: &grovev1alpha1.HeadlessServiceConfig{
							PublishNotReadyAddresses: true,
						},
						TerminationDelay: &metav1.Duration{Duration: 15 * time.Minute},
						Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{
							{
								Name: "frontend",
								Labels: map[string]string{
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-frontend",
									commonconsts.KubeLabelDynamoComponent:           "Frontend",
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypeFrontend,
									commonconsts.KubeLabelDynamoSubComponentType:    "test-sub-component",
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
									"nvidia.com/label1":                             "label1",
									"nvidia.com/label2":                             "label2",
								},
								Annotations: map[string]string{
									"nvidia.com/annotation1": "annotation1",
									"nvidia.com/annotation2": "annotation2",
								},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "frontend",
									Replicas:     1,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										Volumes: []corev1.Volume{
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										TerminationGracePeriodSeconds: ptr.To(int64(10)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										ImagePullSecrets: []corev1.LocalObjectReference{
											{
												Name: "frontend-secret",
											},
										},
										RestartPolicy: corev1.RestartPolicyAlways,
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "frontend-image",
												Command: []string{
													"/bin/sh",
													"-c",
													"echo $FRONTEND_ENV_1",
												},
												Args: []string{
													"--frontend-env-1",
													"1",
												},
												EnvFrom: []corev1.EnvFromSource{
													{
														SecretRef: &corev1.SecretEnvSource{
															LocalObjectReference: corev1.LocalObjectReference{
																Name: "frontend-secret",
															},
														},
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromInt(8080),
														},
													},
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/ready",
															Port: intstr.FromInt(8080),
														},
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYN_HTTP_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoServicePort),
													},
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "FRONTEND_ENV_1",
														Value: "1",
													},
													{
														Name:  "DYNAMO_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoServicePort),
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoNamespacePrefixEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypeFrontend,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name:  "MODEL_EXPRESS_URL",
														Value: "model-express-url",
													},
													{
														Name:  "PROMETHEUS_ENDPOINT",
														Value: "http://localhost:9090",
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("1"),
														corev1.ResourceMemory: resource.MustParse("1Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("1"),
														corev1.ResourceMemory:                 resource.MustParse("1Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      "shared-memory",
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoContainerPortName,
														ContainerPort: int32(commonconsts.DynamoServicePort),
													},
												},
											},
										},
									},
								},
							},
							{
								Name: "planner",
								Labels: map[string]string{
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-planner",
									commonconsts.KubeLabelDynamoComponent:           "Planner",
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypePlanner,
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
								},
								Annotations: map[string]string{},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "planner",
									Replicas:     2,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										Volumes: []corev1.Volume{
											{
												Name: "dynamo-pvc",
												VolumeSource: corev1.VolumeSource{
													PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
														ClaimName: "dynamo-pvc",
													},
												},
											},
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										ServiceAccountName:            commonconsts.PlannerServiceAccountName,
										TerminationGracePeriodSeconds: ptr.To(int64(60)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										RestartPolicy: corev1.RestartPolicyAlways,

										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "planner-image",
												Command: []string{
													"/bin/sh",
													"-c",
													"echo $PLANNER_ENV_1",
												},
												Args: []string{
													"--planner-env-1",
													"1",
												},
												EnvFrom: []corev1.EnvFromSource{
													{
														SecretRef: &corev1.SecretEnvSource{
															LocalObjectReference: corev1.LocalObjectReference{
																Name: "planner-secret",
															},
														},
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromInt(8080),
														},
													},
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/ready",
															Port: intstr.FromInt(8080),
														},
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "PLANNER_ENV_1",
														Value: "2",
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypePlanner,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name:  "MODEL_EXPRESS_URL",
														Value: "model-express-url",
													},
													{
														Name:  "PROMETHEUS_ENDPOINT",
														Value: "http://localhost:9090",
													},
													{
														Name:  "PLANNER_PROMETHEUS_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoPlannerMetricsPort),
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("2"),
														corev1.ResourceMemory: resource.MustParse("2Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("2"),
														corev1.ResourceMemory:                 resource.MustParse("2Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("2"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      "dynamo-pvc",
														MountPath: "/planner",
													},
													{
														Name:      "shared-memory",
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoMetricsPortName,
														ContainerPort: int32(commonconsts.DynamoPlannerMetricsPort),
													},
												},
											},
										},
									},
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "test_generate_grove_pod_gang_set_multinode sglang",
			args: args{
				ctx: context.Background(),
				controllerConfig: &configv1alpha1.OperatorConfiguration{
					Infrastructure: configv1alpha1.InfrastructureConfiguration{
						ETCDAddress: "etcd-address",
						NATSAddress: "nats-address",
					},
					Orchestrators: configv1alpha1.OrchestratorConfiguration{
						Grove: configv1alpha1.GroveConfiguration{
							TerminationDelay: metav1.Duration{Duration: 15 * time.Minute},
						},
					},
				},
				dynamoDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamo-graph-deployment",
						Namespace: "test-namespace",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						Envs: []corev1.EnvVar{
							{
								Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
								Value: "1",
							},
						},
						BackendFramework: string(BackendFrameworkSGLang),
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"Frontend": {
								Replicas: &[]int32{1}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "1",
									},
								},
								ComponentType: commonconsts.ComponentTypeFrontend,
								Envs: []corev1.EnvVar{
									{
										Name:  "FRONTEND_ENV_1",
										Value: "1",
									},
								},
								EnvFromSecret: &[]string{"frontend-secret"}[0],
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/health",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/ready",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									PodSpec: &corev1.PodSpec{
										ImagePullSecrets: []corev1.LocalObjectReference{
											{
												Name: "frontend-secret",
											},
										},
										TerminationGracePeriodSeconds: ptr.To(int64(10)),
									},
									MainContainer: &corev1.Container{
										Command: []string{
											"/bin/sh",
											"-c",
											"echo $FRONTEND_ENV_1",
										},
										Args: []string{
											"--frontend-env-1",
											"1",
										},
										Image: "frontend-image",
									},
								},
							},
							"worker": {
								Multinode: &v1alpha1.MultinodeSpec{
									NodeCount: 3,
								},
								ExtraPodMetadata: &v1alpha1.ExtraPodMetadata{
									Annotations: map[string]string{
										"nvidia.com/annotation1": "annotation1",
										"nvidia.com/annotation2": "annotation2",
									},
									Labels: map[string]string{
										"nvidia.com/label1": "label1",
										"nvidia.com/label2": "label2",
									},
								},
								Replicas:         &[]int32{5}[0],
								ComponentType:    commonconsts.ComponentTypeWorker,
								SubComponentType: "test-sub-component",
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									MainContainer: &corev1.Container{
										Image: "worker-image",
										Command: []string{
											"/bin/sh",
											"-c",
										},
										Args: []string{
											"python3 -m dynamo.sglang --custom-flag custom-value",
										},
									},
								},
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
										GPU:    "2",
									},
								},
								Envs: []corev1.EnvVar{
									{
										Name:  "WORKER_ENV_1",
										Value: "1",
									},
								},
							},
							"Planner": {
								ComponentType: commonconsts.ComponentTypePlanner,
								Replicas:      &[]int32{2}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
										GPU:    "2",
									},
								},
								Envs: []corev1.EnvVar{
									{
										Name:  "PLANNER_ENV_1",
										Value: "2",
									},
								},
								VolumeMounts: []v1alpha1.VolumeMount{
									{
										Name:       "dynamo-pvc",
										MountPoint: "/planner",
									},
								},
								EnvFromSecret: &[]string{"planner-secret"}[0],
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/health",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/ready",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									MainContainer: &corev1.Container{
										Command: []string{
											"/bin/sh",
											"-c",
											"echo $PLANNER_ENV_1",
										},
										Args: []string{
											"--planner-env-1",
											"1",
										},
										Image: "planner-image",
									},
								},
							},
						},
					},
				},
			},
			want: &grovev1alpha1.PodCliqueSet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-dynamo-graph-deployment",
					Namespace: "test-namespace",
				},
				Spec: grovev1alpha1.PodCliqueSetSpec{
					Replicas: 1,
					Template: grovev1alpha1.PodCliqueSetTemplateSpec{
						HeadlessServiceConfig: &grovev1alpha1.HeadlessServiceConfig{
							PublishNotReadyAddresses: true,
						},
						TerminationDelay: &metav1.Duration{Duration: 15 * time.Minute},
						PodCliqueScalingGroupConfigs: []grovev1alpha1.PodCliqueScalingGroupConfig{
							{
								Name: "worker",
								CliqueNames: []string{
									"worker-ldr",
									"worker-wkr",
								},
								Replicas:     ptr.To(int32(5)),
								MinAvailable: ptr.To(int32(1)),
							},
						},
						// StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeExplicit),
						StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder),
						Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{
							{
								Name: "worker-ldr",
								Labels: map[string]string{
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypeWorker,
									commonconsts.KubeLabelDynamoSubComponentType:    "test-sub-component",
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-worker",
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoComponent:           "worker",
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
									"nvidia.com/label1":                             "label1",
									"nvidia.com/label2":                             "label2",
								},
								Annotations: map[string]string{
									"nvidia.com/annotation1": "annotation1",
									"nvidia.com/annotation2": "annotation2",
								},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "worker-ldr",
									Replicas:     1,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										RestartPolicy:                 corev1.RestartPolicyAlways,
										TerminationGracePeriodSeconds: ptr.To(int64(60)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										Volumes: []corev1.Volume{
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "worker-image",
												Command: []string{
													"/bin/sh",
													"-c",
												},
												Args: []string{
													"python3 -m dynamo.sglang --dist-init-addr $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-worker-ldr-0.$(GROVE_HEADLESS_SERVICE):29500 --nnodes 3 --node-rank 0 --custom-flag custom-value",
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoSystemPortName,
														ContainerPort: int32(commonconsts.DynamoSystemPort),
													},
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoNixlPortName,
														ContainerPort: int32(commonconsts.DynamoNixlPort),
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "DYN_SYSTEM_ENABLED",
														Value: "true",
													},
													{
														Name:  "DYN_SYSTEM_PORT",
														Value: "9090",
													},
													{
														Name:  "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS",
														Value: "[\"generate\"]",
													},
													{
														Name:  "WORKER_ENV_1",
														Value: "1",
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypeWorker,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_HEALTH_CHECK_ENABLED",
														Value: "false",
													},
													{
														Name:  "NIXL_TELEMETRY_ENABLE",
														Value: "n",
													},
													{
														Name:  "NIXL_TELEMETRY_EXPORTER",
														Value: "prometheus",
													},
													{
														Name:  "NIXL_TELEMETRY_PROMETHEUS_PORT",
														Value: "19090",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("2"),
														corev1.ResourceMemory: resource.MustParse("2Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("2"),
														corev1.ResourceMemory:                 resource.MustParse("2Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("2"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      commonconsts.KubeValueNameSharedMemory,
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/live",
															Port: intstr.FromString(commonconsts.DynamoSystemPortName),
														},
													},
													TimeoutSeconds:   4,
													PeriodSeconds:    5,
													SuccessThreshold: 0,
													FailureThreshold: 1,
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromString(commonconsts.DynamoSystemPortName),
														},
													},
													TimeoutSeconds:   4,
													PeriodSeconds:    10,
													SuccessThreshold: 0,
													FailureThreshold: 3,
												},
												StartupProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/live",
															Port: intstr.FromString(commonconsts.DynamoSystemPortName),
														},
													},
													TimeoutSeconds:   5,
													PeriodSeconds:    10,
													SuccessThreshold: 0,
													FailureThreshold: 720,
												},
											},
										},
									},
								},
							},
							{
								Name: "worker-wkr",
								Labels: map[string]string{
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypeWorker,
									commonconsts.KubeLabelDynamoSubComponentType:    "test-sub-component",
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-worker",
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoComponent:           "worker",
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
									"nvidia.com/label1":                             "label1",
									"nvidia.com/label2":                             "label2",
								},
								Annotations: map[string]string{
									"nvidia.com/annotation1": "annotation1",
									"nvidia.com/annotation2": "annotation2",
								},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "worker-wkr",
									Replicas:     2,
									MinAvailable: ptr.To(int32(1)),
									// StartsAfter: []string{"worker-ldr"},
									PodSpec: corev1.PodSpec{
										RestartPolicy:                 corev1.RestartPolicyAlways,
										TerminationGracePeriodSeconds: ptr.To(int64(60)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										Volumes: []corev1.Volume{
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "worker-image",
												Command: []string{
													"/bin/sh",
													"-c",
												},
												Args: []string{
													"python3 -m dynamo.sglang --dist-init-addr $(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-worker-ldr-0.$(GROVE_HEADLESS_SERVICE):29500 --nnodes 3 --node-rank $((GROVE_PCLQ_POD_INDEX + 1)) --custom-flag custom-value",
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoSystemPortName,
														ContainerPort: int32(commonconsts.DynamoSystemPort),
													},
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoNixlPortName,
														ContainerPort: int32(commonconsts.DynamoNixlPort),
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "DYN_SYSTEM_ENABLED",
														Value: "true",
													},
													{
														Name:  "DYN_SYSTEM_PORT",
														Value: "9090",
													},
													{
														Name:  "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS",
														Value: "[\"generate\"]",
													},
													{
														Name:  "WORKER_ENV_1",
														Value: "1",
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypeWorker,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_HEALTH_CHECK_ENABLED",
														Value: "false",
													},
													{
														Name:  "NIXL_TELEMETRY_ENABLE",
														Value: "n",
													},
													{
														Name:  "NIXL_TELEMETRY_EXPORTER",
														Value: "prometheus",
													},
													{
														Name:  "NIXL_TELEMETRY_PROMETHEUS_PORT",
														Value: "19090",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("2"),
														corev1.ResourceMemory: resource.MustParse("2Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("2"),
														corev1.ResourceMemory:                 resource.MustParse("2Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("2"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      commonconsts.KubeValueNameSharedMemory,
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
											},
										},
									},
								},
							},
							{
								Name: "frontend",
								Labels: map[string]string{
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-frontend",
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypeFrontend,
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoComponent:           "Frontend",
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
								},
								Annotations: map[string]string{},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "frontend",
									Replicas:     1,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										Volumes: []corev1.Volume{
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										ImagePullSecrets: []corev1.LocalObjectReference{
											{
												Name: "frontend-secret",
											},
										},
										TerminationGracePeriodSeconds: ptr.To(int64(10)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										RestartPolicy: corev1.RestartPolicyAlways,
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "frontend-image",
												Command: []string{
													"/bin/sh",
													"-c",
													"echo $FRONTEND_ENV_1",
												},
												Args: []string{
													"--frontend-env-1",
													"1",
												},
												EnvFrom: []corev1.EnvFromSource{
													{
														SecretRef: &corev1.SecretEnvSource{
															LocalObjectReference: corev1.LocalObjectReference{
																Name: "frontend-secret",
															},
														},
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromInt(8080),
														},
													},
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/ready",
															Port: intstr.FromInt(8080),
														},
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYN_HTTP_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoServicePort),
													},
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "FRONTEND_ENV_1",
														Value: "1",
													},
													{
														Name:  "DYNAMO_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoServicePort),
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoNamespacePrefixEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypeFrontend,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("1"),
														corev1.ResourceMemory: resource.MustParse("1Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("1"),
														corev1.ResourceMemory:                 resource.MustParse("1Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"),
													},
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoContainerPortName,
														ContainerPort: int32(commonconsts.DynamoServicePort),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      commonconsts.KubeValueNameSharedMemory,
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
											},
										},
									},
								},
							},
							{
								Name: "planner",
								Labels: map[string]string{
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-planner",
									commonconsts.KubeLabelDynamoComponent:           "Planner",
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypePlanner,
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
								},
								Annotations: map[string]string{},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "planner",
									Replicas:     2,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										TerminationGracePeriodSeconds: ptr.To(int64(60)),
										ServiceAccountName:            commonconsts.PlannerServiceAccountName,
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										RestartPolicy: corev1.RestartPolicyAlways,
										Volumes: []corev1.Volume{
											{
												Name: "dynamo-pvc",
												VolumeSource: corev1.VolumeSource{
													PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
														ClaimName: "dynamo-pvc",
													},
												},
											},
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "planner-image",
												Command: []string{
													"/bin/sh",
													"-c",
													"echo $PLANNER_ENV_1",
												},
												Args: []string{
													"--planner-env-1",
													"1",
												},
												EnvFrom: []corev1.EnvFromSource{
													{
														SecretRef: &corev1.SecretEnvSource{
															LocalObjectReference: corev1.LocalObjectReference{
																Name: "planner-secret",
															},
														},
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromInt(8080),
														},
													},
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/ready",
															Port: intstr.FromInt(8080),
														},
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "PLANNER_ENV_1",
														Value: "2",
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypePlanner,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name:  "PLANNER_PROMETHEUS_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoPlannerMetricsPort),
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("2"),
														corev1.ResourceMemory: resource.MustParse("2Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("2"),
														corev1.ResourceMemory:                 resource.MustParse("2Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("2"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      "dynamo-pvc",
														MountPath: "/planner",
													},
													{
														Name:      "shared-memory",
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoMetricsPortName,
														ContainerPort: int32(commonconsts.DynamoPlannerMetricsPort),
													},
												},
											},
										},
									},
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "test_generate_grove_pod_gang_set_multinode vllm",
			args: args{
				ctx: context.Background(),
				controllerConfig: &configv1alpha1.OperatorConfiguration{
					Infrastructure: configv1alpha1.InfrastructureConfiguration{
						ETCDAddress: "etcd-address",
						NATSAddress: "nats-address",
					},
					Orchestrators: configv1alpha1.OrchestratorConfiguration{
						Grove: configv1alpha1.GroveConfiguration{
							TerminationDelay: metav1.Duration{Duration: 15 * time.Minute},
						},
					},
				},
				dynamoDeployment: &v1alpha1.DynamoGraphDeployment{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "test-dynamo-graph-deployment",
						Namespace: "test-namespace",
					},
					Spec: v1alpha1.DynamoGraphDeploymentSpec{
						Envs: []corev1.EnvVar{
							{
								Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
								Value: "1",
							},
						},
						BackendFramework: string(BackendFrameworkVLLM),
						Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
							"Frontend": {
								Replicas:      &[]int32{1}[0],
								ComponentType: commonconsts.ComponentTypeFrontend,
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "1",
										Memory: "1Gi",
										GPU:    "1",
									},
								},
								Envs: []corev1.EnvVar{
									{
										Name:  "FRONTEND_ENV_1",
										Value: "1",
									},
								},
								EnvFromSecret: &[]string{"frontend-secret"}[0],
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/health",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/ready",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									PodSpec: &corev1.PodSpec{
										ImagePullSecrets: []corev1.LocalObjectReference{
											{
												Name: "frontend-secret",
											},
										},
										TerminationGracePeriodSeconds: ptr.To(int64(10)),
									},
									MainContainer: &corev1.Container{
										Command: []string{
											"/bin/sh",
											"-c",
											"echo $FRONTEND_ENV_1",
										},
										Args: []string{
											"--frontend-env-1",
											"1",
										},
										Image: "frontend-image",
									},
								},
							},
							"worker": {

								Multinode: &v1alpha1.MultinodeSpec{
									NodeCount: 3,
								},
								ExtraPodMetadata: &v1alpha1.ExtraPodMetadata{
									Annotations: map[string]string{
										"nvidia.com/annotation1": "annotation1",
										"nvidia.com/annotation2": "annotation2",
									},
									Labels: map[string]string{
										"nvidia.com/label1": "label1",
										"nvidia.com/label2": "label2",
									},
								},
								Replicas:      &[]int32{5}[0],
								ComponentType: commonconsts.ComponentTypeWorker,
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									MainContainer: &corev1.Container{
										Image: "worker-image",
										Command: []string{
											"python3",
											"-m",
											"dynamo.vllm",
										},
										Args: []string{
											"--custom-flag",
											"custom-value",
											"--tensor-parallel-size",
											"4",
											"--pipeline-parallel-size",
											"1",
										},
										StartupProbe: &corev1.Probe{
											ProbeHandler: corev1.ProbeHandler{
												HTTPGet: &corev1.HTTPGetAction{
													Path: "/startup",
													Port: intstr.FromInt(8080),
												},
											},
										},
									},
								},
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/ready",
											Port: intstr.FromInt(8080),
										},
									},
								},
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/health",
											Port: intstr.FromInt(8080),
										},
									},
								},
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
										GPU:    "2",
									},
								},
								Envs: []corev1.EnvVar{
									{
										Name:  "WORKER_ENV_1",
										Value: "1",
									},
								},
							},
							"Planner": {

								ComponentType: commonconsts.ComponentTypePlanner,
								Replicas:      &[]int32{2}[0],
								Resources: &v1alpha1.Resources{
									Requests: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
									},
									Limits: &v1alpha1.ResourceItem{
										CPU:    "2",
										Memory: "2Gi",
										GPU:    "2",
									},
								},
								Envs: []corev1.EnvVar{
									{
										Name:  "PLANNER_ENV_1",
										Value: "2",
									},
								},
								VolumeMounts: []v1alpha1.VolumeMount{
									{
										Name:       "dynamo-pvc",
										MountPoint: "/planner",
									},
								},
								EnvFromSecret: &[]string{"planner-secret"}[0],
								LivenessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/health",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ReadinessProbe: &corev1.Probe{
									ProbeHandler: corev1.ProbeHandler{
										HTTPGet: &corev1.HTTPGetAction{
											Path: "/ready",
											Port: intstr.FromInt(8080),
										},
									},
								},
								ExtraPodSpec: &v1alpha1.ExtraPodSpec{
									MainContainer: &corev1.Container{
										Command: []string{
											"/bin/sh",
											"-c",
											"echo $PLANNER_ENV_1",
										},
										Args: []string{
											"--planner-env-1",
											"1",
										},
										Image: "planner-image",
									},
								},
							},
						},
					},
				},
			},
			want: &grovev1alpha1.PodCliqueSet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-dynamo-graph-deployment",
					Namespace: "test-namespace",
				},
				Spec: grovev1alpha1.PodCliqueSetSpec{
					Replicas: 1,
					Template: grovev1alpha1.PodCliqueSetTemplateSpec{
						StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder),
						HeadlessServiceConfig: &grovev1alpha1.HeadlessServiceConfig{
							PublishNotReadyAddresses: true,
						},
						TerminationDelay: &metav1.Duration{Duration: 15 * time.Minute},
						PodCliqueScalingGroupConfigs: []grovev1alpha1.PodCliqueScalingGroupConfig{
							{
								Name: "worker",
								CliqueNames: []string{
									"worker-ldr",
									"worker-wkr",
								},
								Replicas:     ptr.To(int32(5)),
								MinAvailable: ptr.To(int32(1)),
							},
						},
						// StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeExplicit),
						Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{
							{
								Name: "worker-ldr",
								Labels: map[string]string{
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-worker",
									commonconsts.KubeLabelDynamoComponent:           "worker",
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypeWorker,
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
									"nvidia.com/label1":                             "label1",
									"nvidia.com/label2":                             "label2",
								},
								Annotations: map[string]string{
									"nvidia.com/annotation1": "annotation1",
									"nvidia.com/annotation2": "annotation2",
								},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "worker-ldr",
									Replicas:     1,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										Volumes: []corev1.Volume{
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										TerminationGracePeriodSeconds: ptr.To(int64(60)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										RestartPolicy: corev1.RestartPolicyAlways,
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "worker-image",
												Command: []string{
													"/bin/sh",
													"-c",
												},
												Args: []string{
													"ray start --head --port=6379 && python3 -m dynamo.vllm --custom-flag custom-value --tensor-parallel-size 4 --pipeline-parallel-size 1 --distributed-executor-backend ray",
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoSystemPortName,
														ContainerPort: int32(commonconsts.DynamoSystemPort),
													},
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoNixlPortName,
														ContainerPort: int32(commonconsts.DynamoNixlPort),
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "DYN_SYSTEM_PORT",
														Value: "9090",
													},
													{
														Name:  "DYN_SYSTEM_ENABLED",
														Value: "true",
													},
													{
														Name:  "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS",
														Value: "[\"generate\"]",
													},
													{
														Name:  "WORKER_ENV_1",
														Value: "1",
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypeWorker,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_HEALTH_CHECK_ENABLED",
														Value: "false",
													},
													{
														Name:  "NIXL_TELEMETRY_ENABLE",
														Value: "n",
													},
													{
														Name:  "NIXL_TELEMETRY_EXPORTER",
														Value: "prometheus",
													},
													{
														Name:  "NIXL_TELEMETRY_PROMETHEUS_PORT",
														Value: "19090",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("2"),
														corev1.ResourceMemory: resource.MustParse("2Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("2"),
														corev1.ResourceMemory:                 resource.MustParse("2Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("2"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      commonconsts.KubeValueNameSharedMemory,
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/ready",
															Port: intstr.FromInt(8080),
														},
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromInt(8080),
														},
													},
												},
												StartupProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/startup",
															Port: intstr.FromInt(8080),
														},
													},
												},
											},
										},
									},
								},
							},
							{
								Name: "worker-wkr",
								Labels: map[string]string{
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypeWorker,
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-worker",
									commonconsts.KubeLabelDynamoComponent:           "worker",
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
									"nvidia.com/label1":                             "label1",
									"nvidia.com/label2":                             "label2",
								},
								Annotations: map[string]string{
									"nvidia.com/annotation1": "annotation1",
									"nvidia.com/annotation2": "annotation2",
								},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "worker-wkr",
									Replicas:     2,
									MinAvailable: ptr.To(int32(1)),
									// StartsAfter: []string{"worker-ldr"},
									PodSpec: corev1.PodSpec{
										TerminationGracePeriodSeconds: ptr.To(int64(60)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										Volumes: []corev1.Volume{
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										RestartPolicy: corev1.RestartPolicyAlways,
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "worker-image",
												Command: []string{
													"/bin/sh",
													"-c",
												},
												Args: []string{
													"ray start --address=$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-worker-ldr-0.$(GROVE_HEADLESS_SERVICE):6379 --block",
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoSystemPortName,
														ContainerPort: int32(commonconsts.DynamoSystemPort),
													},
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoNixlPortName,
														ContainerPort: int32(commonconsts.DynamoNixlPort),
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "DYN_SYSTEM_PORT",
														Value: "9090",
													},
													{
														Name:  "DYN_SYSTEM_ENABLED",
														Value: "true",
													},
													{
														Name:  "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS",
														Value: "[\"generate\"]",
													},
													{
														Name:  "WORKER_ENV_1",
														Value: "1",
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypeWorker,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_HEALTH_CHECK_ENABLED",
														Value: "false",
													},
													{
														Name:  "NIXL_TELEMETRY_ENABLE",
														Value: "n",
													},
													{
														Name:  "NIXL_TELEMETRY_EXPORTER",
														Value: "prometheus",
													},
													{
														Name:  "NIXL_TELEMETRY_PROMETHEUS_PORT",
														Value: "19090",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("2"),
														corev1.ResourceMemory: resource.MustParse("2Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("2"),
														corev1.ResourceMemory:                 resource.MustParse("2Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("2"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      commonconsts.KubeValueNameSharedMemory,
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
											},
										},
									},
								},
							},
							{
								Name: "frontend",
								Labels: map[string]string{
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypeFrontend,
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-frontend",
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoComponent:           "Frontend",
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
								},
								Annotations: map[string]string{},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "frontend",
									Replicas:     1,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										Volumes: []corev1.Volume{
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										ImagePullSecrets: []corev1.LocalObjectReference{
											{
												Name: "frontend-secret",
											},
										},
										TerminationGracePeriodSeconds: ptr.To(int64(10)),
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										RestartPolicy: corev1.RestartPolicyAlways,
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "frontend-image",
												Command: []string{
													"/bin/sh",
													"-c",
													"echo $FRONTEND_ENV_1",
												},
												Args: []string{
													"--frontend-env-1",
													"1",
												},
												EnvFrom: []corev1.EnvFromSource{
													{
														SecretRef: &corev1.SecretEnvSource{
															LocalObjectReference: corev1.LocalObjectReference{
																Name: "frontend-secret",
															},
														},
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromInt(8080),
														},
													},
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/ready",
															Port: intstr.FromInt(8080),
														},
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYN_HTTP_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoServicePort),
													},
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "FRONTEND_ENV_1",
														Value: "1",
													},
													{
														Name:  "DYNAMO_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoServicePort),
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoNamespacePrefixEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypeFrontend,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("1"),
														corev1.ResourceMemory: resource.MustParse("1Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("1"),
														corev1.ResourceMemory:                 resource.MustParse("1Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"),
													},
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoContainerPortName,
														ContainerPort: int32(commonconsts.DynamoServicePort),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      commonconsts.KubeValueNameSharedMemory,
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
											},
										},
									},
								},
							},
							{
								Name: "planner",
								Labels: map[string]string{
									commonconsts.KubeLabelMetricsEnabled:            commonconsts.KubeLabelValueTrue,
									commonconsts.KubeLabelDynamoSelector:            "test-dynamo-graph-deployment-planner",
									commonconsts.KubeLabelDynamoComponent:           "Planner",
									commonconsts.KubeLabelDynamoGraphDeploymentName: "test-dynamo-graph-deployment",
									commonconsts.KubeLabelDynamoComponentType:       commonconsts.ComponentTypePlanner,
									commonconsts.KubeLabelDynamoNamespace:           "test-namespace-test-dynamo-graph-deployment",
								},
								Annotations: map[string]string{},
								Spec: grovev1alpha1.PodCliqueSpec{
									RoleName:     "planner",
									Replicas:     2,
									MinAvailable: ptr.To(int32(1)),
									PodSpec: corev1.PodSpec{
										TerminationGracePeriodSeconds: ptr.To(int64(60)),
										ServiceAccountName:            commonconsts.PlannerServiceAccountName,
										SecurityContext: &corev1.PodSecurityContext{
											FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
										},
										Volumes: []corev1.Volume{
											{
												Name: "dynamo-pvc",
												VolumeSource: corev1.VolumeSource{
													PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
														ClaimName: "dynamo-pvc",
													},
												},
											},
											{
												Name: "shared-memory",
												VolumeSource: corev1.VolumeSource{
													EmptyDir: &corev1.EmptyDirVolumeSource{
														Medium:    corev1.StorageMediumMemory,
														SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
													},
												},
											},
										},
										RestartPolicy: corev1.RestartPolicyAlways,
										Containers: []corev1.Container{
											{
												Name:  commonconsts.MainContainerName,
												Image: "planner-image",
												Command: []string{
													"/bin/sh",
													"-c",
													"echo $PLANNER_ENV_1",
												},
												Args: []string{
													"--planner-env-1",
													"1",
												},
												EnvFrom: []corev1.EnvFromSource{
													{
														SecretRef: &corev1.SecretEnvSource{
															LocalObjectReference: corev1.LocalObjectReference{
																Name: "planner-secret",
															},
														},
													},
												},
												LivenessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/health",
															Port: intstr.FromInt(8080),
														},
													},
												},
												ReadinessProbe: &corev1.Probe{
													ProbeHandler: corev1.ProbeHandler{
														HTTPGet: &corev1.HTTPGetAction{
															Path: "/ready",
															Port: intstr.FromInt(8080),
														},
													},
												},
												Ports: []corev1.ContainerPort{
													{
														Protocol:      corev1.ProtocolTCP,
														Name:          commonconsts.DynamoMetricsPortName,
														ContainerPort: int32(commonconsts.DynamoPlannerMetricsPort),
													},
												},
												Env: []corev1.EnvVar{
													{
														Name:  "DYNAMO_POD_GANG_SET_REPLICAS",
														Value: "1",
													},
													{
														Name:  "PLANNER_ENV_1",
														Value: "2",
													},
													{
														Name:  "NATS_SERVER",
														Value: "nats-address",
													},
													{
														Name:  "ETCD_ENDPOINTS",
														Value: "etcd-address",
													},
													{
														Name:  commonconsts.DynamoNamespaceEnvVar,
														Value: "test-namespace-test-dynamo-graph-deployment",
													},
													{
														Name:  commonconsts.DynamoComponentEnvVar,
														Value: commonconsts.ComponentTypePlanner,
													},
													{
														Name:  commonconsts.DynamoDiscoveryBackendEnvVar,
														Value: "kubernetes",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAME",
														Value: "test-dynamo-graph-deployment",
													},
													{
														Name:  "DYN_PARENT_DGD_K8S_NAMESPACE",
														Value: "test-namespace",
													},
													{
														Name:  "PLANNER_PROMETHEUS_PORT",
														Value: fmt.Sprintf("%d", commonconsts.DynamoPlannerMetricsPort),
													},
													{
														Name: "POD_NAME",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.name",
															},
														},
													},
													{
														Name: "POD_NAMESPACE",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.namespace",
															},
														},
													},
													{
														Name: "POD_UID",
														ValueFrom: &corev1.EnvVarSource{
															FieldRef: &corev1.ObjectFieldSelector{
																FieldPath: "metadata.uid",
															},
														},
													},
												},
												Resources: corev1.ResourceRequirements{
													Requests: corev1.ResourceList{
														corev1.ResourceCPU:    resource.MustParse("2"),
														corev1.ResourceMemory: resource.MustParse("2Gi"),
													},
													Limits: corev1.ResourceList{
														corev1.ResourceCPU:                    resource.MustParse("2"),
														corev1.ResourceMemory:                 resource.MustParse("2Gi"),
														corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("2"),
													},
												},
												VolumeMounts: []corev1.VolumeMount{
													{
														Name:      "dynamo-pvc",
														MountPath: "/planner",
													},
													{
														Name:      "shared-memory",
														MountPath: commonconsts.DefaultSharedMemoryMountPath,
													},
												},
											},
										},
									},
								},
							},
						},
					},
				},
			},
			wantErr: false,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := GenerateGrovePodCliqueSet(tt.args.ctx, tt.args.dynamoDeployment, tt.args.controllerConfig, &controller_common.RuntimeConfig{}, nil, nil, nil, nil)
			if (err != nil) != tt.wantErr {
				t.Errorf("GenerateGrovePodCliqueSet() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			sort.Slice(got.Spec.Template.Cliques, func(i, j int) bool {
				return got.Spec.Template.Cliques[i].Name < got.Spec.Template.Cliques[j].Name
			})
			sort.Slice(tt.want.Spec.Template.Cliques, func(i, j int) bool {
				return tt.want.Spec.Template.Cliques[i].Name < tt.want.Spec.Template.Cliques[j].Name
			})

			// Sort environment variables for all containers in all cliques
			for _, clique := range got.Spec.Template.Cliques {
				for i := range clique.Spec.PodSpec.Containers {
					clique.Spec.PodSpec.Containers[i].Env = sortEnvVars(clique.Spec.PodSpec.Containers[i].Env)
				}
			}
			for _, clique := range tt.want.Spec.Template.Cliques {
				for i := range clique.Spec.PodSpec.Containers {
					clique.Spec.PodSpec.Containers[i].Env = sortEnvVars(clique.Spec.PodSpec.Containers[i].Env)
				}
			}

			if diff := cmp.Diff(got, tt.want); diff != "" {
				t.Errorf("GenerateGrovePodCliqueSet() mismatch (-want +got):\n%s", diff)
			}
		})
	}
}

func sortEnvVars(envs []corev1.EnvVar) []corev1.EnvVar {
	sorted := make([]corev1.EnvVar, len(envs))
	copy(sorted, envs)
	sort.Slice(sorted, func(i, j int) bool {
		return sorted[i].Name < sorted[j].Name
	})
	return sorted
}

func Test_GeneratePodCliqueSetGlobalDynamoNamespace(t *testing.T) {
	dynamoDeployment := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-dynamo-graph",
			Namespace: "k8s-namespace",
		},
		Spec: v1alpha1.DynamoGraphDeploymentSpec{
			Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType:         commonconsts.ComponentTypeFrontend,
					GlobalDynamoNamespace: true,
					Replicas:              ptr.To(int32(1)),
				},
				"Planner": {
					ComponentType: commonconsts.ComponentTypePlanner,
					Replicas:      ptr.To(int32(1)),
				},
			},
		},
	}

	got, err := GenerateGrovePodCliqueSet(context.Background(), dynamoDeployment, &configv1alpha1.OperatorConfiguration{}, &controller_common.RuntimeConfig{}, nil, nil, nil, nil)
	if !assert.NoError(t, err) {
		return
	}

	if !assert.Len(t, got.Spec.Template.Cliques, 2) {
		return
	}

	for _, clique := range got.Spec.Template.Cliques {
		switch clique.Name {
		case "frontend":
			assert.Equal(t, commonconsts.GlobalDynamoNamespace, clique.Labels[commonconsts.KubeLabelDynamoNamespace])
			assertDYNNamespace(t, clique.Spec.PodSpec, commonconsts.GlobalDynamoNamespace)
		case "planner":
			expectedNamespace := fmt.Sprintf("%s-%s", dynamoDeployment.Namespace, dynamoDeployment.Name)
			assert.Equal(t, expectedNamespace, clique.Labels[commonconsts.KubeLabelDynamoNamespace])
			assertDYNNamespace(t, clique.Spec.PodSpec, expectedNamespace)
		default:
			t.Errorf("GenerateGrovePodCliqueSet() clique = %v, want %v", clique.Name, "frontend or planner")
		}
	}
}

func assertDYNNamespace(t *testing.T, podSpec corev1.PodSpec, expectedNamespace string) {
	if assert.Len(t, podSpec.Containers, 1) {
		foundDYNNamespace := false
		for _, env := range podSpec.Containers[0].Env {
			if env.Name == commonconsts.DynamoNamespaceEnvVar {
				assert.Equal(t, expectedNamespace, env.Value)
				foundDYNNamespace = true
				break
			}
		}
		assert.True(t, foundDYNNamespace, fmt.Sprintf("%s not found in container environment variables", commonconsts.DynamoNamespaceEnvVar))
	}
}

// Mock SecretsRetriever for testing
type mockSecretsRetriever struct{}

func (m *mockSecretsRetriever) RetrieveImagePullSecrets(ctx context.Context, deployment *v1alpha1.DynamoGraphDeployment) ([]corev1.LocalObjectReference, error) {
	return []corev1.LocalObjectReference{}, nil
}

func (m *mockSecretsRetriever) GetSecrets(namespace, registry string) ([]string, error) {
	return []string{}, nil
}

// Mock SecretsRetriever that returns secrets for testing docker secrets functionality
type mockSecretsRetrieverWithSecrets struct{}

func (m *mockSecretsRetrieverWithSecrets) RetrieveImagePullSecrets(ctx context.Context, deployment *v1alpha1.DynamoGraphDeployment) ([]corev1.LocalObjectReference, error) {
	return []corev1.LocalObjectReference{
		{Name: "test-docker-secret"},
	}, nil
}

func (m *mockSecretsRetrieverWithSecrets) GetSecrets(namespace, registry string) ([]string, error) {
	// Return some mock secrets when called
	return []string{"test-docker-secret"}, nil
}

func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	dynamoDeployment := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-deployment",
			Namespace: "default",
		},
	}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name              string
		component         *v1alpha1.DynamoComponentDeploymentSharedSpec
		backendFramework  BackendFramework
		role              Role
		numberOfNodes     int32
		expectError       bool
		expectContains    []string
		expectNotContains []string
	}{
		{
			name: "SGLang single node worker",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python3 -m dynamo.sglang"},
					},
				},
			},
			backendFramework:  BackendFrameworkSGLang,
			role:              RoleMain,
			numberOfNodes:     1,
			expectError:       false,
			expectContains:    []string{"python3", "-m", "dynamo.sglang"},
			expectNotContains: []string{"dist-init-addr", "nnodes", "tp-size"},
		},
		{
			name: "SGLang multinode leader",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python3 -m dynamo.sglang"},
					},
				},
			},
			backendFramework: BackendFrameworkSGLang,
			role:             RoleLeader,
			numberOfNodes:    3,
			expectError:      false,
			expectContains:   []string{"python3", "-m", "dynamo.sglang", "dist-init-addr", "nnodes", "node-rank"},
		},
		{
			name: "SGLang multinode worker",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python3 -m dynamo.sglang"},
					},
				},
			},
			backendFramework: BackendFrameworkSGLang,
			role:             RoleWorker,
			numberOfNodes:    3,
			expectError:      false,
			expectContains:   []string{"python3", "-m", "dynamo.sglang", "dist-init-addr", "nnodes", "node-rank"},
		},
		{
			name: "SGLang with user command override",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,

				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Command: []string{"custom", "command"},
					},
				},
			},
			backendFramework: BackendFrameworkSGLang,
			role:             RoleMain,
			numberOfNodes:    1,
			expectError:      false,
			expectContains:   []string{},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GeneratePodSpecForComponent(
				tt.component,
				tt.backendFramework,
				secretsRetriever,
				dynamoDeployment,
				tt.role,
				tt.numberOfNodes,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"worker",
				nil, // No checkpoint info in tests
			)

			if tt.expectError {
				if err == nil {
					t.Errorf("GeneratePodSpecForComponent() expected error, got nil")
				}
				return
			}

			if err != nil {
				t.Errorf("GeneratePodSpecForComponent() unexpected error: %v", err)
				return
			}

			// Check container exists
			if len(podSpec.Containers) == 0 {
				t.Errorf("GeneratePodSpecForComponent() no containers in podSpec")
				return
			}

			container := podSpec.Containers[0]

			// Check command and args contain expected strings
			allArgs := append(container.Command, container.Args...)
			allArgsStr := strings.Join(allArgs, " ")

			for _, expected := range tt.expectContains {
				if !strings.Contains(allArgsStr, expected) {
					t.Errorf("GeneratePodSpecForComponent() args = %v, should contain %s", allArgs, expected)
				}
			}

			for _, notExpected := range tt.expectNotContains {
				if strings.Contains(allArgsStr, notExpected) {
					t.Errorf("GeneratePodSpecForComponent() args = %v, should NOT contain %s", allArgs, notExpected)
				}
			}

			// Check that container name is set
			if container.Name != commonconsts.MainContainerName {
				t.Errorf("GeneratePodSpecForComponent() container name = %s, want main", container.Name)
			}
		})
	}
}

func TestGeneratePodSpecForComponent_VLLM(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	dynamoDeployment := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-deployment",
			Namespace: "default",
		},
	}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name              string
		component         *v1alpha1.DynamoComponentDeploymentSharedSpec
		backendFramework  BackendFramework
		role              Role
		numberOfNodes     int32
		expectError       bool
		expectContains    []string
		expectNotContains []string
	}{
		{
			name: "VLLM single node worker",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python3", "-m", "dynamo.vllm"},
					},
				},
			},
			backendFramework:  BackendFrameworkVLLM,
			role:              RoleMain,
			numberOfNodes:     1,
			expectError:       false,
			expectContains:    []string{"python3", "-m", "dynamo.vllm"},
			expectNotContains: []string{"ray start"},
		},
		{
			name: "VLLM multinode leader",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python3", "-m", "dynamo.vllm", "--tensor-parallel-size", "4", "--pipeline-parallel-size", "1"},
					},
				},
				Resources: &v1alpha1.Resources{
					Limits: &v1alpha1.ResourceItem{
						GPU: "2",
					},
				},
			},
			backendFramework: BackendFrameworkVLLM,
			role:             RoleLeader,
			numberOfNodes:    3,
			expectError:      false,
			expectContains:   []string{"ray start --head --port=6379", "python3", "-m", "dynamo.vllm"},
		},
		{
			name: "VLLM multinode worker",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python3", "-m", "dynamo.vllm", "--tensor-parallel-size", "4", "--pipeline-parallel-size", "1"},
					},
				},
				Resources: &v1alpha1.Resources{
					Limits: &v1alpha1.ResourceItem{
						GPU: "2",
					},
				},
			},
			backendFramework:  BackendFrameworkVLLM,
			role:              RoleWorker,
			numberOfNodes:     3,
			expectError:       false,
			expectContains:    []string{"ray start --address=$(GROVE_PCSG_NAME)-$(GROVE_PCSG_INDEX)-worker-ldr-0.$(GROVE_HEADLESS_SERVICE):6379 --block"},
			expectNotContains: []string{"python3 -m dynamo.vllm"},
		},
		{
			name: "VLLM worker single node",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python3", "-m", "dynamo.vllm", "--is-prefill-worker"},
					},
				},
			},
			backendFramework:  BackendFrameworkVLLM,
			role:              RoleMain,
			numberOfNodes:     1,
			expectError:       false,
			expectContains:    []string{"python3", "-m", "dynamo.vllm", "--is-prefill-worker"},
			expectNotContains: []string{"ray start"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GeneratePodSpecForComponent(
				tt.component,
				tt.backendFramework,
				secretsRetriever,
				dynamoDeployment,
				tt.role,
				tt.numberOfNodes,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"worker",
				nil, // No checkpoint info in tests
			)

			if tt.expectError {
				if err == nil {
					t.Errorf("GeneratePodSpecForComponent() expected error, got nil")
				}
				return
			}

			if err != nil {
				t.Errorf("GeneratePodSpecForComponent() unexpected error: %v", err)
				return
			}

			// Check container exists
			if len(podSpec.Containers) == 0 {
				t.Errorf("GeneratePodSpecForComponent() no containers in podSpec")
				return
			}

			container := podSpec.Containers[0]

			// Check command and args contain expected strings
			allArgs := append(container.Command, container.Args...)
			allArgsStr := strings.Join(allArgs, " ")

			for _, expected := range tt.expectContains {
				if !strings.Contains(allArgsStr, expected) {
					t.Errorf("GeneratePodSpecForComponent() args = %v, should contain %s", allArgs, expected)
				}
			}

			for _, notExpected := range tt.expectNotContains {
				if strings.Contains(allArgsStr, notExpected) {
					t.Errorf("GeneratePodSpecForComponent() args = %v, should NOT contain %s", allArgs, notExpected)
				}
			}
		})
	}
}

func TestGeneratePodSpecForComponent_UnsupportedBackend(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	dynamoDeployment := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-deployment",
			Namespace: "default",
		},
	}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	component := &v1alpha1.DynamoComponentDeploymentSharedSpec{
		ComponentType: commonconsts.ComponentTypeWorker,
	}

	tests := []struct {
		name             string
		backendFramework BackendFramework
		expectError      bool
		errorContains    string
	}{
		{
			name:             "TRTLLM backend implemented",
			backendFramework: BackendFrameworkTRTLLM,
			expectError:      false,
		},
		{
			name:             "unknown backend",
			backendFramework: BackendFramework("unknown"),
			expectError:      true,
			errorContains:    "unsupported backend framework",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			_, err := GeneratePodSpecForComponent(
				component,
				tt.backendFramework,
				secretsRetriever,
				dynamoDeployment,
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"worker",
				nil, // No checkpoint info in tests
			)

			if tt.expectError {
				if err == nil {
					t.Errorf("GeneratePodSpecForComponent() expected error, got nil")
					return
				}
				if !strings.Contains(err.Error(), tt.errorContains) {
					t.Errorf("GeneratePodSpecForComponent() error = %v, should contain %s", err, tt.errorContains)
				}
			} else {
				if err != nil {
					t.Errorf("GeneratePodSpecForComponent() unexpected error: %v", err)
				}
			}
		})
	}
}

func TestExpandRolesForService(t *testing.T) {
	tests := []struct {
		name            string
		serviceName     string
		numberOfNodes   int32
		serviceReplicas int32
		expected        []ServiceRole
	}{
		{
			name:            "single node",
			serviceName:     "test-service",
			numberOfNodes:   1,
			serviceReplicas: 2,
			expected: []ServiceRole{
				{Name: "test-service", Role: RoleMain, Replicas: 2},
			},
		},
		{
			name:          "multinode 2 nodes",
			serviceName:   "test-service",
			numberOfNodes: 2,
			expected: []ServiceRole{
				{Name: "test-service-ldr", Role: RoleLeader, Replicas: 1},
				{Name: "test-service-wkr", Role: RoleWorker, Replicas: 1},
			},
		},
		{
			name:          "multinode 5 nodes",
			serviceName:   "test-service",
			numberOfNodes: 5,
			expected: []ServiceRole{
				{Name: "test-service-ldr", Role: RoleLeader, Replicas: 1},
				{Name: "test-service-wkr", Role: RoleWorker, Replicas: 4},
			},
		},
		{
			name:            "zero nodes should return main",
			serviceName:     "test-service",
			numberOfNodes:   0,
			serviceReplicas: 1,
			expected: []ServiceRole{
				{Name: "test-service", Role: RoleMain, Replicas: 1},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := expandRolesForService(tt.serviceName, &tt.serviceReplicas, tt.numberOfNodes)
			if !reflect.DeepEqual(result, tt.expected) {
				t.Errorf("expandRolesForService() = %v, want %v", result, tt.expected)
			}
		})
	}
}

func TestRoleEnum(t *testing.T) {
	// Test that role constants are defined correctly
	if RoleLeader != "leader" {
		t.Errorf("RoleLeader = %v, want \"leader\"", RoleLeader)
	}
	if RoleWorker != "worker" {
		t.Errorf("RoleWorker = %v, want \"worker\"", RoleWorker)
	}
	if RoleMain != "main" {
		t.Errorf("RoleMain = %v, want \"main\"", RoleMain)
	}

	// Test that roles can be compared
	roles := []Role{RoleLeader, RoleWorker, RoleMain}
	for _, role := range roles {
		switch role {
		case RoleLeader, RoleWorker, RoleMain:
			// Expected
		default:
			t.Errorf("Unexpected role value: %v", role)
		}
	}
}

func TestBackendFrameworkEnum(t *testing.T) {
	// Test that backend framework constants are defined correctly
	if BackendFrameworkSGLang != "sglang" {
		t.Errorf("BackendFrameworkSGLang = %v, want \"sglang\"", BackendFrameworkSGLang)
	}
	if BackendFrameworkVLLM != "vllm" {
		t.Errorf("BackendFrameworkVLLM = %v, want \"vllm\"", BackendFrameworkVLLM)
	}
	if BackendFrameworkTRTLLM != "trtllm" {
		t.Errorf("BackendFrameworkTRTLLM = %v, want \"trtllm\"", BackendFrameworkTRTLLM)
	}

	// Test that frameworks can be compared
	frameworks := []BackendFramework{BackendFrameworkSGLang, BackendFrameworkVLLM, BackendFrameworkTRTLLM}
	for _, framework := range frameworks {
		switch framework {
		case BackendFrameworkSGLang, BackendFrameworkVLLM, BackendFrameworkTRTLLM:
			// Expected
		default:
			t.Errorf("Unexpected framework value: %v", framework)
		}
	}
}

func TestServiceRoleStruct(t *testing.T) {
	// Test ServiceRole struct creation and field access
	sr := ServiceRole{
		Name:     "test-service",
		Role:     RoleLeader,
		Replicas: 3,
	}

	if sr.Name != "test-service" {
		t.Errorf("ServiceRole.Name = %v, want \"test-service\"", sr.Name)
	}
	if sr.Role != RoleLeader {
		t.Errorf("ServiceRole.Role = %v, want %v", sr.Role, RoleLeader)
	}
	if sr.Replicas != 3 {
		t.Errorf("ServiceRole.Replicas = %v, want 3", sr.Replicas)
	}
}

func TestDetectBackendFrameworkFromArgs(t *testing.T) {
	tests := []struct {
		name        string
		command     []string
		args        []string
		expected    BackendFramework
		expectError bool
	}{
		{
			name:     "detect VLLM from args",
			command:  []string{"/bin/sh", "-c"},
			args:     []string{"python -m dynamo.vllm.worker --model test"},
			expected: BackendFrameworkVLLM,
		},
		{
			name:     "detect SGLang from args",
			command:  []string{"/bin/sh", "-c"},
			args:     []string{"python -m dynamo.sglang --model test"},
			expected: BackendFrameworkSGLang,
		},
		{
			name:     "detect TRTLLM from args",
			command:  []string{"/bin/sh", "-c"},
			args:     []string{"python -m dynamo.trtllm.worker --model test"},
			expected: BackendFrameworkTRTLLM,
		},
		{
			name:     "detect from complex command with pipes",
			command:  []string{},
			args:     []string{"echo start && python -m dynamo.vllm.worker --model test | tee /tmp/log"},
			expected: BackendFrameworkVLLM,
		},
		{
			name:     "detect from python3.11",
			command:  []string{},
			args:     []string{"python3.11 -m dynamo.sglang"},
			expected: BackendFrameworkSGLang,
		},
		{
			name:     "no backend detected",
			command:  []string{"/bin/sh", "-c"},
			args:     []string{"echo hello world"},
			expected: BackendFrameworkNoop,
		},
		{
			name:        "multiple backends detected",
			command:     []string{},
			args:        []string{"python -m dynamo.vllm.worker && python -m dynamo.sglang"},
			expectError: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result, err := detectBackendFrameworkFromArgs(tt.command, tt.args)

			if tt.expectError {
				if err == nil {
					t.Errorf("detectBackendFrameworkFromArgs() expected error, got none")
				}
				return
			}

			if err != nil {
				t.Errorf("detectBackendFrameworkFromArgs() unexpected error: %v", err)
				return
			}

			if result != tt.expected {
				t.Errorf("detectBackendFrameworkFromArgs() = %v, want %v", result, tt.expected)
			}
		})
	}
}

func TestDetermineBackendFramework(t *testing.T) {
	tests := []struct {
		name                     string
		componentType            string
		command                  []string
		args                     []string
		explicitBackendFramework string
		expected                 BackendFramework
		expectError              bool
		errorContains            string
	}{
		{
			name:          "non-worker component returns noop",
			componentType: "frontend",
			command:       []string{"/bin/sh", "-c"},
			args:          []string{"echo hello world"},
			expected:      BackendFrameworkNoop,
		},
		{
			name:          "worker with VLLM detection",
			componentType: "worker",
			command:       []string{},
			args:          []string{"python -m dynamo.vllm.worker --model test"},
			expected:      BackendFrameworkVLLM,
		},
		{
			name:                     "worker with explicit framework only",
			componentType:            "worker",
			explicitBackendFramework: "sglang",
			expected:                 BackendFrameworkSGLang,
		},
		{
			name:                     "worker with detected matching explicit",
			componentType:            "worker",
			args:                     []string{"python -m dynamo.sglang"},
			explicitBackendFramework: "sglang",
			expected:                 BackendFrameworkSGLang,
		},
		{
			name:                     "worker with detected conflicting explicit",
			componentType:            "worker",
			args:                     []string{"python -m dynamo.vllm.worker"},
			explicitBackendFramework: "sglang",
			expectError:              true,
			errorContains:            "backend framework mismatch",
		},
		{
			name:          "worker with no detection, no explicit - returns noop",
			componentType: "worker",
			expected:      BackendFrameworkNoop,
			expectError:   false,
		},
		{
			name:          "worker with detection failure, no explicit - returns noop",
			componentType: "worker",
			args:          []string{"echo hello world"},
			expected:      BackendFrameworkNoop,
			expectError:   false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result, err := determineBackendFramework(
				tt.componentType,
				tt.command,
				tt.args,
				tt.explicitBackendFramework,
			)

			if tt.expectError {
				if err == nil {
					t.Errorf("determineBackendFramework() expected error, got none")
					return
				}
				if tt.errorContains != "" && !strings.Contains(err.Error(), tt.errorContains) {
					t.Errorf("determineBackendFramework() error = %v, should contain %q", err, tt.errorContains)
				}
				return
			}

			if err != nil {
				t.Errorf("determineBackendFramework() unexpected error: %v", err)
				return
			}

			if result != tt.expected {
				t.Errorf("determineBackendFramework() = %v, want %v", result, tt.expected)
			}
		})
	}
}

func TestGetBackendFrameworkFromComponent(t *testing.T) {
	tests := []struct {
		name          string
		component     *v1alpha1.DynamoComponentDeploymentSharedSpec
		deployment    *v1alpha1.DynamoGraphDeployment
		expected      BackendFramework
		expectError   bool
		errorContains string
	}{
		{
			name: "detect from args - VLLM",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: "worker", // Worker component
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python -m dynamo.vllm.worker --model test"},
					},
				},
			},
			deployment: &v1alpha1.DynamoGraphDeployment{},
			expected:   BackendFrameworkVLLM,
		},
		{
			name: "explicit framework only",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: "worker", // Worker component
			},
			deployment: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
				},
			},
			expected: BackendFrameworkSGLang,
		},
		{
			name: "detected matches explicit",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: "worker", // Worker component
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python -m dynamo.sglang"},
					},
				},
			},
			deployment: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
				},
			},
			expected: BackendFrameworkSGLang,
		},
		{
			name: "detected conflicts with explicit",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: "worker", // Worker component
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"python -m dynamo.vllm.worker"},
					},
				},
			},
			deployment: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
				},
			},
			expectError:   true,
			errorContains: "backend framework mismatch",
		},
		{
			name: "non-worker component returns noop",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: "frontend", // Frontend component
			},
			deployment: &v1alpha1.DynamoGraphDeployment{},
			expected:   BackendFrameworkNoop,
		},
		{
			name: "worker with no detection, no explicit - returns noop",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: "worker", // Worker component
			},
			deployment:  &v1alpha1.DynamoGraphDeployment{},
			expected:    BackendFrameworkNoop,
			expectError: false,
		},
		{
			name: "worker with detection failure, no explicit - returns noop",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: "worker", // Worker component
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Args: []string{"echo hello world"},
					},
				},
			},
			deployment:  &v1alpha1.DynamoGraphDeployment{},
			expected:    BackendFrameworkNoop,
			expectError: false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result, err := getBackendFrameworkFromComponent(tt.component, tt.deployment)

			if tt.expectError {
				if err == nil {
					t.Errorf("getBackendFrameworkFromComponent() expected error, got none")
					return
				}
				if tt.errorContains != "" && !strings.Contains(err.Error(), tt.errorContains) {
					t.Errorf("getBackendFrameworkFromComponent() error = %v, should contain %q", err, tt.errorContains)
				}
				return
			}

			if err != nil {
				t.Errorf("getBackendFrameworkFromComponent() unexpected error: %v", err)
				return
			}

			if result != tt.expected {
				t.Errorf("getBackendFrameworkFromComponent() = %v, want %v", result, tt.expected)
			}
		})
	}
}

func TestApplyCliqueStartupDependencies(t *testing.T) {
	tests := []struct {
		name              string
		roles             []ServiceRole
		backendFramework  BackendFramework
		numberOfNodes     int32
		expectedDeps      map[string][]string // clique name -> expected StartsAfter dependencies
		expectStartupType bool
	}{
		{
			name: "vllm_multinode_applies_dependencies",
			roles: []ServiceRole{
				{Name: "service-ldr", Role: RoleLeader, Replicas: 1},
				{Name: "service-wkr", Role: RoleWorker, Replicas: 2},
			},
			backendFramework: BackendFrameworkVLLM,
			numberOfNodes:    3,
			expectedDeps: map[string][]string{
				"service-ldr": nil,
				"service-wkr": nil,
			},
			expectStartupType: false,
		},
		{
			name: "sglang_multinode_applies_dependencies",
			roles: []ServiceRole{
				{Name: "service-ldr", Role: RoleLeader, Replicas: 1},
				{Name: "service-wkr", Role: RoleWorker, Replicas: 2},
			},
			backendFramework: BackendFrameworkSGLang,
			numberOfNodes:    3,
			expectedDeps: map[string][]string{
				"service-ldr": nil,
				"service-wkr": nil,
			},
			expectStartupType: false,
		},
		{
			name: "trtllm_multinode_applies_dependencies",
			roles: []ServiceRole{
				{Name: "service-ldr", Role: RoleLeader, Replicas: 1},
				{Name: "service-wkr", Role: RoleWorker, Replicas: 2},
			},
			backendFramework: BackendFrameworkTRTLLM,
			numberOfNodes:    3,
			expectedDeps: map[string][]string{
				"service-ldr": {"service-wkr"},
				"service-wkr": nil,
			},
			expectStartupType: true,
		},
		{
			name: "single_node_no_dependencies",
			roles: []ServiceRole{
				{Name: "service", Role: RoleMain, Replicas: 1},
			},
			backendFramework: BackendFrameworkTRTLLM,
			numberOfNodes:    1,
			expectedDeps: map[string][]string{
				"service": nil,
			},
			expectStartupType: false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			// Create a PodCliqueSet with cliques matching the roles
			gangSet := &grovev1alpha1.PodCliqueSet{
				Spec: grovev1alpha1.PodCliqueSetSpec{
					Template: grovev1alpha1.PodCliqueSetTemplateSpec{
						Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{},
					},
				},
			}

			// Add cliques for each role
			for _, role := range tt.roles {
				clique := &grovev1alpha1.PodCliqueTemplateSpec{
					Name: strings.ToLower(role.Name),
					Spec: grovev1alpha1.PodCliqueSpec{
						RoleName: strings.ToLower(role.Name),
						Replicas: role.Replicas,
					},
				}
				gangSet.Spec.Template.Cliques = append(gangSet.Spec.Template.Cliques, clique)
			}

			// Apply dependencies
			applyCliqueStartupDependencies(gangSet, tt.roles, tt.backendFramework, tt.numberOfNodes)

			// Verify StartupType
			if tt.expectStartupType {
				if gangSet.Spec.Template.StartupType == nil || *gangSet.Spec.Template.StartupType != grovev1alpha1.CliqueStartupTypeExplicit {
					t.Errorf("Expected StartupType to be CliqueStartupTypeExplicit, got %v", gangSet.Spec.Template.StartupType)
				}
			} else {
				if gangSet.Spec.Template.StartupType != nil {
					t.Errorf("Expected StartupType to be nil, got %v", *gangSet.Spec.Template.StartupType)
				}
			}

			// Verify dependencies for each clique
			for _, clique := range gangSet.Spec.Template.Cliques {
				expectedDeps, exists := tt.expectedDeps[clique.Name]
				if !exists {
					t.Errorf("Unexpected clique %s", clique.Name)
					continue
				}

				if !reflect.DeepEqual(clique.Spec.StartsAfter, expectedDeps) {
					t.Errorf("Clique %s: expected StartsAfter %v, got %v", clique.Name, expectedDeps, clique.Spec.StartsAfter)
				}
			}
		})
	}
}

func TestGetCliqueStartupDependencies(t *testing.T) {
	tests := []struct {
		name              string
		role              Role
		backendFramework  BackendFramework
		leaderCliqueName  string
		workerCliqueNames []string
		expected          []string
	}{
		{
			name:              "trtllm_leader_depends_on_workers",
			role:              RoleLeader,
			backendFramework:  BackendFrameworkTRTLLM,
			leaderCliqueName:  "service-ldr",
			workerCliqueNames: []string{"service-wkr1", "service-wkr2"},
			expected:          []string{"service-wkr1", "service-wkr2"},
		},
		{
			name:              "trtllm_worker_has_no_dependencies",
			role:              RoleWorker,
			backendFramework:  BackendFrameworkTRTLLM,
			leaderCliqueName:  "service-ldr",
			workerCliqueNames: []string{"service-wkr"},
			expected:          nil,
		},
		{
			name:              "leader_with_empty_worker_names",
			role:              RoleLeader,
			backendFramework:  BackendFrameworkTRTLLM,
			leaderCliqueName:  "service-ldr",
			workerCliqueNames: nil,
			expected:          nil,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := getCliqueStartupDependencies(
				tt.role,
				tt.backendFramework,
				tt.leaderCliqueName,
				tt.workerCliqueNames,
			)

			if !reflect.DeepEqual(result, tt.expected) {
				t.Errorf("getCliqueStartupDependencies() = %v, want %v", result, tt.expected)
			}
		})
	}
}

func TestGenerateGrovePodCliqueSet_StartsAfterDependencies(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}

	tests := []struct {
		name              string
		backendFramework  string
		expectedDeps      map[string][]string // clique name -> expected StartsAfter dependencies
		expectStartupType bool
	}{
		{
			name:             "vllm_worker_starts_after_leader",
			backendFramework: string(BackendFrameworkVLLM),
			expectedDeps: map[string][]string{
				"main-wkr": nil, // worker starts after leader
				"main-ldr": nil, // leader has no dependencies
			},
			expectStartupType: false,
		},
		{
			name:             "sglang_worker_starts_after_leader",
			backendFramework: string(BackendFrameworkSGLang),
			expectedDeps: map[string][]string{
				"main-wkr": nil, // worker starts after leader
				"main-ldr": nil, // leader has no dependencies
			},
			expectStartupType: false,
		},
		{
			name:             "trtllm_leader_starts_after_worker",
			backendFramework: string(BackendFrameworkTRTLLM),
			expectedDeps: map[string][]string{
				"main-ldr": {"main-wkr"}, // leader starts after worker
				"main-wkr": nil,          // worker has no dependencies
			},
			expectStartupType: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			dynamoDeployment := &v1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-deployment",
					Namespace: "default",
				},
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: tt.backendFramework,
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &v1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
							ComponentType: "worker", // Must be worker to trigger backend detection
							Replicas:      ptr.To(int32(1)),
							Resources: &v1alpha1.Resources{
								Requests: &v1alpha1.ResourceItem{
									GPU: "1", // 1 GPU per node
								},
							},
						},
					},
				},
			}

			controllerConfig := &configv1alpha1.OperatorConfiguration{
				Infrastructure: configv1alpha1.InfrastructureConfiguration{
					ETCDAddress: "etcd-av1alpha1",
					NATSAddress: "nats-address",
				},
			}

			got, err := GenerateGrovePodCliqueSet(context.Background(), dynamoDeployment, controllerConfig, &controller_common.RuntimeConfig{}, secretsRetriever, nil, nil, nil)
			if err != nil {
				t.Errorf("GenerateGrovePodCliqueSet() error = %v", err)
				return
			}

			// Verify that StartupType is set to Explicit
			if tt.expectStartupType {
				if got.Spec.Template.StartupType == nil || *got.Spec.Template.StartupType != grovev1alpha1.CliqueStartupTypeExplicit {
					t.Errorf("Expected StartupType to be CliqueStartupTypeExplicit, got %v", got.Spec.Template.StartupType)
				}
			} else {
				if got.Spec.Template.StartupType == nil || *got.Spec.Template.StartupType != grovev1alpha1.CliqueStartupTypeAnyOrder {
					t.Errorf("Expected StartupType to be CliqueStartupTypeAnyOrder, got %v", got.Spec.Template.StartupType)
				}
			}

			// Verify StartsAfter dependencies for each clique
			cliqueMap := make(map[string]*grovev1alpha1.PodCliqueTemplateSpec)
			for _, clique := range got.Spec.Template.Cliques {
				cliqueMap[clique.Name] = clique
			}

			for cliqueName, expectedDeps := range tt.expectedDeps {
				clique, exists := cliqueMap[cliqueName]
				if !exists {
					t.Errorf("Expected clique %s not found", cliqueName)
					continue
				}

				if expectedDeps == nil {
					if len(clique.Spec.StartsAfter) != 0 {
						t.Errorf("Clique %s should have no StartsAfter dependencies, but has %v", cliqueName, clique.Spec.StartsAfter)
					}
				} else {
					if len(clique.Spec.StartsAfter) != len(expectedDeps) {
						t.Errorf("Clique %s expected %d StartsAfter dependencies, got %d", cliqueName, len(expectedDeps), len(clique.Spec.StartsAfter))
						continue
					}

					for i, expectedDep := range expectedDeps {
						if i >= len(clique.Spec.StartsAfter) || clique.Spec.StartsAfter[i] != expectedDep {
							t.Errorf("Clique %s expected StartsAfter[%d] = %s, got %v", cliqueName, i, expectedDep, clique.Spec.StartsAfter)
						}
					}
				}
			}
		})
	}
}

func TestGenerateBasePodSpec_Frontend(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}
	dynamoDeployment := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-deployment",
			Namespace: "default",
		},
	}

	tests := []struct {
		name             string
		component        *v1alpha1.DynamoComponentDeploymentSharedSpec
		backendFramework BackendFramework
		wantEnvVars      map[string]string
		wantErr          bool
	}{
		{
			name: "frontend with default command",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
			},
			backendFramework: BackendFrameworkVLLM,
			wantEnvVars: map[string]string{
				"DYN_HTTP_PORT": fmt.Sprintf("%d", commonconsts.DynamoServicePort),
			},
		},
		{
			name: "frontend with overriding env var",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				Envs: []corev1.EnvVar{
					{
						Name:  "DYN_HTTP_PORT",
						Value: "3000",
					},
				},
			},
			backendFramework: BackendFrameworkVLLM,
			wantEnvVars: map[string]string{
				"DYN_HTTP_PORT": "3000",
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				tt.backendFramework,
				secretsRetriever,
				dynamoDeployment.Name,
				dynamoDeployment.Namespace,
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if (err != nil) != tt.wantErr {
				t.Errorf("GenerateBasePodSpec() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			if tt.wantErr {
				return
			}

			// Check command and args
			wantCommand := []string{"python3"}
			wantArgs := []string{"-m", "dynamo.frontend"}
			if !reflect.DeepEqual(podSpec.Containers[0].Command, wantCommand) {
				t.Errorf("GenerateBasePodSpec() command = %v, want %v",
					podSpec.Containers[0].Command, wantCommand)
			}
			if !reflect.DeepEqual(podSpec.Containers[0].Args, wantArgs) {
				t.Errorf("GenerateBasePodSpec() args = %v, want %v",
					podSpec.Containers[0].Args, wantArgs)
			}

			// Check environment variables
			envVars := make(map[string]string)
			for _, env := range podSpec.Containers[0].Env {
				envVars[env.Name] = env.Value
			}
			for k, v := range tt.wantEnvVars {
				if envVars[k] != v {
					t.Errorf("GenerateBasePodSpec() env var %s = %v, want %v",
						k, envVars[k], v)
				}
			}
		})
	}
}

func TestGenerateBasePodSpec_PlannerServiceAccount(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name               string
		component          *v1alpha1.DynamoComponentDeploymentSharedSpec
		expectedServiceAcc string
	}{
		{
			name: "Planner component should have planner service account",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypePlanner,
			},
			expectedServiceAcc: commonconsts.PlannerServiceAccountName,
		},
		{
			name: "Planner service account should not be set for non-planner components",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
			},
			expectedServiceAcc: "",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				BackendFrameworkSGLang,
				secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if err != nil {
				t.Errorf("GenerateBasePodSpec() error = %v", err)
				return
			}

			if podSpec.ServiceAccountName != tt.expectedServiceAcc {
				t.Errorf("GenerateBasePodSpec() serviceAccountName = %v, want %v",
					podSpec.ServiceAccountName, tt.expectedServiceAcc)
			}
		})
	}
}

func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
	tests := []struct {
		name                     string
		component                *v1alpha1.DynamoComponentDeploymentSharedSpec
		secretsRetriever         SecretsRetriever
		expectedImagePullSecrets []corev1.LocalObjectReference
	}{
		{
			name: "disable docker secrets annotation set to true",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				Annotations: map[string]string{
					commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueTrue,
				},
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Image: "test-registry/test-image:latest",
					},
				},
			},
			secretsRetriever:         &mockSecretsRetrieverWithSecrets{},
			expectedImagePullSecrets: nil, // Should be nil when disabled
		},
		{
			name: "disable docker secrets annotation set to false",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				Annotations: map[string]string{
					commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueFalse,
				},
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Image: "test-registry/test-image:latest",
					},
				},
			},
			secretsRetriever: &mockSecretsRetrieverWithSecrets{},
			expectedImagePullSecrets: []corev1.LocalObjectReference{
				{Name: "test-docker-secret"},
			}, // Should be present when enabled
		},
		{
			name: "disable docker secrets annotation not set (default behavior)",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Image: "test-registry/test-image:latest",
					},
				},
			},
			secretsRetriever: &mockSecretsRetrieverWithSecrets{},
			expectedImagePullSecrets: []corev1.LocalObjectReference{
				{Name: "test-docker-secret"},
			}, // Should be present by default
		},
		{
			name: "disable docker secrets annotation set to invalid value",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				Annotations: map[string]string{
					commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: "invalid",
				},
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Image: "test-registry/test-image:latest",
					},
				},
			},
			secretsRetriever: &mockSecretsRetrieverWithSecrets{},
			expectedImagePullSecrets: []corev1.LocalObjectReference{
				{Name: "test-docker-secret"},
			}, // Should be present when annotation is not "true"
		},
		{
			name: "disable docker secrets but no secrets retriever",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				Annotations: map[string]string{
					commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueFalse,
				},
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Image: "test-registry/test-image:latest",
					},
				},
			},
			secretsRetriever:         nil,
			expectedImagePullSecrets: nil, // Should be nil when no retriever
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			controllerConfig := &configv1alpha1.OperatorConfiguration{}

			podSpec, err := GenerateBasePodSpec(
				tt.component,
				BackendFrameworkNoop,
				tt.secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if err != nil {
				t.Errorf("GenerateBasePodSpec() error = %v", err)
				return
			}

			if !reflect.DeepEqual(podSpec.ImagePullSecrets, tt.expectedImagePullSecrets) {
				t.Errorf("GenerateBasePodSpec() ImagePullSecrets = %v, want %v",
					podSpec.ImagePullSecrets, tt.expectedImagePullSecrets)
			}
		})
	}
}

func TestGenerateBasePodSpec_DiscoverBackend(t *testing.T) {
	tests := []struct {
		name             string
		component        *v1alpha1.DynamoComponentDeploymentSharedSpec
		controllerConfig *configv1alpha1.OperatorConfiguration
		wantEnvVar       string
	}{
		{
			name: "Kubernetes discovery backend should set env var to kubernetes",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				Annotations: map[string]string{
					commonconsts.KubeAnnotationDynamoDiscoveryBackend: "kubernetes",
				},
			},
			controllerConfig: &configv1alpha1.OperatorConfiguration{},
			wantEnvVar:       "kubernetes",
		},
		{
			name: "Kubernetes discovery from controller config should set env var to kubernetes",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				Annotations: map[string]string{},
			},
			controllerConfig: &configv1alpha1.OperatorConfiguration{
				Discovery: configv1alpha1.DiscoveryConfiguration{
					Backend: "kubernetes",
				},
			},
			wantEnvVar: "kubernetes",
		},
		{
			name: "Etcd discovery backend annotation should not set env var",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				Annotations: map[string]string{
					commonconsts.KubeAnnotationDynamoDiscoveryBackend: "etcd",
				},
			},
			controllerConfig: &configv1alpha1.OperatorConfiguration{
				Discovery: configv1alpha1.DiscoveryConfiguration{
					Backend: "kubernetes",
				},
			},
			wantEnvVar: "", // etcd is the runtime default, no env var needed
		},
		{
			name: "Etcd discovery from controller config should not set env var",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				Annotations: map[string]string{},
			},
			controllerConfig: &configv1alpha1.OperatorConfiguration{
				Discovery: configv1alpha1.DiscoveryConfiguration{
					Backend: "etcd",
				},
			},
			wantEnvVar: "", // etcd is the runtime default, no env var needed
		},
		{
			name: "Empty discovery backend defaults to kubernetes",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				Annotations: map[string]string{
					commonconsts.KubeAnnotationDynamoDiscoveryBackend: "",
				},
			},
			controllerConfig: &configv1alpha1.OperatorConfiguration{
				Discovery: configv1alpha1.DiscoveryConfiguration{
					Backend: "",
				},
			},
			wantEnvVar: "kubernetes", // empty defaults to kubernetes
		},
		{
			name:             "Discovery backend not set defaults to kubernetes",
			component:        &v1alpha1.DynamoComponentDeploymentSharedSpec{},
			controllerConfig: &configv1alpha1.OperatorConfiguration{},
			wantEnvVar:       "kubernetes", // not set defaults to kubernetes
		},
	}
	secretsRetriever := &mockSecretsRetriever{}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				BackendFrameworkSGLang,
				secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				tt.controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)
			if !assert.NoError(t, err) {
				return
			}
			if tt.wantEnvVar != "" {
				assert.Contains(t, podSpec.Containers[0].Env, corev1.EnvVar{Name: commonconsts.DynamoDiscoveryBackendEnvVar, Value: tt.wantEnvVar})
			} else {
				for _, env := range podSpec.Containers[0].Env {
					if env.Name == commonconsts.DynamoDiscoveryBackendEnvVar {
						t.Errorf("GenerateBasePodSpec() Discover backend env var should not be set, got %s", env.Value)
					}
				}
			}
		})
	}
}

func TestGenerateBasePodSpec_Worker(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name            string
		component       *v1alpha1.DynamoComponentDeploymentSharedSpec
		expectedPodSpec *corev1.PodSpec
	}{
		{
			name: "Worker component with DynamoNamespace set",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				Envs: []corev1.EnvVar{
					{Name: "ANOTHER_COMPONENTENV", Value: "true"},
				},
				ComponentType:   commonconsts.ComponentTypeWorker,
				DynamoNamespace: ptr.To("default-test-deployment"), // Namespace set by caller
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					MainContainer: &corev1.Container{
						Command: []string{"python3"},
						Args:    []string{"-m", "dynamo.worker"},
						Env: []corev1.EnvVar{
							{Name: "ANOTHER_CONTAINER_ENV", Value: "true"},
						},
					},
				},
			},
			expectedPodSpec: &corev1.PodSpec{
				Containers: []corev1.Container{
					{
						Name:    commonconsts.MainContainerName,
						Command: []string{"python3"},
						Args:    []string{"-m", "dynamo.worker"},
						Env: []corev1.EnvVar{
							{Name: "ANOTHER_COMPONENTENV", Value: "true"},
							{Name: "ANOTHER_CONTAINER_ENV", Value: "true"},
							{Name: commonconsts.DynamoComponentEnvVar, Value: "worker"},
							{Name: commonconsts.DynamoDiscoveryBackendEnvVar, Value: "kubernetes"},
							{Name: "DYN_HEALTH_CHECK_ENABLED", Value: "false"},
							{Name: commonconsts.DynamoNamespaceEnvVar, Value: "default-test-deployment"},
							{Name: "DYN_PARENT_DGD_K8S_NAME", Value: "test-deployment"},
							{Name: "DYN_PARENT_DGD_K8S_NAMESPACE", Value: "default"},
							{Name: "DYN_SYSTEM_ENABLED", Value: "true"},
							{Name: "DYN_SYSTEM_PORT", Value: "9090"},
							{Name: "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS", Value: "[\"generate\"]"},
							{Name: "NIXL_TELEMETRY_ENABLE", Value: "n"},
							{Name: "NIXL_TELEMETRY_EXPORTER", Value: "prometheus"},
							{Name: "NIXL_TELEMETRY_PROMETHEUS_PORT", Value: "19090"},
							{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{
								FieldRef: &corev1.ObjectFieldSelector{
									FieldPath: "metadata.name",
								},
							}},
							{Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{
								FieldRef: &corev1.ObjectFieldSelector{
									FieldPath: "metadata.namespace",
								},
							}},
							{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{
								FieldRef: &corev1.ObjectFieldSelector{
									FieldPath: "metadata.uid",
								},
							}},
						},
						VolumeMounts: []corev1.VolumeMount{
							{
								Name:      "shared-memory",
								MountPath: "/dev/shm",
							},
						},
						LivenessProbe: &corev1.Probe{
							ProbeHandler: corev1.ProbeHandler{
								HTTPGet: &corev1.HTTPGetAction{
									Path: "/live",
									Port: intstr.FromString(commonconsts.DynamoSystemPortName),
								},
							},
							PeriodSeconds:    5,
							TimeoutSeconds:   4,
							FailureThreshold: 1,
						},
						ReadinessProbe: &corev1.Probe{
							ProbeHandler: corev1.ProbeHandler{
								HTTPGet: &corev1.HTTPGetAction{
									Path: "/health",
									Port: intstr.FromString(commonconsts.DynamoSystemPortName),
								},
							},
							PeriodSeconds:    10,
							TimeoutSeconds:   4,
							FailureThreshold: 3,
						},
						StartupProbe: &corev1.Probe{
							ProbeHandler: corev1.ProbeHandler{
								HTTPGet: &corev1.HTTPGetAction{
									Path: "/live",
									Port: intstr.FromString(commonconsts.DynamoSystemPortName),
								},
							},
							PeriodSeconds:    10,
							TimeoutSeconds:   5,
							FailureThreshold: 720,
						},
						Ports: []corev1.ContainerPort{
							{
								Name:          commonconsts.DynamoSystemPortName,
								ContainerPort: int32(commonconsts.DynamoSystemPort),
								Protocol:      corev1.ProtocolTCP,
							},
							{
								Name:          commonconsts.DynamoNixlPortName,
								ContainerPort: int32(commonconsts.DynamoNixlPort),
								Protocol:      corev1.ProtocolTCP,
							},
						},
					},
				},
				RestartPolicy:                 corev1.RestartPolicyAlways,
				TerminationGracePeriodSeconds: ptr.To(int64(60)),
				SecurityContext: &corev1.PodSecurityContext{
					// Only fsGroup is injected by default for volume permissions
					FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
				},
				Volumes: []corev1.Volume{
					{
						Name: "shared-memory",
						VolumeSource: corev1.VolumeSource{
							EmptyDir: &corev1.EmptyDirVolumeSource{
								Medium:    corev1.StorageMediumMemory,
								SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
							},
						},
					},
				},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				BackendFrameworkSGLang,
				secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if err != nil {
				t.Errorf("GenerateBasePodSpec() error = %v", err)
				return
			}

			diff := cmp.Diff(tt.expectedPodSpec, podSpec)
			if diff != "" {
				t.Errorf("GenerateBasePodSpec() podSpec = %v, want %v, diff = %v", podSpec, tt.expectedPodSpec, diff)
			}
		})
	}
}

func TestGenerateBasePodSpec_VolumeMounts(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name           string
		component      *v1alpha1.DynamoComponentDeploymentSharedSpec
		expectError    bool
		expectedPVCs   []string
		expectedMounts []corev1.VolumeMount
	}{
		{
			name: "valid volumeMounts",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:       "test-pvc",
						MountPoint: "/data",
					},
				},
			},
			expectError:  false,
			expectedPVCs: []string{"test-pvc"},
			expectedMounts: []corev1.VolumeMount{
				{Name: "test-pvc", MountPath: "/data"},
				{Name: "shared-memory", MountPath: "/dev/shm"},
			},
		},
		{
			name: "multiple volumeMounts",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{Name: "pvc1", MountPoint: "/data1"},
					{Name: "pvc2", MountPoint: "/data2"},
				},
			},
			expectError:  false,
			expectedPVCs: []string{"pvc1", "pvc2"},
			expectedMounts: []corev1.VolumeMount{
				{Name: "pvc1", MountPath: "/data1"},
				{Name: "pvc2", MountPath: "/data2"},
				{Name: "shared-memory", MountPath: "/dev/shm"},
			},
		},
		{
			name: "empty volumeMount name",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{Name: "", MountPoint: "/data"},
				},
			},
			expectError: true,
		},
		{
			name: "empty volumeMount mountPoint",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{Name: "test-pvc", MountPoint: ""},
				},
			},
			expectError: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				BackendFrameworkVLLM,
				secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if tt.expectError {
				if err == nil {
					t.Errorf("GenerateBasePodSpec() expected error, got nil")
				}
				return
			}

			if err != nil {
				t.Errorf("GenerateBasePodSpec() unexpected error: %v", err)
				return
			}

			// Check expected PVCs are present in volumes
			for _, expectedPVC := range tt.expectedPVCs {
				found := false
				for _, volume := range podSpec.Volumes {
					if volume.Name == expectedPVC && volume.PersistentVolumeClaim != nil {
						if volume.PersistentVolumeClaim.ClaimName == expectedPVC {
							found = true
							break
						}
					}
				}
				if !found {
					t.Errorf("GenerateBasePodSpec() expected PVC volume %s not found", expectedPVC)
				}
			}

			// Check expected mounts are present
			if len(podSpec.Containers) == 0 {
				t.Errorf("GenerateBasePodSpec() no containers found")
				return
			}

			container := podSpec.Containers[0]
			for _, expectedMount := range tt.expectedMounts {
				found := false
				for _, mount := range container.VolumeMounts {
					if mount.Name == expectedMount.Name && mount.MountPath == expectedMount.MountPath {
						found = true
						break
					}
				}
				if !found {
					t.Errorf("GenerateBasePodSpec() expected volume mount %+v not found", expectedMount)
				}
			}
		})
	}
}

func TestGenerateBasePodSpec_ResourceClaims(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name                   string
		component              *v1alpha1.DynamoComponentDeploymentSharedSpec
		expectError            bool
		expectedResourceClaims []corev1.ResourceClaim
		expectedPodClaims      []corev1.PodResourceClaim
		expectedVolumes        []corev1.Volume
	}{
		{
			name: "component with resource claims",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				Resources: &v1alpha1.Resources{
					Requests: &v1alpha1.ResourceItem{
						CPU:    "130",
						Memory: "800Gi",
					},
					Limits: &v1alpha1.ResourceItem{
						CPU:    "130",
						Memory: "800Gi",
						GPU:    "4",
					},
					Claims: []corev1.ResourceClaim{
						{
							Name: "compute-domain-channel",
						},
					},
				},
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					PodSpec: &corev1.PodSpec{
						ResourceClaims: []corev1.PodResourceClaim{
							{
								Name:                      "compute-domain-channel",
								ResourceClaimTemplateName: ptr.To("trtllm-test-compute-domain-channel"),
							},
						},
						Volumes: []corev1.Volume{
							{
								Name: "model-storage",
								VolumeSource: corev1.VolumeSource{
									PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
										ClaimName: "dynamo-pvc",
									},
								},
							},
						},
					},
					MainContainer: &corev1.Container{
						Image: "rohanv672/dynamo:v0.5.1-trtllm",
						Args: []string{
							"python3 -m dynamo.trtllm --model-path /data/deepseek-r1 --served-model-name deepseek-ai/DeepSeek-R1 --extra-engine-args /data/engine_configs/wide_ep_agg.yaml",
						},
						Command: []string{"/bin/sh", "-c"},
						VolumeMounts: []corev1.VolumeMount{
							{
								Name:      "model-storage",
								MountPath: "/data",
							},
						},
					},
				},
			},
			expectError: false,
			expectedResourceClaims: []corev1.ResourceClaim{
				{
					Name: "compute-domain-channel",
				},
			},
			expectedPodClaims: []corev1.PodResourceClaim{
				{
					Name:                      "compute-domain-channel",
					ResourceClaimTemplateName: ptr.To("trtllm-test-compute-domain-channel"),
				},
			},
			expectedVolumes: []corev1.Volume{
				{
					Name: "model-storage",
					VolumeSource: corev1.VolumeSource{
						PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
							ClaimName: "dynamo-pvc",
						},
					},
				},
				{
					Name: "shared-memory",
					VolumeSource: corev1.VolumeSource{
						EmptyDir: &corev1.EmptyDirVolumeSource{
							Medium:    corev1.StorageMediumMemory,
							SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
						},
					},
				},
			},
		},
		{
			name: "component with multiple resource claims",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeWorker,
				Resources: &v1alpha1.Resources{
					Claims: []corev1.ResourceClaim{
						{
							Name: "compute-domain-channel",
						},
						{
							Name: "network-domain-channel",
						},
					},
				},
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					PodSpec: &corev1.PodSpec{
						ResourceClaims: []corev1.PodResourceClaim{
							{
								Name:                      "compute-domain-channel",
								ResourceClaimTemplateName: ptr.To("compute-template"),
							},
							{
								Name:                      "network-domain-channel",
								ResourceClaimTemplateName: ptr.To("network-template"),
							},
						},
					},
					MainContainer: &corev1.Container{
						Image:   "test-image",
						Command: []string{"python3"},
						Args:    []string{"-m", "dynamo.worker"},
					},
				},
			},
			expectError: false,
			expectedResourceClaims: []corev1.ResourceClaim{
				{
					Name: "compute-domain-channel",
				},
				{
					Name: "network-domain-channel",
				},
			},
			expectedPodClaims: []corev1.PodResourceClaim{
				{
					Name:                      "compute-domain-channel",
					ResourceClaimTemplateName: ptr.To("compute-template"),
				},
				{
					Name:                      "network-domain-channel",
					ResourceClaimTemplateName: ptr.To("network-template"),
				},
			},
		},
		{
			name: "component without resource claims",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				Resources: &v1alpha1.Resources{
					Requests: &v1alpha1.ResourceItem{
						CPU:    "1",
						Memory: "1Gi",
					},
				},
			},
			expectError:            false,
			expectedResourceClaims: nil,
			expectedPodClaims:      nil,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				BackendFrameworkTRTLLM,
				secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if tt.expectError {
				if err == nil {
					t.Errorf("GenerateBasePodSpec() expected error, got nil")
				}
				return
			}

			if err != nil {
				t.Errorf("GenerateBasePodSpec() unexpected error: %v", err)
				return
			}

			// Check containers exist
			if len(podSpec.Containers) == 0 {
				t.Errorf("GenerateBasePodSpec() no containers found")
				return
			}

			container := podSpec.Containers[0]

			// Check resource claims in container resources using reflect.DeepEqual
			if !reflect.DeepEqual(container.Resources.Claims, tt.expectedResourceClaims) {
				t.Errorf("GenerateBasePodSpec() resource claims mismatch:\ngot:  %+v\nwant: %+v",
					container.Resources.Claims, tt.expectedResourceClaims)
			}

			// Check pod resource claims using reflect.DeepEqual
			if !reflect.DeepEqual(podSpec.ResourceClaims, tt.expectedPodClaims) {
				t.Errorf("GenerateBasePodSpec() pod resource claims mismatch:\ngot:  %+v\nwant: %+v",
					podSpec.ResourceClaims, tt.expectedPodClaims)
			}

			// Check expected volumes if specified using reflect.DeepEqual
			if tt.expectedVolumes != nil {
				if !reflect.DeepEqual(podSpec.Volumes, tt.expectedVolumes) {
					t.Errorf("GenerateBasePodSpec() volumes mismatch:\ngot:  %+v\nwant: %+v",
						podSpec.Volumes, tt.expectedVolumes)
				}
			}

			// Verify resource requests and limits are properly set when claims are present
			if len(tt.expectedResourceClaims) > 0 {
				// Check that standard resources are still processed correctly
				if tt.component.Resources != nil {
					if tt.component.Resources.Requests != nil {
						if tt.component.Resources.Requests.CPU != "" {
							if container.Resources.Requests == nil {
								t.Errorf("GenerateBasePodSpec() expected CPU request to be set")
							} else if cpu, exists := container.Resources.Requests[corev1.ResourceCPU]; !exists || cpu.IsZero() {
								t.Errorf("GenerateBasePodSpec() expected CPU request to be set")
							}
						}
						if tt.component.Resources.Requests.Memory != "" {
							if container.Resources.Requests == nil {
								t.Errorf("GenerateBasePodSpec() expected Memory request to be set")
							} else if memory, exists := container.Resources.Requests[corev1.ResourceMemory]; !exists || memory.IsZero() {
								t.Errorf("GenerateBasePodSpec() expected Memory request to be set")
							}
						}
					}
					if tt.component.Resources.Limits != nil {
						if tt.component.Resources.Limits.GPU != "" {
							if container.Resources.Limits == nil {
								t.Errorf("GenerateBasePodSpec() expected GPU limit to be set")
							} else if gpu, exists := container.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")]; !exists || gpu.IsZero() {
								t.Errorf("GenerateBasePodSpec() expected GPU limit to be set")
							}
						}
					}
				}
			}
		})
	}
}

func TestGenerateBasePodSpec_UseAsCompilationCache_BackendSupport(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name             string
		component        *v1alpha1.DynamoComponentDeploymentSharedSpec
		backendFramework BackendFramework
		expectError      bool
		expectedMount    *corev1.VolumeMount
	}{
		{
			name: "useAsCompilationCache with custom mount point",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:                  "cache-pvc",
						MountPoint:            "/custom/cache",
						UseAsCompilationCache: true,
					},
				},
			},
			backendFramework: BackendFrameworkVLLM,
			expectError:      false,
			expectedMount:    &corev1.VolumeMount{Name: "cache-pvc", MountPath: "/custom/cache"},
		},
		{
			name: "useAsCompilationCache with default mount point for VLLM",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:                  "cache-pvc",
						UseAsCompilationCache: true,
					},
				},
			},
			backendFramework: BackendFrameworkVLLM,
			expectError:      false,
			expectedMount:    &corev1.VolumeMount{Name: "cache-pvc", MountPath: commonconsts.DefaultVLLMCacheMountPoint},
		},
		{
			name: "useAsCompilationCache without mount point for SGLang - should error",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:                  "cache-pvc",
						UseAsCompilationCache: true,
					},
				},
			},
			backendFramework: BackendFrameworkSGLang,
			expectError:      true, // SGLang doesn't support compilation cache, requires explicit mount point
			expectedMount:    nil,
		},
		{
			name: "useAsCompilationCache with explicit mount point for SGLang - should work",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:                  "cache-pvc",
						MountPoint:            "/custom/sglang/cache",
						UseAsCompilationCache: true,
					},
				},
			},
			backendFramework: BackendFrameworkSGLang,
			expectError:      false,
			expectedMount:    &corev1.VolumeMount{Name: "cache-pvc", MountPath: "/custom/sglang/cache"},
		},
		{
			name: "useAsCompilationCache without mount point for TensorRT-LLM - should error",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:                  "cache-pvc",
						UseAsCompilationCache: true,
					},
				},
			},
			backendFramework: BackendFrameworkTRTLLM,
			expectError:      true, // TensorRT-LLM doesn't support compilation cache, requires explicit mount point
			expectedMount:    nil,
		},
		{
			name: "useAsCompilationCache with explicit mount point for TensorRT-LLM - should work",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:                  "cache-pvc",
						MountPoint:            "/custom/trtllm/cache",
						UseAsCompilationCache: true,
					},
				},
			},
			backendFramework: BackendFrameworkTRTLLM,
			expectError:      false,
			expectedMount:    &corev1.VolumeMount{Name: "cache-pvc", MountPath: "/custom/trtllm/cache"},
		},
		{
			name: "no useAsCompilationCache volumes - should be ignored",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				VolumeMounts: []v1alpha1.VolumeMount{
					{
						Name:       "regular-pvc",
						MountPoint: "/data",
					},
				},
			},
			backendFramework: BackendFrameworkVLLM,
			expectError:      false,
			expectedMount:    nil, // Should be ignored, not error
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				tt.backendFramework,
				secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if tt.expectError {
				if err == nil {
					t.Errorf("GenerateBasePodSpec() expected error, got nil")
				}
				return
			}

			if err != nil {
				t.Errorf("GenerateBasePodSpec() unexpected error: %v", err)
				return
			}

			if tt.expectedMount != nil {
				// Check PVC volume exists
				found := false
				for _, volume := range podSpec.Volumes {
					if volume.Name == tt.expectedMount.Name && volume.PersistentVolumeClaim != nil {
						if volume.PersistentVolumeClaim.ClaimName == tt.expectedMount.Name {
							found = true
							break
						}
					}
				}
				if !found {
					t.Errorf("GenerateBasePodSpec() expected PVC volume %s not found", tt.expectedMount.Name)
				}

				// Check volume mount exists
				if len(podSpec.Containers) == 0 {
					t.Errorf("GenerateBasePodSpec() no containers found")
					return
				}

				container := podSpec.Containers[0]
				found = false
				for _, mount := range container.VolumeMounts {
					if mount.Name == tt.expectedMount.Name && mount.MountPath == tt.expectedMount.MountPath {
						found = true
						break
					}
				}
				if !found {
					t.Errorf("GenerateBasePodSpec() expected volume mount %+v not found", tt.expectedMount)
				}
			}
		})
	}
}

func TestGenerateBasePodSpec_SecurityContext(t *testing.T) {
	secretsRetriever := &mockSecretsRetriever{}
	controllerConfig := &configv1alpha1.OperatorConfiguration{}

	tests := []struct {
		name                    string
		component               *v1alpha1.DynamoComponentDeploymentSharedSpec
		expectedSecurityContext *corev1.PodSecurityContext
		description             string
	}{
		{
			name: "no security context provided - should apply fsGroup default only",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
			},
			expectedSecurityContext: &corev1.PodSecurityContext{
				FSGroup: ptr.To(int64(commonconsts.DefaultSecurityContextFSGroup)),
			},
			description: "Operator should only inject fsGroup for volume permissions, not UID/GID (backward compatible)",
		},
		{
			name: "full security context override - should use user values",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					PodSpec: &corev1.PodSpec{
						SecurityContext: &corev1.PodSecurityContext{
							RunAsNonRoot: ptr.To(true),
							RunAsUser:    ptr.To(int64(5000)),
							RunAsGroup:   ptr.To(int64(5000)),
							FSGroup:      ptr.To(int64(5000)),
						},
					},
				},
			},
			expectedSecurityContext: &corev1.PodSecurityContext{
				RunAsNonRoot: ptr.To(true),
				RunAsUser:    ptr.To(int64(5000)),
				RunAsGroup:   ptr.To(int64(5000)),
				FSGroup:      ptr.To(int64(5000)),
			},
			description: "User-provided security context should completely override defaults",
		},
		{
			name: "partial security context override - user gets full control",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					PodSpec: &corev1.PodSpec{
						SecurityContext: &corev1.PodSecurityContext{
							RunAsUser:  ptr.To(int64(2000)),
							RunAsGroup: ptr.To(int64(3000)),
						},
					},
				},
			},
			expectedSecurityContext: &corev1.PodSecurityContext{
				RunAsUser:  ptr.To(int64(2000)),
				RunAsGroup: ptr.To(int64(3000)),
			},
			description: "Partial user override gets full control - no defaults injected",
		},
		{
			name: "only fsGroup override - user gets full control",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					PodSpec: &corev1.PodSpec{
						SecurityContext: &corev1.PodSecurityContext{
							FSGroup: ptr.To(int64(7000)),
						},
					},
				},
			},
			expectedSecurityContext: &corev1.PodSecurityContext{
				FSGroup: ptr.To(int64(7000)),
			},
			description: "Only fsGroup override - user gets full control, no defaults injected",
		},
		{
			name: "fsGroup 2000 example - exactly what user requested",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					PodSpec: &corev1.PodSpec{
						SecurityContext: &corev1.PodSecurityContext{
							FSGroup: ptr.To(int64(2000)),
						},
					},
				},
			},
			expectedSecurityContext: &corev1.PodSecurityContext{
				FSGroup: ptr.To(int64(2000)),
			},
			description: "User sets fsGroup:2000, gets ONLY that - critical for allowing root users",
		},
		{
			name: "OpenShift-style namespace range - should use user values",
			component: &v1alpha1.DynamoComponentDeploymentSharedSpec{
				ComponentType: commonconsts.ComponentTypeFrontend,
				ExtraPodSpec: &v1alpha1.ExtraPodSpec{
					PodSpec: &corev1.PodSpec{
						SecurityContext: &corev1.PodSecurityContext{
							RunAsNonRoot: ptr.To(true),
							RunAsUser:    ptr.To(int64(1000700001)),
							RunAsGroup:   ptr.To(int64(1000700001)),
							FSGroup:      ptr.To(int64(1000700001)),
						},
					},
				},
			},
			expectedSecurityContext: &corev1.PodSecurityContext{
				RunAsNonRoot: ptr.To(true),
				RunAsUser:    ptr.To(int64(1000700001)),
				RunAsGroup:   ptr.To(int64(1000700001)),
				FSGroup:      ptr.To(int64(1000700001)),
			},
			description: "OpenShift namespace UID/GID ranges should be respected",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			podSpec, err := GenerateBasePodSpec(
				tt.component,
				BackendFrameworkNoop,
				secretsRetriever,
				"test-deployment",
				"default",
				RoleMain,
				1,
				controllerConfig,
				commonconsts.MultinodeDeploymentTypeGrove,
				"test-service",
				nil, // No checkpoint info in tests
			)

			if err != nil {
				t.Errorf("GenerateBasePodSpec() unexpected error: %v", err)
				return
			}

			// Compare the entire SecurityContext using cmp.Diff
			if diff := cmp.Diff(tt.expectedSecurityContext, podSpec.SecurityContext); diff != "" {
				t.Errorf("GenerateBasePodSpec() SecurityContext mismatch (-want +got):\n%s\nDescription: %s", diff, tt.description)
			}
		})
	}
}

func TestDetermineGroveRestartState(t *testing.T) {
	restartID := "restart-1"
	oldRestartID := "restart-0"

	tests := []struct {
		name          string
		dgd           *v1alpha1.DynamoGraphDeployment
		restartStatus *v1alpha1.RestartStatus
		want          *RestartState
		wantNil       bool
		wantSvcs      []string // expected services to annotate (sorted)
		wantTimestamp *string
	}{
		{
			name: "restartStatus nil returns nil",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
				},
			},
			wantNil: true,
		},
		{
			name: "spec.restart.at nil and restartStatus.observedAt nil returns nil",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
					Restart: &v1alpha1.Restart{},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: "",
			},
			wantNil: true,
		},
		{
			name: "new parallel restart annotates all services",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
					Restart: &v1alpha1.Restart{
						ID: restartID,
						Strategy: &v1alpha1.RestartStrategy{
							Type: v1alpha1.RestartStrategyTypeParallel,
						},
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: restartID,
				Phase:      v1alpha1.RestartPhaseRestarting,
				InProgress: []string{"Frontend", "Worker"},
			},
			wantSvcs:      []string{"Frontend", "Worker"},
			wantTimestamp: ptr.To(restartID),
		},
		{
			name: "new sequential restart annotates only first service",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
					Restart: &v1alpha1.Restart{
						ID: restartID,
						Strategy: &v1alpha1.RestartStrategy{
							Type:  v1alpha1.RestartStrategyTypeSequential,
							Order: []string{"Worker", "Frontend"},
						},
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: restartID,
				Phase:      v1alpha1.RestartPhaseRestarting,
				InProgress: []string{"Worker"},
			},
			wantSvcs:      []string{"Worker"},
			wantTimestamp: ptr.To(restartID),
		},
		{
			name: "sequential restart in progress annotates completed + in-progress",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
						"Backend":  {},
					},
					Restart: &v1alpha1.Restart{
						ID: restartID,
						Strategy: &v1alpha1.RestartStrategy{
							Type:  v1alpha1.RestartStrategyTypeSequential,
							Order: []string{"Frontend", "Worker", "Backend"},
						},
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: restartID,
				Phase:      v1alpha1.RestartPhaseRestarting,
				InProgress: []string{"Worker"},
			},
			wantSvcs:      []string{"Frontend", "Worker"}, // Frontend completed, Worker in progress
			wantTimestamp: ptr.To(restartID),
		},
		{
			name: "default restart in progress annotates completed + in-progress",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
						"Backend":  {},
					},
					Restart: &v1alpha1.Restart{
						ID: restartID,
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: restartID,
				Phase:      v1alpha1.RestartPhaseRestarting,
				InProgress: []string{"Worker"},
			},
			wantSvcs:      []string{"Frontend", "Worker", "Backend"},
			wantTimestamp: ptr.To(restartID),
		},
		{
			name: "completed restart with empty spec restart preserves all annotations",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: oldRestartID,
				Phase:      v1alpha1.RestartPhaseCompleted,
			},
			wantSvcs:      []string{"Frontend", "Worker"},
			wantTimestamp: ptr.To(oldRestartID),
		},
		{
			name: "completed restart",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
					Restart: &v1alpha1.Restart{
						ID: restartID,
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: restartID,
				Phase:      v1alpha1.RestartPhaseCompleted,
			},
			wantSvcs:      []string{"Frontend", "Worker"},
			wantTimestamp: ptr.To(restartID),
		},
		{
			name: "new restart after completed restart",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
					Restart: &v1alpha1.Restart{
						ID: restartID, // new time
						Strategy: &v1alpha1.RestartStrategy{
							Type: v1alpha1.RestartStrategyTypeParallel,
						},
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: oldRestartID,
				Phase:      v1alpha1.RestartPhaseCompleted,
			},
			wantSvcs:      []string{"Frontend", "Worker"},
			wantTimestamp: ptr.To(restartID),
		},
		{
			name: "superseded restart returns nil - preserves existing annotations via fallback",
			dgd: &v1alpha1.DynamoGraphDeployment{
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
						"Worker":   {},
					},
					Restart: &v1alpha1.Restart{
						ID: restartID,
					},
				},
			},
			restartStatus: &v1alpha1.RestartStatus{
				ObservedID: restartID,
				Phase:      v1alpha1.RestartPhaseSuperseded,
			},
			wantNil: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := DetermineRestartState(tt.dgd, tt.restartStatus)

			if tt.wantNil {
				if got != nil {
					t.Errorf("DetermineGroveRestartState() = %v, want nil", got)
				}
				return
			}

			if got == nil {
				t.Errorf("DetermineGroveRestartState() = nil, want non-nil")
				return
			}

			var gotSvcs []string
			for svc, shouldAnnotate := range got.ServicesToAnnotate {
				if shouldAnnotate {
					gotSvcs = append(gotSvcs, svc)
				}
			}
			sort.Strings(gotSvcs)
			sort.Strings(tt.wantSvcs)

			if !reflect.DeepEqual(gotSvcs, tt.wantSvcs) {
				t.Errorf("DetermineGroveRestartState() services = %v, want %v", gotSvcs, tt.wantSvcs)
			}
			if tt.wantTimestamp != nil && (got.Timestamp != *tt.wantTimestamp) {
				t.Errorf("DetermineGroveRestartState() timestamp = %v, want %v", got.Timestamp, *tt.wantTimestamp)
			}
		})
	}
}

func TestGroveRestartStateShouldAnnotateService(t *testing.T) {
	tests := []struct {
		name        string
		state       *RestartState
		serviceName string
		want        bool
	}{
		{
			name:        "nil state returns false",
			state:       nil,
			serviceName: "Frontend",
			want:        false,
		},
		{
			name: "nil services map returns false",
			state: &RestartState{
				Timestamp:          "2024-01-01T00:00:00Z",
				ServicesToAnnotate: nil,
			},
			serviceName: "Frontend",
			want:        false,
		},
		{
			name: "service in map returns true",
			state: &RestartState{
				Timestamp:          "2024-01-01T00:00:00Z",
				ServicesToAnnotate: map[string]bool{"Frontend": true, "Worker": true},
			},
			serviceName: "Frontend",
			want:        true,
		},
		{
			name: "service not in map returns false",
			state: &RestartState{
				Timestamp:          "2024-01-01T00:00:00Z",
				ServicesToAnnotate: map[string]bool{"Frontend": true},
			},
			serviceName: "Worker",
			want:        false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := tt.state.ShouldAnnotateService(tt.serviceName); got != tt.want {
				t.Errorf("ShouldAnnotateService() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestGenerateGrovePodCliqueSet_RestartAnnotations(t *testing.T) {
	restartTimestamp := "2024-01-05T10:00:00Z"

	tests := []struct {
		name                     string
		restartState             *RestartState
		services                 map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec
		wantAnnotationsPerClique map[string]bool              // clique name -> should have restart annotation
		wantPreservedAnnotations map[string]map[string]string // clique name -> preserved annotations to verify
	}{
		{
			name:         "nil restartState - no annotations",
			restartState: nil,
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType: commonconsts.ComponentTypeFrontend,
					Replicas:      ptr.To(int32(1)),
				},
				"Worker": {
					ComponentType: commonconsts.ComponentTypeWorker,
					Replicas:      ptr.To(int32(1)),
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"frontend": false,
				"worker":   false,
			},
		},
		{
			name: "nil ServicesToAnnotate - no annotations",
			restartState: &RestartState{
				Timestamp:          restartTimestamp,
				ServicesToAnnotate: nil,
			},
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType: commonconsts.ComponentTypeFrontend,
					Replicas:      ptr.To(int32(1)),
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"frontend": false,
			},
		},
		{
			name: "all services annotated - parallel restart",
			restartState: &RestartState{
				Timestamp:          restartTimestamp,
				ServicesToAnnotate: map[string]bool{"Frontend": true, "Worker": true},
			},
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType: commonconsts.ComponentTypeFrontend,
					Replicas:      ptr.To(int32(1)),
				},
				"Worker": {
					ComponentType: commonconsts.ComponentTypeWorker,
					Replicas:      ptr.To(int32(1)),
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"frontend": true,
				"worker":   true,
			},
		},
		{
			name: "only first service annotated - sequential restart start",
			restartState: &RestartState{
				Timestamp:          restartTimestamp,
				ServicesToAnnotate: map[string]bool{"Frontend": true},
			},
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType: commonconsts.ComponentTypeFrontend,
					Replicas:      ptr.To(int32(1)),
				},
				"Worker": {
					ComponentType: commonconsts.ComponentTypeWorker,
					Replicas:      ptr.To(int32(1)),
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"frontend": true,
				"worker":   false,
			},
		},
		{
			name: "completed services keep annotation - sequential restart in progress",
			restartState: &RestartState{
				Timestamp:          restartTimestamp,
				ServicesToAnnotate: map[string]bool{"Frontend": true, "Worker": true},
			},
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType: commonconsts.ComponentTypeFrontend,
					Replicas:      ptr.To(int32(1)),
				},
				"Worker": {
					ComponentType: commonconsts.ComponentTypeWorker,
					Replicas:      ptr.To(int32(1)),
				},
				"Backend": {
					ComponentType: commonconsts.ComponentTypeWorker,
					Replicas:      ptr.To(int32(1)),
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"frontend": true,
				"worker":   true,
				"backend":  false,
			},
		},
		{
			name: "service not in DGD spec - annotation still applied if in ServicesToAnnotate",
			restartState: &RestartState{
				Timestamp:          restartTimestamp,
				ServicesToAnnotate: map[string]bool{"Frontend": true, "NonExistent": true},
			},
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType: commonconsts.ComponentTypeFrontend,
					Replicas:      ptr.To(int32(1)),
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"frontend": true,
			},
		},
		{
			name: "multinode service - all cliques get restart annotation",
			restartState: &RestartState{
				Timestamp:          restartTimestamp,
				ServicesToAnnotate: map[string]bool{"Worker": true},
			},
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Worker": {
					ComponentType: commonconsts.ComponentTypeWorker,
					Replicas:      ptr.To(int32(2)),
					Multinode: &v1alpha1.MultinodeSpec{
						NodeCount: 2,
					},
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"worker-ldr": true,
				"worker-wkr": true,
			},
		},
		{
			name: "preserves existing annotations when adding restart annotation",
			restartState: &RestartState{
				Timestamp:          restartTimestamp,
				ServicesToAnnotate: map[string]bool{"Frontend": true},
			},
			services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"Frontend": {
					ComponentType: commonconsts.ComponentTypeFrontend,
					Replicas:      ptr.To(int32(1)),
					Annotations: map[string]string{
						"custom-annotation": "custom-value",
						"another-key":       "another-value",
					},
				},
			},
			wantAnnotationsPerClique: map[string]bool{
				"frontend": true,
			},
			wantPreservedAnnotations: map[string]map[string]string{
				"frontend": {
					"custom-annotation": "custom-value",
					"another-key":       "another-value",
				},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			dgd := &v1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-dgd",
					Namespace: "default",
				},
				Spec: v1alpha1.DynamoGraphDeploymentSpec{
					Services: tt.services,
				},
			}

			controllerConfig := &configv1alpha1.OperatorConfiguration{
				Infrastructure: configv1alpha1.InfrastructureConfiguration{
					ETCDAddress: "etcd-address",
					NATSAddress: "nats-address",
				},
			}

			got, err := GenerateGrovePodCliqueSet(context.Background(), dgd, controllerConfig, &controller_common.RuntimeConfig{}, nil, tt.restartState, nil, nil)
			if err != nil {
				t.Fatalf("GenerateGrovePodCliqueSet() error = %v", err)
			}

			// Build a map of clique annotations
			cliqueAnnotations := make(map[string]map[string]string)
			for _, clique := range got.Spec.Template.Cliques {
				cliqueAnnotations[clique.Name] = clique.Annotations
			}

			// Verify restart annotations per clique
			for cliqueName, shouldHaveAnnotation := range tt.wantAnnotationsPerClique {
				annotations := cliqueAnnotations[cliqueName]

				if shouldHaveAnnotation {
					if annotations == nil {
						t.Errorf("Clique %q: expected restart annotation, but annotations is nil", cliqueName)
						continue
					}
					restartValue, exists := annotations[commonconsts.RestartAnnotation]
					if !exists {
						t.Errorf("Clique %q: expected restart annotation %q, but not found. Annotations: %v",
							cliqueName, commonconsts.RestartAnnotation, annotations)
						continue
					}
					if restartValue != restartTimestamp {
						t.Errorf("Clique %q: restart annotation value = %q, want %q",
							cliqueName, restartValue, restartTimestamp)
					}
				} else {
					if annotations != nil {
						if _, exists := annotations[commonconsts.RestartAnnotation]; exists {
							t.Errorf("Clique %q: unexpected restart annotation found", cliqueName)
						}
					}
				}
			}

			// Verify no unexpected restart annotations on cliques not in wantAnnotationsPerClique
			for cliqueName, annotations := range cliqueAnnotations {
				if _, specified := tt.wantAnnotationsPerClique[cliqueName]; !specified {
					if annotations != nil {
						if _, exists := annotations[commonconsts.RestartAnnotation]; exists {
							t.Errorf("Clique %q: unexpected restart annotation found (clique not in wantAnnotationsPerClique)", cliqueName)
						}
					}
				}
			}

			// Verify preserved annotations
			for cliqueName, expectedAnnotations := range tt.wantPreservedAnnotations {
				annotations := cliqueAnnotations[cliqueName]
				if annotations == nil {
					t.Errorf("Clique %q: expected preserved annotations, but annotations is nil", cliqueName)
					continue
				}
				for key, expectedValue := range expectedAnnotations {
					if actualValue, exists := annotations[key]; !exists {
						t.Errorf("Clique %q: expected preserved annotation %q, but not found", cliqueName, key)
					} else if actualValue != expectedValue {
						t.Errorf("Clique %q: preserved annotation %q = %q, want %q",
							cliqueName, key, actualValue, expectedValue)
					}
				}
			}
		})
	}
}

func TestIsWorkerComponent(t *testing.T) {
	workers := []string{commonconsts.ComponentTypeWorker, commonconsts.ComponentTypePrefill, commonconsts.ComponentTypeDecode}
	nonWorkers := []string{commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypePlanner, commonconsts.ComponentTypeEPP, "custom", ""}

	for _, ct := range workers {
		assert.True(t, IsWorkerComponent(ct), "%s should be a worker", ct)
	}
	for _, ct := range nonWorkers {
		assert.False(t, IsWorkerComponent(ct), "%s should not be a worker", ct)
	}
}

func TestRollingUpdateContext_InProgress(t *testing.T) {
	assert.False(t, RollingUpdateContext{}.InProgress())
	assert.False(t, RollingUpdateContext{NewWorkerHash: "abc"}.InProgress())
	assert.True(t, RollingUpdateContext{OldWorkerReplicas: map[string]int32{"w": 1}}.InProgress())
}

func TestGetDCDResourceName(t *testing.T) {
	dgd := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{Name: "my-dgd"},
		Spec: v1alpha1.DynamoGraphDeploymentSpec{
			Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"prefill":  {ComponentType: commonconsts.ComponentTypePrefill},
				"decode":   {ComponentType: commonconsts.ComponentTypeDecode},
				"worker":   {ComponentType: commonconsts.ComponentTypeWorker},
				"frontend": {ComponentType: commonconsts.ComponentTypeFrontend},
			},
		},
	}

	// Workers get hash suffix
	assert.Equal(t, "my-dgd-prefill-abc12345", GetDCDResourceName(dgd, "prefill", "abc12345"))
	assert.Equal(t, "my-dgd-decode-abc12345", GetDCDResourceName(dgd, "decode", "abc12345"))
	assert.Equal(t, "my-dgd-worker-abc12345", GetDCDResourceName(dgd, "worker", "abc12345"))

	// Non-workers never get hash suffix
	assert.Equal(t, "my-dgd-frontend", GetDCDResourceName(dgd, "frontend", "abc12345"))

	// Empty hash — workers don't get suffix
	assert.Equal(t, "my-dgd-prefill", GetDCDResourceName(dgd, "prefill", ""))
}

func TestGenerateSingleDCD_RollingUpdateContext(t *testing.T) {
	dgd := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{Name: "my-dgd", Namespace: "ns"},
		Spec: v1alpha1.DynamoGraphDeploymentSpec{
			Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"prefill":  {ComponentType: commonconsts.ComponentTypePrefill, Replicas: ptr.To(int32(4))},
				"frontend": {ComponentType: commonconsts.ComponentTypeFrontend, Replicas: ptr.To(int32(1))},
			},
		},
	}

	ctx := context.Background()
	ruCtx := RollingUpdateContext{
		NewWorkerHash:     "aabb1122",
		OldWorkerReplicas: map[string]int32{"prefill": 2},
		NewWorkerReplicas: map[string]int32{"prefill": 2},
	}

	dcds, err := GenerateDynamoComponentsDeployments(ctx, dgd, nil, &RestartState{}, nil, ruCtx)
	assert.NoError(t, err)

	// Worker DCD: hash suffix in name, hash label, replica override
	prefillDCD := dcds["prefill"]
	assert.Equal(t, "my-dgd-prefill-aabb1122", prefillDCD.Name)
	assert.Equal(t, "aabb1122", prefillDCD.Labels[commonconsts.KubeLabelDynamoWorkerHash])
	assert.Equal(t, int32(2), *prefillDCD.Spec.Replicas)

	// Non-worker DCD: no hash suffix, no hash label, original replicas
	frontendDCD := dcds["frontend"]
	assert.Equal(t, "my-dgd-frontend", frontendDCD.Name)
	assert.Empty(t, frontendDCD.Labels[commonconsts.KubeLabelDynamoWorkerHash])
	assert.Equal(t, int32(1), *frontendDCD.Spec.Replicas)
}

func TestGenerateSingleDCD_NoRollingUpdate(t *testing.T) {
	dgd := &v1alpha1.DynamoGraphDeployment{
		ObjectMeta: metav1.ObjectMeta{Name: "my-dgd", Namespace: "ns"},
		Spec: v1alpha1.DynamoGraphDeploymentSpec{
			Services: map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
				"worker": {ComponentType: commonconsts.ComponentTypeWorker, Replicas: ptr.To(int32(3))},
			},
		},
	}

	dcds, err := GenerateDynamoComponentsDeployments(context.Background(), dgd, nil, &RestartState{}, nil, RollingUpdateContext{})
	assert.NoError(t, err)

	dcd := dcds["worker"]
	assert.Equal(t, "my-dgd-worker", dcd.Name)
	assert.Empty(t, dcd.Labels[commonconsts.KubeLabelDynamoWorkerHash])
	assert.Equal(t, int32(3), *dcd.Spec.Replicas)
}

func TestGenerateComponentContext_WorkerHashSuffix(t *testing.T) {
	// Worker with hash label gets WorkerHashSuffix
	component := &v1alpha1.DynamoComponentDeploymentSharedSpec{
		ComponentType: commonconsts.ComponentTypeWorker,
		Labels:        map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "abc123"},
	}
	compCtx := generateComponentContext(component, "dgd", "ns", 1, "kubernetes")
	assert.Equal(t, "abc123", compCtx.WorkerHashSuffix)

	// Worker without hash label
	component2 := &v1alpha1.DynamoComponentDeploymentSharedSpec{
		ComponentType: commonconsts.ComponentTypeWorker,
	}
	compCtx2 := generateComponentContext(component2, "dgd", "ns", 1, "kubernetes")
	assert.Empty(t, compCtx2.WorkerHashSuffix)

	// Frontend never gets WorkerHashSuffix, even with the label
	component3 := &v1alpha1.DynamoComponentDeploymentSharedSpec{
		ComponentType: commonconsts.ComponentTypeFrontend,
		Labels:        map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "abc123"},
	}
	compCtx3 := generateComponentContext(component3, "dgd", "ns", 1, "kubernetes")
	assert.Empty(t, compCtx3.WorkerHashSuffix)
}

func TestWorkerDefaults_WorkerHashSuffixEnvVar(t *testing.T) {
	w := NewWorkerDefaults()

	// With suffix
	container, err := w.GetBaseContainer(ComponentContext{
		DynamoNamespace:  "ns-dgd",
		ComponentType:    commonconsts.ComponentTypeWorker,
		WorkerHashSuffix: "abc123",
	})
	assert.NoError(t, err)
	found := false
	for _, env := range container.Env {
		if env.Name == commonconsts.DynamoNamespaceWorkerSuffixEnvVar {
			assert.Equal(t, "abc123", env.Value)
			found = true
		}
	}
	assert.True(t, found, "DYN_NAMESPACE_WORKER_SUFFIX should be set")

	// Without suffix — env var should NOT be present
	container2, err := w.GetBaseContainer(ComponentContext{
		DynamoNamespace: "ns-dgd",
		ComponentType:   commonconsts.ComponentTypeWorker,
	})
	assert.NoError(t, err)
	for _, env := range container2.Env {
		assert.NotEqual(t, commonconsts.DynamoNamespaceWorkerSuffixEnvVar, env.Name,
			"DYN_NAMESPACE_WORKER_SUFFIX should not be set when suffix is empty")
	}
}

func TestFrontendDefaults_NamespacePrefixEnvVar(t *testing.T) {
	f := NewFrontendDefaults()
	container, err := f.GetBaseContainer(ComponentContext{
		DynamoNamespace: "myns-mydgd",
		ComponentType:   commonconsts.ComponentTypeFrontend,
	})
	assert.NoError(t, err)
	found := false
	for _, env := range container.Env {
		if env.Name == commonconsts.DynamoNamespacePrefixEnvVar {
			assert.Equal(t, "myns-mydgd", env.Value)
			found = true
		}
	}
	assert.True(t, found, "DYN_NAMESPACE_PREFIX should be set on frontend")
}

func TestPropagateDGDAnnotations(t *testing.T) {
	tests := []struct {
		name               string
		dgdAnnotations     map[string]string
		serviceAnnotations map[string]string
		expectedAnnotation map[string]string
	}{
		{
			name: "DGD annotation propagates to empty service annotations",
			dgdAnnotations: map[string]string{
				commonconsts.KubeAnnotationVLLMDistributedExecutorBackend: "ray",
			},
			serviceAnnotations: nil,
			expectedAnnotation: map[string]string{
				commonconsts.KubeAnnotationVLLMDistributedExecutorBackend: "ray",
			},
		},
		{
			name: "service-level annotation takes precedence over DGD",
			dgdAnnotations: map[string]string{
				commonconsts.KubeAnnotationVLLMDistributedExecutorBackend: "ray",
			},
			serviceAnnotations: map[string]string{
				commonconsts.KubeAnnotationVLLMDistributedExecutorBackend: "mp",
			},
			expectedAnnotation: map[string]string{
				commonconsts.KubeAnnotationVLLMDistributedExecutorBackend: "mp",
			},
		},
		{
			name:               "no DGD annotation, no service annotation",
			dgdAnnotations:     nil,
			serviceAnnotations: nil,
			expectedAnnotation: nil,
		},
		{
			name: "origin version also propagates",
			dgdAnnotations: map[string]string{
				commonconsts.KubeAnnotationDynamoOperatorOriginVersion: "1.0.0",
			},
			serviceAnnotations: nil,
			expectedAnnotation: map[string]string{
				commonconsts.KubeAnnotationDynamoOperatorOriginVersion: "1.0.0",
			},
		},
		{
			name: "unrelated DGD annotations are not propagated",
			dgdAnnotations: map[string]string{
				"some-other-annotation": "value",
			},
			serviceAnnotations: nil,
			expectedAnnotation: nil,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			component := &v1alpha1.DynamoComponentDeploymentSharedSpec{
				Annotations: tt.serviceAnnotations,
			}
			propagateDGDAnnotations(tt.dgdAnnotations, component)

			if tt.expectedAnnotation == nil {
				assert.True(t, len(component.Annotations) == 0 || component.Annotations == nil,
					"expected no annotations, got %v", component.Annotations)
			} else {
				for k, v := range tt.expectedAnnotation {
					assert.Equal(t, v, component.Annotations[k], "annotation %s mismatch", k)
				}
			}
		})
	}
}
