dynamographdeployment_test.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package validation

import (
	"context"
	"sort"
	"strings"
	"testing"

	nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1"
	"github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) {
	var (
		validReplicas    = int32(3)
		negativeReplicas = int32(-1)
		pvcName          = "test-pvc"
		trueVal          = true
		falseVal         = false
	)

	tests := []struct {
		name        string
		deployment  *nvidiacomv1alpha1.DynamoGraphDeployment
		wantErr     bool
		errMsg      string
		errContains bool
	}{
		{
			name: "valid deployment with services",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Replicas: &validReplicas,
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "no services",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{},
				},
			},
			wantErr: true,
			errMsg:  "spec.services must have at least one service",
		},
		{
			name: "service with invalid replicas",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Replicas: &negativeReplicas,
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main].replicas must be non-negative",
		},
		{
			name: "service with invalid ingress",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"gateway": {
							Ingress: &nvidiacomv1alpha1.IngressSpec{
								Enabled: true,
								Host:    "",
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[gateway].ingress.host is required when ingress is enabled",
		},
		{
			name: "pvc with create=true and missing storageClass",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					PVCs: []nvidiacomv1alpha1.PVC{
						{
							Create:           &trueVal,
							Name:             &pvcName,
							StorageClass:     "",
							Size:             resource.MustParse("10Gi"),
							VolumeAccessMode: corev1.ReadWriteOnce,
						},
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.pvcs[0].storageClass is required when create is true",
		},
		{
			name: "pvc with create=true and missing size",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					PVCs: []nvidiacomv1alpha1.PVC{
						{
							Create:           &trueVal,
							Name:             &pvcName,
							StorageClass:     "standard",
							Size:             resource.Quantity{},
							VolumeAccessMode: corev1.ReadWriteOnce,
						},
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.pvcs[0].size is required when create is true",
		},
		{
			name: "pvc with create=true and missing volumeAccessMode",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					PVCs: []nvidiacomv1alpha1.PVC{
						{
							Create:           &trueVal,
							Name:             &pvcName,
							StorageClass:     "standard",
							Size:             resource.MustParse("10Gi"),
							VolumeAccessMode: "",
						},
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.pvcs[0].volumeAccessMode is required when create is true",
		},
		{
			name: "pvc with create=false and missing fields",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					PVCs: []nvidiacomv1alpha1.PVC{
						{
							Create: &falseVal,
							Name:   &pvcName,
						},
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "pvc with missing name",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					PVCs: []nvidiacomv1alpha1.PVC{
						{
							Create: &falseVal,
						},
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.pvcs[0].name is required",
		},
		{
			name: "pvc with multiple errors (name, storageClass, size, volumeAccessMode all missing)",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					PVCs: []nvidiacomv1alpha1.PVC{
						{
							Create: &trueVal,
						},
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr:     true,
			errMsg:      "spec.pvcs[0].name is required\nspec.pvcs[0].storageClass is required when create is true\nspec.pvcs[0].size is required when create is true\nspec.pvcs[0].volumeAccessMode is required when create is true",
			errContains: true,
		},
		{
			name: "valid pvc with create=true",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					PVCs: []nvidiacomv1alpha1.PVC{
						{
							Create:           &trueVal,
							Name:             &pvcName,
							StorageClass:     "standard",
							Size:             resource.MustParse("10Gi"),
							VolumeAccessMode: corev1.ReadWriteOnce,
						},
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "service with invalid volume mount",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							VolumeMounts: []nvidiacomv1alpha1.VolumeMount{
								{
									Name:                  "data",
									UseAsCompilationCache: false,
								},
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main].volumeMounts[0].mountPoint is required when useAsCompilationCache is false",
		},
		{
			name: "service with invalid shared memory",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							SharedMemory: &nvidiacomv1alpha1.SharedMemorySpec{
								Disabled: false,
								Size:     resource.Quantity{},
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main].sharedMemory.size is required when disabled is false",
		},
		// Restart validation test cases
		{
			name: "restart with nil at",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "",
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.restart.id is required",
		},
		{
			name: "restart with valid id and no strategy",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "restart-id",
					},
				},
			},
			wantErr: false,
		},
		{
			name: "restart with parallel strategy and order specified",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main":    {},
						"prefill": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "restart-id",
						Strategy: &nvidiacomv1alpha1.RestartStrategy{
							Type:  nvidiacomv1alpha1.RestartStrategyTypeParallel,
							Order: []string{"main", "prefill"},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.restart.strategy.order cannot be specified when strategy is parallel",
		},
		{
			name: "restart with sequential strategy and duplicate services in order",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main":    {},
						"prefill": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "restart-id",
						Strategy: &nvidiacomv1alpha1.RestartStrategy{
							Type:  nvidiacomv1alpha1.RestartStrategyTypeSequential,
							Order: []string{"main", "main", "prefill"},
						},
					},
				},
			},
			wantErr:     true,
			errMsg:      "spec.restart.strategy.order must be unique",
			errContains: true,
		},
		{
			name: "restart with sequential strategy and unknown service in order",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main":    {},
						"prefill": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "restart-id",
						Strategy: &nvidiacomv1alpha1.RestartStrategy{
							Type:  nvidiacomv1alpha1.RestartStrategyTypeSequential,
							Order: []string{"main", "unknown"},
						},
					},
				},
			},
			wantErr:     true,
			errMsg:      "spec.restart.strategy.order contains unknown service: unknown",
			errContains: true,
		},
		{
			name: "restart with sequential strategy and missing service in order",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main":    {},
						"prefill": {},
						"decode":  {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "restart-id",
						Strategy: &nvidiacomv1alpha1.RestartStrategy{
							Type:  nvidiacomv1alpha1.RestartStrategyTypeSequential,
							Order: []string{"main", "prefill"},
						},
					},
				},
			},
			wantErr:     true,
			errMsg:      "spec.restart.strategy.order must have the same number of unique services as the deployment",
			errContains: true,
		},
		{
			name: "restart with valid sequential strategy and order",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main":    {},
						"prefill": {},
						"decode":  {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "restart-id",
						Strategy: &nvidiacomv1alpha1.RestartStrategy{
							Type:  nvidiacomv1alpha1.RestartStrategyTypeSequential,
							Order: []string{"prefill", "decode", "main"},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "restart with sequential strategy and empty order is valid",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "restart-id",
						Strategy: &nvidiacomv1alpha1.RestartStrategy{
							Type:  nvidiacomv1alpha1.RestartStrategyTypeSequential,
							Order: []string{},
						},
					},
				},
			},
			wantErr: false,
		},
		// Service name length validation tests
		{
			name: "service name too long for single-node deployment",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "verylongdynamographdeploymentname",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"VeryLongServiceNameThatExceedsLimit": {},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "combined resource name length",
		},
		{
			name: "service name too long for multinode deployment",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "vllm-agg",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"VllmPrefillWorker": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "combined resource name length",
		},
		{
			name: "valid service name length for single-node",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "dgd",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Frontend": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "valid service name length for multinode",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "dgd",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "boundary case - exactly at 45 char limit for single-node",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					// DGD name (3 chars) + service name (42 chars) = 45 chars (exactly at limit)
					Name:      "dgd",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						// 42 character service name
						"abcdefghijklmnopqrstuvwxyz0123456789ABCDEF": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "boundary case - one char over limit for single-node",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					// DGD name (3 chars) + service name (43 chars) = 46 chars (over limit)
					Name:      "dgd",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						// 43 character service name
						"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG": {},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "combined resource name length 46 exceeds 45-character limit",
		},
		// Grove disabled tests - service name length validation should be skipped
		{
			name: "long service name allowed when Grove disabled via annotation",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "verylongdynamographdeploymentname",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"VeryLongServiceNameThatExceedsLimit": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "long multinode service name allowed when Grove disabled via annotation",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "vllm-agg",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"VllmPrefillWorker": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "Grove annotation case insensitive - FALSE",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "verylongdynamographdeploymentname",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "FALSE",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"VeryLongServiceNameThatExceedsLimit": {},
					},
				},
			},
			wantErr: false,
		},
		// Annotation validation test cases
		{
			name: "valid annotation vllm-distributed-executor-backend=mp",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationVLLMDistributedExecutorBackend: "mp",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "valid annotation vllm-distributed-executor-backend=ray",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationVLLMDistributedExecutorBackend: "ray",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "valid annotation vllm-distributed-executor-backend case insensitive MP",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationVLLMDistributedExecutorBackend: "MP",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "invalid annotation vllm-distributed-executor-backend",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationVLLMDistributedExecutorBackend: "invalid",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: true,
			errMsg:  `annotation nvidia.com/vllm-distributed-executor-backend has invalid value "invalid": must be "mp" or "ray"`,
		},
		{
			name: "no annotations is valid",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "valid annotation dynamo-operator-origin-version with semver",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationDynamoOperatorOriginVersion: "1.0.0",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "valid annotation dynamo-operator-origin-version with pre-release",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationDynamoOperatorOriginVersion: "1.0.0",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "valid annotation dynamo-operator-origin-version fallback version",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationDynamoOperatorOriginVersion: "0.0.0-unknown",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "invalid annotation dynamo-operator-origin-version not semver",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationDynamoOperatorOriginVersion: "not-a-version",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: true,
			errMsg:  `annotation nvidia.com/dynamo-operator-origin-version has invalid value "not-a-version": must be valid semver`,
		},
		// Topology constraint validation tests
		{
			name: "no topology constraints is valid (backward compatible)",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "valid topology constraints with spec and service level",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("zone"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("block"),
							},
						},
						"Frontend": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("zone"),
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "spec-level with topologyProfile only (no packDomain) is rejected when service lacks constraint",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
						"Frontend": {},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "spec.services[Frontend].topologyConstraint is required because spec.topologyConstraint.packDomain is not set",
		},
		{
			name: "spec-level set but service has no topology constraint is valid (inherits)",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("zone"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "invalid packDomain format at spec level",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("INVALID!"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "is not a valid topology domain",
		},
		{
			name: "service domain equal to spec-level is valid (no hierarchy check without CRD)",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("rack"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "mixed: spec-level with some services having constraints and some not",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("zone"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
						"Frontend": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "topologyProfile missing at spec level when service has constraint",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "spec.topologyConstraint with topologyProfile is required",
		},
		{
			name: "topologyProfile empty at spec level when service has constraint",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						PackDomain: nvidiacomv1alpha1.TopologyDomain("zone"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "topologyProfile is required",
		},
		{
			name: "service-level topologyConstraint without packDomain is rejected",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("zone"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{},
						},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "packDomain is required",
		},
		{
			name: "invalid packDomain format at service level",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("zone"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("INVALID!"),
							},
						},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "is not a valid topology domain",
		},
		{
			name: "service domain narrower than spec-level is valid",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
						PackDomain:      nvidiacomv1alpha1.TopologyDomain("zone"),
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("host"),
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "no spec packDomain but all services have topology constraint is valid",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
						"Frontend": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("zone"),
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "no spec packDomain and service missing topology constraint is rejected",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationEnableGrove: "false",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					TopologyConstraint: &nvidiacomv1alpha1.SpecTopologyConstraint{
						TopologyProfile: "test-topology",
					},
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"Worker": {
							TopologyConstraint: &nvidiacomv1alpha1.TopologyConstraint{
								PackDomain: nvidiacomv1alpha1.TopologyDomain("rack"),
							},
						},
						"Frontend": {},
					},
				},
			},
			wantErr:     true,
			errContains: true,
			errMsg:      "spec.services[Frontend].topologyConstraint is required because spec.topologyConstraint.packDomain is not set",
		},
		{
			name: "both annotations valid",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "both annotations invalid",
			deployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-graph",
					Namespace: "default",
					Annotations: map[string]string{
						consts.KubeAnnotationDynamoOperatorOriginVersion:    "bad",
						consts.KubeAnnotationVLLMDistributedExecutorBackend: "invalid",
					},
				},
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {},
					},
				},
			},
			wantErr:     true,
			errMsg:      "annotation nvidia.com/dynamo-operator-origin-version has invalid value \"bad\": must be valid semver\nannotation nvidia.com/vllm-distributed-executor-backend has invalid value \"invalid\": must be \"mp\" or \"ray\"",
			errContains: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			validator := NewDynamoGraphDeploymentValidator(tt.deployment)
			_, err := validator.Validate(context.Background())

			if (err != nil) != tt.wantErr {
				t.Errorf("DynamoGraphDeploymentValidator.Validate() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if tt.wantErr {
				if tt.errContains {
					// For multiple errors, check that all expected error messages are present
					errStr := err.Error()
					for _, expectedMsg := range strings.Split(tt.errMsg, "\n") {
						if !strings.Contains(errStr, expectedMsg) {
							t.Errorf("DynamoGraphDeploymentValidator.Validate() error message = %v, want to contain %v", errStr, expectedMsg)
						}
					}
				} else {
					if err.Error() != tt.errMsg {
						t.Errorf("DynamoGraphDeploymentValidator.Validate() error message = %v, want %v", err.Error(), tt.errMsg)
					}
				}
			}
		})
	}
}

func TestDynamoGraphDeploymentValidator_ValidateUpdate(t *testing.T) {
	tests := []struct {
		name            string
		oldDeployment   *nvidiacomv1alpha1.DynamoGraphDeployment
		newDeployment   *nvidiacomv1alpha1.DynamoGraphDeployment
		wantErr         bool
		wantWarnings    bool
		errMsg          string
		expectedWarnMsg string
	}{
		{
			name: "no changes",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
				},
			},
			wantErr: false,
		},
		{
			name: "changing backend framework",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "vllm",
				},
			},
			wantErr:         true,
			wantWarnings:    true,
			errMsg:          "spec.backendFramework is immutable and cannot be changed after creation",
			expectedWarnMsg: "Changing spec.backendFramework may cause unexpected behavior",
		},
		{
			name: "adding single service is prohibited",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend": {},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend":  {},
						"frontend": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "service topology is immutable and cannot be modified after creation: services added: [frontend]",
		},
		{
			name: "adding multiple services is prohibited",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend": {},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend":  {},
						"cache":    {},
						"frontend": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "service topology is immutable and cannot be modified after creation: services added: [cache frontend]",
		},
		{
			name: "removing single service is prohibited",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend":  {},
						"frontend": {},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "service topology is immutable and cannot be modified after creation: services removed: [frontend]",
		},
		{
			name: "removing multiple services is prohibited",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend":  {},
						"cache":    {},
						"frontend": {},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "service topology is immutable and cannot be modified after creation: services removed: [cache frontend]",
		},
		{
			name: "adding and removing services simultaneously is prohibited",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend": {},
						"cache":   {},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend":  {},
						"frontend": {},
					},
				},
			},
			wantErr: true,
			errMsg:  "service topology is immutable and cannot be modified after creation: services added: [frontend], services removed: [cache]",
		},
		{
			name: "modifying service specifications is allowed",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend": {
							Replicas: func() *int32 { r := int32(1); return &r }(),
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend": {
							Replicas: func() *int32 { r := int32(3); return &r }(),
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "service topology unchanged with same services",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend":  {},
						"frontend": {},
						"cache":    {},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"backend":  {},
						"frontend": {},
						"cache":    {},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "changing service from single-node to multi-node",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							// Single-node (nil Multinode)
							Multinode: nil,
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							// Multi-node (NodeCount > 1)
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main] cannot change node topology (between single-node and multi-node) after creation",
		},
		{
			name: "changing service from multi-node to single-node",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							// Multi-node
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 3,
							},
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							// Single-node (nil Multinode)
							Multinode: nil,
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main] cannot change node topology (between single-node and multi-node) after creation",
		},
		{
			name: "changing multinode NodeCount within multi-node range is allowed",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 4,
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "keeping service as single-node is allowed",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: nil,
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: nil,
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "keeping service as multi-node is allowed",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 3,
							},
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 3,
							},
						},
					},
				},
			},
			wantErr: false,
		},
		{
			name: "changing from single-node (NodeCount=1) to multi-node (NodeCount=2)",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 1,
							},
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main] cannot change node topology (between single-node and multi-node) after creation",
		},
		{
			name: "changing from multi-node (NodeCount=2) to single-node (NodeCount=1)",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 1,
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main] cannot change node topology (between single-node and multi-node) after creation",
		},
		{
			name: "multiple services with one changing topology",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: nil,
						},
						"prefill": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							// Changing from single-node to multi-node
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 3,
							},
						},
						"prefill": {
							// Keeping as multi-node (OK)
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 4,
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.services[main] cannot change node topology (between single-node and multi-node) after creation",
		},
		{
			name: "adding new service with multinode is not allowed", // service topology is immutable
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: nil,
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: nil,
						},
						"decode": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 4,
							},
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "service topology is immutable and cannot be modified after creation: services added: [decode]",
		},
		{
			name: "adding new service without multinode is not allowed", // service topology is immutable
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"main": {
							Multinode: &nvidiacomv1alpha1.MultinodeSpec{
								NodeCount: 2,
							},
						},
						"gateway": {
							// New service without multinode - should be allowed
							Multinode: nil,
						},
					},
				},
			},
			wantErr: true,
			errMsg:  "service topology is immutable and cannot be modified after creation: services added: [gateway]",
		},
		{
			name: "restart.id change while rolling update Pending - rejected",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "old-restart-id",
					},
				},
				Status: nvidiacomv1alpha1.DynamoGraphDeploymentStatus{
					RollingUpdate: &nvidiacomv1alpha1.RollingUpdateStatus{
						Phase: nvidiacomv1alpha1.RollingUpdatePhasePending,
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "new-restart-id",
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.restart.id cannot be changed while a rolling update is Pending",
		},
		{
			name: "restart.id change while rolling update InProgress - rejected",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "old-restart-id",
					},
				},
				Status: nvidiacomv1alpha1.DynamoGraphDeploymentStatus{
					RollingUpdate: &nvidiacomv1alpha1.RollingUpdateStatus{
						Phase: nvidiacomv1alpha1.RollingUpdatePhaseInProgress,
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "new-restart-id",
					},
				},
			},
			wantErr: true,
			errMsg:  "spec.restart.id cannot be changed while a rolling update is InProgress",
		},
		{
			name: "restart.id change while rolling update Completed - allowed",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "old-restart-id",
					},
				},
				Status: nvidiacomv1alpha1.DynamoGraphDeploymentStatus{
					RollingUpdate: &nvidiacomv1alpha1.RollingUpdateStatus{
						Phase: nvidiacomv1alpha1.RollingUpdatePhaseCompleted,
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "new-restart-id",
					},
				},
			},
			wantErr: false,
		},
		{
			name: "restart.id change with no rolling update - allowed",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "old-restart-id",
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "new-restart-id",
					},
				},
			},
			wantErr: false,
		},
		{
			name: "spec change without restart.id change during rolling update - allowed",
			oldDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {
							Replicas: func() *int32 { r := int32(1); return &r }(),
						},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "same-restart-id",
					},
				},
				Status: nvidiacomv1alpha1.DynamoGraphDeploymentStatus{
					RollingUpdate: &nvidiacomv1alpha1.RollingUpdateStatus{
						Phase: nvidiacomv1alpha1.RollingUpdatePhaseInProgress,
					},
				},
			},
			newDeployment: &nvidiacomv1alpha1.DynamoGraphDeployment{
				Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{
					BackendFramework: "sglang",
					Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
						"worker": {
							Replicas: func() *int32 { r := int32(3); return &r }(),
						},
					},
					Restart: &nvidiacomv1alpha1.Restart{
						ID: "same-restart-id",
					},
				},
			},
			wantErr: false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			validator := NewDynamoGraphDeploymentValidator(tt.newDeployment)
			// Pass nil userInfo and empty operatorPrincipal - these tests don't modify replicas, so it's safe
			warnings, err := validator.ValidateUpdate(tt.oldDeployment, nil, "")

			if (err != nil) != tt.wantErr {
				t.Errorf("DynamoGraphDeploymentValidator.ValidateUpdate() error = %v, wantErr %v", err, tt.wantErr)
				return
			}

			if tt.wantErr && !strings.Contains(err.Error(), tt.errMsg) {
				t.Errorf("DynamoGraphDeploymentValidator.ValidateUpdate() error message = %v, want to contain %v", err.Error(), tt.errMsg)
			}

			if tt.wantWarnings && len(warnings) == 0 {
				t.Errorf("DynamoGraphDeploymentValidator.ValidateUpdate() expected warnings but got none")
			}

			if tt.wantWarnings && len(warnings) > 0 && warnings[0] != tt.expectedWarnMsg {
				t.Errorf("DynamoGraphDeploymentValidator.ValidateUpdate() warning = %v, want %v", warnings[0], tt.expectedWarnMsg)
			}
		})
	}
}

func TestGetServiceNames(t *testing.T) {
	tests := []struct {
		name     string
		services map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec
		want     map[string]struct{}
	}{
		{
			name:     "empty services",
			services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{},
			want:     map[string]struct{}{},
		},
		{
			name: "single service",
			services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
				"backend": {},
			},
			want: map[string]struct{}{
				"backend": {},
			},
		},
		{
			name: "multiple services",
			services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
				"backend":  {},
				"frontend": {},
				"cache":    {},
			},
			want: map[string]struct{}{
				"backend":  {},
				"frontend": {},
				"cache":    {},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := getServiceNames(tt.services)
			if len(got) != len(tt.want) {
				t.Errorf("getServiceNames() length = %v, want %v", len(got), len(tt.want))
				return
			}
			for name := range tt.want {
				if _, exists := got[name]; !exists {
					t.Errorf("getServiceNames() missing service %v", name)
				}
			}
		})
	}
}

func TestDifference(t *testing.T) {
	tests := []struct {
		name string
		a    map[string]struct{}
		b    map[string]struct{}
		want []string
	}{
		{
			name: "empty sets",
			a:    map[string]struct{}{},
			b:    map[string]struct{}{},
			want: nil,
		},
		{
			name: "a is empty",
			a:    map[string]struct{}{},
			b: map[string]struct{}{
				"backend": {},
			},
			want: nil,
		},
		{
			name: "b is empty",
			a: map[string]struct{}{
				"backend": {},
			},
			b:    map[string]struct{}{},
			want: []string{"backend"},
		},
		{
			name: "no difference - identical sets",
			a: map[string]struct{}{
				"backend":  {},
				"frontend": {},
			},
			b: map[string]struct{}{
				"backend":  {},
				"frontend": {},
			},
			want: nil,
		},
		{
			name: "single element difference",
			a: map[string]struct{}{
				"backend":  {},
				"frontend": {},
			},
			b: map[string]struct{}{
				"backend": {},
			},
			want: []string{"frontend"},
		},
		{
			name: "multiple element difference",
			a: map[string]struct{}{
				"backend":  {},
				"frontend": {},
				"cache":    {},
			},
			b: map[string]struct{}{
				"backend": {},
			},
			want: []string{"cache", "frontend"},
		},
		{
			name: "completely different sets",
			a: map[string]struct{}{
				"frontend": {},
				"cache":    {},
			},
			b: map[string]struct{}{
				"backend": {},
			},
			want: []string{"cache", "frontend"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := difference(tt.a, tt.b)

			// Sort both slices for comparison (since map iteration order is undefined)
			sort.Strings(got)
			want := make([]string, len(tt.want))
			copy(want, tt.want)
			sort.Strings(want)

			if len(got) != len(want) {
				t.Errorf("difference() length = %v, want %v", len(got), len(want))
				return
			}

			for i := range got {
				if got[i] != want[i] {
					t.Errorf("difference() = %v, want %v", got, want)
					return
				}
			}
		})
	}
}