dynamographdeploymentrequest_controller_test.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package controller

import (
	"context"
	"time"

	configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1"
	dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1"
	nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1"
	commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common"
	"github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/tools/record"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

const (
	defaultNamespace = "default"
)

// MockRBACManager implements RBACManager for testing
type MockRBACManager struct {
	EnsureServiceAccountWithRBACFunc func(ctx context.Context, targetNamespace, serviceAccountName, clusterRoleName string) error
}

func (m *MockRBACManager) EnsureServiceAccountWithRBAC(ctx context.Context, targetNamespace, serviceAccountName, clusterRoleName string) error {
	if m.EnsureServiceAccountWithRBACFunc != nil {
		return m.EnsureServiceAccountWithRBACFunc(ctx, targetNamespace, serviceAccountName, clusterRoleName)
	}
	return nil
}

var _ = Describe("DynamoGraphDeploymentRequest Controller", func() {
	const (
		timeout  = time.Second * 10
		interval = time.Millisecond * 250
	)

	var (
		reconciler *DynamoGraphDeploymentRequestReconciler
		recorder   *record.FakeRecorder
	)

	BeforeEach(func() {
		recorder = record.NewFakeRecorder(100)
		reconciler = &DynamoGraphDeploymentRequestReconciler{
			Client:   k8sClient,
			Recorder: recorder,
			Config: &configv1alpha1.OperatorConfiguration{
				Namespace: configv1alpha1.NamespaceConfiguration{
					Restricted: "",
				},
				RBAC: configv1alpha1.RBACConfiguration{
					DGDRProfilingClusterRoleName: "test-cluster-role",
				},
			},
			RuntimeConfig: &commonController.RuntimeConfig{},
			RBACManager:   &MockRBACManager{},
		}
	})

	Context("When reconciling initial DGDR", func() {
		It("Should validate spec and transition to Pending", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-initial"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:     "test-model",
					Backend:   "vllm",
					Image:     "test-profiler:latest",
					AutoApply: ptr.To(true),
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// First reconcile: Empty -> Pending
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      dgdrName,
					Namespace: namespace,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Check status
			Eventually(func() nvidiacomv1beta1.DGDRPhase {
				var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
				_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
				return updated.Status.Phase
			}, timeout, interval).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))

			// Verify observedGeneration is set
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
			Expect(updated.Status.ObservedGeneration).Should(Equal(updated.Generation))
		})

		It("Should pass validation with minimal config", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-minimal"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile - should succeed with minimal config
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      dgdrName,
					Namespace: namespace,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Check status transitions to Pending (not Failed)
			Eventually(func() nvidiacomv1beta1.DGDRPhase {
				var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
				_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
				return updated.Status.Phase
			}, timeout, interval).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))
		})
	})

	Context("When creating profiling job", func() {
		It("Should create online profiling job", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-profiling-online"
			namespace := defaultNamespace

			// Create ConfigMap for DGD base config
			configMap := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-config",
					Namespace: namespace,
				},
				Data: map[string]string{
					"disagg.yaml": "test: config",
				},
			}
			Expect(k8sClient.Create(ctx, configMap)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, configMap) }()

			// Create ServiceAccount
			sa := &corev1.ServiceAccount{
				ObjectMeta: metav1.ObjectMeta{
					Name:      ServiceAccountProfilingJob,
					Namespace: namespace,
				},
			}
			Expect(k8sClient.Create(ctx, sa)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, sa) }()

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
					Annotations: map[string]string{
						"nvidia.com/dgdr-config-map-ref": `{"name":"test-config","key":"disagg.yaml"}`,
					},
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile multiple times to move through states
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Second reconcile: Pending -> Profiling
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify profiling job was created
			Eventually(func() bool {
				jobName := getProfilingJobName(dgdr)
				job := &batchv1.Job{}
				err := k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)
				return err == nil
			}, timeout, interval).Should(BeTrue())

			// Verify job has correct labels
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{}
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)
			Expect(job.Labels[nvidiacomv1beta1.LabelApp]).Should(Equal(nvidiacomv1beta1.LabelValueDynamoProfiler))
			Expect(job.Labels[nvidiacomv1beta1.LabelDGDR]).Should(Equal(dgdrName))

			// Verify job has profiler container
			Expect(job.Spec.Template.Spec.Containers).Should(HaveLen(2))
			Expect(job.Spec.Template.Spec.Containers[0].Name).Should(Equal(ContainerNameProfiler))
			Expect(job.Spec.Template.Spec.Containers[1].Name).Should(Equal(ContainerNameOutputCopier))

			// Verify emptyDir volume (not PVC)
			Expect(job.Spec.Template.Spec.Volumes).Should(ContainElement(
				corev1.Volume{
					Name: VolumeNameProfilingOutput,
					VolumeSource: corev1.VolumeSource{
						EmptyDir: &corev1.EmptyDirVolumeSource{},
					},
				},
			))

			// Clean up job
			_ = k8sClient.Delete(ctx, job)
		})

		It("Should create offline (AIC) profiling job", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-profiling-aic"
			namespace := defaultNamespace

			// Create ServiceAccount
			sa := &corev1.ServiceAccount{
				ObjectMeta: metav1.ObjectMeta{
					Name:      ServiceAccountProfilingJob,
					Namespace: namespace,
				},
			}
			Expect(k8sClient.Create(ctx, sa)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, sa) }()

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:          "test-model",
					Backend:        "trtllm",
					Image:          "test-profiler:latest",
					SearchStrategy: "rapid",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify job was created with AIC label
			Eventually(func() string {
				jobName := getProfilingJobName(dgdr)
				job := &batchv1.Job{}
				if err := k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job); err != nil {
					return ""
				}
				return job.Labels[nvidiacomv1beta1.LabelApp]
			}, timeout, interval).Should(Equal(nvidiacomv1beta1.LabelValueDynamoProfiler))

			// Clean up
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{}
			if err := k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job); err == nil {
				_ = k8sClient.Delete(ctx, job)
			}
		})
	})

	Context("When profiling completes", func() {
		It("Should generate DGD spec from ConfigMap", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-profiling-complete"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Update status to Profiling using Status subresource
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create completed profiling job
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{
				ObjectMeta: metav1.ObjectMeta{
					Name:      jobName,
					Namespace: namespace,
				},
				Spec: batchv1.JobSpec{
					Template: corev1.PodTemplateSpec{
						Spec: corev1.PodSpec{
							Containers: []corev1.Container{{
								Name:  "test",
								Image: "test",
							}},
							RestartPolicy: corev1.RestartPolicyNever,
						},
					},
				},
				Status: batchv1.JobStatus{
					Conditions: []batchv1.JobCondition{{
						Type:   batchv1.JobComplete,
						Status: corev1.ConditionTrue,
					}},
				},
			}
			Expect(k8sClient.Create(ctx, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()

			// Update job status to completed using Status subresource
			job.Status.Conditions = []batchv1.JobCondition{{
				Type:   batchv1.JobComplete,
				Status: corev1.ConditionTrue,
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Create output ConfigMap with DGD spec
			dgdYAML := `apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd
spec:
  services:
    Frontend:
      replicas: 1`

			outputConfigMapName := getOutputConfigMapName(dgdr)
			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      outputConfigMapName,
					Namespace: namespace,
				},
				Data: map[string]string{
					ProfilingOutputFile: dgdYAML,
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Reconcile to process the profiling completion
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Get the updated DGDR
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())

			// Check that DGD spec was generated (stored in annotation)
			Expect(updated.Annotations["nvidia.com/generated-dgd-spec"]).NotTo(BeEmpty())

			// autoApply defaults to true in v1beta1, so after profiling the DGDR transitions to Deploying
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseDeploying))
		})
	})

	Context("When autoApply is enabled", func() {
		It("Should create DGD after profiling", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-autoapply"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
					AutoApply: ptr.To(true),
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Update status to Profiling using Status subresource
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create completed profiling job
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{
				ObjectMeta: metav1.ObjectMeta{
					Name:      jobName,
					Namespace: namespace,
				},
				Spec: batchv1.JobSpec{
					Template: corev1.PodTemplateSpec{
						Spec: corev1.PodSpec{
							Containers: []corev1.Container{{
								Name:  "test",
								Image: "test",
							}},
							RestartPolicy: corev1.RestartPolicyNever,
						},
					},
				},
				Status: batchv1.JobStatus{
					Conditions: []batchv1.JobCondition{{
						Type:   batchv1.JobComplete,
						Status: corev1.ConditionTrue,
					}},
				},
			}
			Expect(k8sClient.Create(ctx, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()

			// Update job status to completed using Status subresource
			job.Status.Conditions = []batchv1.JobCondition{{
				Type:   batchv1.JobComplete,
				Status: corev1.ConditionTrue,
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Create output ConfigMap
			dgdYAML := `apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd-auto
spec:
  services:
    Frontend:
      replicas: 1`

			outputConfigMapName := getOutputConfigMapName(dgdr)
			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      outputConfigMapName,
					Namespace: namespace,
				},
				Data: map[string]string{
					ProfilingOutputFile: dgdYAML,
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Reconcile to generate spec (transitions to Deploying because autoApply=true)
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Get updated DGDR and check state is Deploying
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseDeploying))

			// Reconcile again to create DGD
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify DGD was created
			dgd := &dgdv1alpha1.DynamoGraphDeployment{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "test-dgd-auto", Namespace: namespace}, dgd)).Should(Succeed())

			// Get final DGDR status
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.DGDName).Should(Equal("test-dgd-auto"))

			// Clean up DGD
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "test-dgd-auto", Namespace: namespace}, dgd)).Should(Succeed())
			_ = k8sClient.Delete(ctx, dgd)
		})
	})

	Context("When enforcing spec immutability", func() {
		It("Should reject spec changes after profiling starts", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-immutable"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile to initialize
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Get current generation
			var current nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &current)).Should(Succeed())
			initialGeneration := current.Generation
			observedGeneration := current.Status.ObservedGeneration

			// Manually set state to Profiling to simulate in-progress profiling
			current.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			Expect(k8sClient.Status().Update(ctx, &current)).Should(Succeed())

			// Try to modify spec
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &current)).Should(Succeed())
			current.Spec.Model = "modified-model"
			Expect(k8sClient.Update(ctx, &current)).Should(Succeed())

			// Reconcile
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify generation changed but observedGeneration stayed the same
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &current)).Should(Succeed())
			Expect(current.Generation).Should(BeNumerically(">", initialGeneration))
			Expect(current.Status.ObservedGeneration).Should(Equal(observedGeneration))
			Expect(current.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseProfiling)) // State unchanged

			// Verify event was recorded
			Eventually(func() bool {
				select {
				case event := <-recorder.Events:
					return event == "Warning SpecChangeRejected Cannot modify spec in phase 'Profiling'. DynamoGraphDeploymentRequest is immutable once profiling starts. Create a new resource with a different name instead."
				default:
					return false
				}
			}, timeout, interval).Should(BeTrue())
		})
	})

	Context("When handling DGD deletion", func() {
		It("Should transition to Failed phase when DGD is deleted", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-dgd-deleted"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
					AutoApply: ptr.To(true),
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Update status to Deployed with Deployment info using Status subresource
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseDeployed
			dgdr.Status.DGDName = "test-dgd-to-delete"
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Reconcile when DGD doesn't exist (simulating deletion)
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Get updated DGDR and check phase transitioned to Failed
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseFailed))
		})
	})
})

var _ = Describe("DGDR Helper Functions", func() {
	Context("getProfilingJobName", func() {
		It("Should return correct job name", func() {
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-dgdr",
				},
			}
			Expect(getProfilingJobName(dgdr)).Should(Equal("profile-test-dgdr"))
		})
	})

	Context("getOutputConfigMapName", func() {
		It("Should return correct ConfigMap name", func() {
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-dgdr",
				},
			}
			Expect(getOutputConfigMapName(dgdr)).Should(Equal("dgdr-output-test-dgdr"))
		})
	})

	Context("isOnlineProfiling", func() {
		It("Should always return true regardless of spec", func() {
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
				},
			}
			Expect(isOnlineProfiling(dgdr)).Should(BeTrue())
		})

		It("Should return true with search strategy rapid", func() {
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:          "test-model",
					Backend:        "trtllm",
					SearchStrategy: "rapid",
				},
			}
			Expect(isOnlineProfiling(dgdr)).Should(BeTrue())
		})

		It("Should return true with search strategy thorough", func() {
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:          "test-model",
					Backend:        "vllm",
					SearchStrategy: "thorough",
				},
			}
			Expect(isOnlineProfiling(dgdr)).Should(BeTrue())
		})

		It("Should return true with nil spec fields", func() {
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model: "test-model",
				},
			}
			Expect(isOnlineProfiling(dgdr)).Should(BeTrue())
		})
	})
})

var _ = Describe("DGDR Validation", func() {
	var reconciler *DynamoGraphDeploymentRequestReconciler

	BeforeEach(func() {
		reconciler = &DynamoGraphDeploymentRequestReconciler{
			Client: k8sClient,
		}
	})

	Context("validateSpec", func() {
		It("Should pass validation for valid spec", func() {
			ctx := context.Background()
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			err := reconciler.validateSpec(ctx, dgdr)
			Expect(err).NotTo(HaveOccurred())
		})

		It("Should pass validation with minimal config", func() {
			ctx := context.Background()
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			// Validation should pass - profiler will auto-generate missing config
			err := reconciler.validateSpec(ctx, dgdr)
			Expect(err).NotTo(HaveOccurred())
		})
	})
})

var _ = Describe("DGDR Profiler Arguments", func() {
	var reconciler *DynamoGraphDeploymentRequestReconciler

	BeforeEach(func() {
		reconciler = &DynamoGraphDeploymentRequestReconciler{
			Client:   k8sClient,
			Recorder: record.NewFakeRecorder(100),
			Config: &configv1alpha1.OperatorConfiguration{
				Namespace: configv1alpha1.NamespaceConfiguration{
					Restricted: "",
				},
			},
			RuntimeConfig: &commonController.RuntimeConfig{},
			RBACManager:   &MockRBACManager{},
		}
	})

	Context("When creating profiling job with inline config", func() {
		It("Should pass config as --config argument for online profiling", func() {
			ctx := context.Background()
			namespace := "default"
			dgdrName := "test-args-online"

			// Create ServiceAccount
			sa := &corev1.ServiceAccount{
				ObjectMeta: metav1.ObjectMeta{
					Name:      ServiceAccountProfilingJob,
					Namespace: namespace,
				},
			}
			Expect(k8sClient.Create(ctx, sa)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, sa) }()

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "trtllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH200SXM,
						NumGPUsPerNode: ptr.To[int32](8),
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(50.0),
						ITL:  ptr.To(10.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Re-fetch DGDR to get proper metadata from API server
			var fetchedDGDR nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &fetchedDGDR)).Should(Succeed())

			// Create profiling job with properly initialized DGDR
			err := reconciler.createProfilingJob(ctx, &fetchedDGDR)
			Expect(err).NotTo(HaveOccurred())

			// Verify job was created
			jobName := getProfilingJobName(&fetchedDGDR)
			job := &batchv1.Job{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)).Should(Succeed())

			// Verify profiler container has --config argument
			profilerContainer := job.Spec.Template.Spec.Containers[0]
			args := profilerContainer.Args

			// Check that --config argument is present
			Expect(args).Should(ContainElement("--config"))

			// Clean up
			_ = k8sClient.Delete(ctx, job)
		})

		It("Should pass config with AI Configurator settings for offline profiling", func() {
			ctx := context.Background()
			namespace := defaultNamespace
			dgdrName := "test-args-offline"

			// Create ServiceAccount
			sa := &corev1.ServiceAccount{
				ObjectMeta: metav1.ObjectMeta{
					Name:      ServiceAccountProfilingJob,
					Namespace: namespace,
				},
			}
			Expect(k8sClient.Create(ctx, sa)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, sa) }()

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:          "test-model",
					Backend:        "trtllm",
					Image:          "test-profiler:latest",
					SearchStrategy: "rapid",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH200SXM,
						NumGPUsPerNode: ptr.To[int32](8),
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(50.0),
						ITL:  ptr.To(10.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Re-fetch DGDR to get proper metadata from API server
			var fetchedDGDR nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &fetchedDGDR)).Should(Succeed())

			// Create profiling job with properly initialized DGDR
			err := reconciler.createProfilingJob(ctx, &fetchedDGDR)
			Expect(err).NotTo(HaveOccurred())

			// Verify job was created
			jobName := getProfilingJobName(&fetchedDGDR)
			job := &batchv1.Job{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)).Should(Succeed())

			// Verify profiler container has --config argument
			profilerContainer := job.Spec.Template.Spec.Containers[0]
			args := profilerContainer.Args

			// Check that --config argument is present
			Expect(args).Should(ContainElement("--config"))

			// Clean up
			_ = k8sClient.Delete(ctx, job)
		})

		It("Should set fsGroup in pod security context for volume permissions", func() {
			ctx := context.Background()
			namespace := "default"
			dgdrName := "test-fsgroup"

			// Create ServiceAccount
			sa := &corev1.ServiceAccount{
				ObjectMeta: metav1.ObjectMeta{
					Name:      ServiceAccountProfilingJob,
					Namespace: namespace,
				},
			}
			Expect(k8sClient.Create(ctx, sa)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, sa) }()

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "trtllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(50.0),
						ITL:  ptr.To(10.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Re-fetch DGDR to get proper metadata from API server
			var fetchedDGDR nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &fetchedDGDR)).Should(Succeed())

			// Create profiling job with properly initialized DGDR
			err := reconciler.createProfilingJob(ctx, &fetchedDGDR)
			Expect(err).NotTo(HaveOccurred())

			// Verify job was created
			jobName := getProfilingJobName(&fetchedDGDR)
			job := &batchv1.Job{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)).Should(Succeed())

			// Verify security context has all security fields set correctly
			podSecurityContext := job.Spec.Template.Spec.SecurityContext
			Expect(podSecurityContext).NotTo(BeNil())
			Expect(podSecurityContext.RunAsNonRoot).NotTo(BeNil())
			Expect(*podSecurityContext.RunAsNonRoot).To(BeTrue())
			Expect(podSecurityContext.RunAsUser).NotTo(BeNil())
			Expect(*podSecurityContext.RunAsUser).To(Equal(int64(1000)))
			Expect(podSecurityContext.RunAsGroup).NotTo(BeNil())
			Expect(*podSecurityContext.RunAsGroup).To(Equal(int64(1000)))
			Expect(podSecurityContext.FSGroup).NotTo(BeNil())
			Expect(*podSecurityContext.FSGroup).To(Equal(int64(1000)))

			// Clean up
			_ = k8sClient.Delete(ctx, job)
		})
	})
})

var _ = Describe("DGDR Error Handling", func() {
	var reconciler *DynamoGraphDeploymentRequestReconciler
	var recorder *record.FakeRecorder

	BeforeEach(func() {
		recorder = record.NewFakeRecorder(100)
		reconciler = &DynamoGraphDeploymentRequestReconciler{
			Client:    k8sClient,
			APIReader: k8sClient,
			Recorder:  recorder,
			Config: &configv1alpha1.OperatorConfiguration{
				Namespace: configv1alpha1.NamespaceConfiguration{
					Restricted: "",
				},
			},
			RuntimeConfig: &commonController.RuntimeConfig{},
			RBACManager:   &MockRBACManager{},
		}
	})

	Context("When profiling job fails", func() {
		It("Should capture detailed error from pod termination state", func() {
			ctx := context.Background()
			namespace := defaultNamespace
			dgdrName := "test-error-capture"

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Set status to Profiling
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create failed job
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{
				ObjectMeta: metav1.ObjectMeta{
					Name:      jobName,
					Namespace: namespace,
				},
				Spec: batchv1.JobSpec{
					Template: corev1.PodTemplateSpec{
						Spec: corev1.PodSpec{
							Containers: []corev1.Container{{
								Name:  ContainerNameProfiler,
								Image: "test",
							}},
							RestartPolicy: corev1.RestartPolicyNever,
						},
					},
				},
				Status: batchv1.JobStatus{
					Conditions: []batchv1.JobCondition{{
						Type:    batchv1.JobFailed,
						Status:  corev1.ConditionTrue,
						Message: "BackoffLimitExceeded",
					}},
				},
			}
			Expect(k8sClient.Create(ctx, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()

			// Update job status
			job.Status.Conditions = []batchv1.JobCondition{{
				Type:    batchv1.JobFailed,
				Status:  corev1.ConditionTrue,
				Message: "BackoffLimitExceeded",
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Create failed pod with termination details
			pod := &corev1.Pod{
				ObjectMeta: metav1.ObjectMeta{
					Name:      jobName + "-pod",
					Namespace: namespace,
					Labels: map[string]string{
						"job-name": jobName,
					},
				},
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{{
						Name:  ContainerNameProfiler,
						Image: "test",
					}},
					RestartPolicy: corev1.RestartPolicyNever,
				},
				Status: corev1.PodStatus{
					Phase: corev1.PodFailed,
					ContainerStatuses: []corev1.ContainerStatus{{
						Name: ContainerNameProfiler,
						State: corev1.ContainerState{
							Terminated: &corev1.ContainerStateTerminated{
								ExitCode: 1,
								Reason:   "Error",
								Message:  "ValueError: Invalid model name for AI Configurator",
							},
						},
					}},
				},
			}
			Expect(k8sClient.Create(ctx, pod)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, pod) }()

			// Reconcile - should capture error details
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify DGDR transitioned to Failed state
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseFailed))

			// Verify error condition contains detailed error
			condition := meta.FindStatusCondition(updated.Status.Conditions, nvidiacomv1beta1.ConditionTypeProfiling)
			Expect(condition).NotTo(BeNil())
			Expect(condition.Status).Should(Equal(metav1.ConditionFalse))
			Expect(condition.Message).Should(ContainSubstring("profiling job failed"))
		})
	})

	Context("When parsing multi-document YAML", func() {
		It("Should extract DGD from ConfigMap + DGD YAML", func() {
			// Multi-document YAML with ConfigMap first, then DGD
			multiDocYAML := `---
apiVersion: v1
kind: ConfigMap
metadata:
  name: test-config
  namespace: default
data:
  some-data: "value"
---
apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd
  namespace: default
spec:
  backendFramework: vllm
  services: {}`

			dgd, err := reconciler.extractDGDFromYAML([]byte(multiDocYAML))
			Expect(err).NotTo(HaveOccurred())
			Expect(dgd).NotTo(BeNil())
			Expect(dgd.Kind).Should(Equal("DynamoGraphDeployment"))
			Expect(dgd.Name).Should(Equal("test-dgd"))
			Expect(dgd.Spec.BackendFramework).Should(Equal("vllm"))
		})

		It("Should extract DGD from single-document YAML", func() {
			// Single document YAML without separator
			singleDocYAML := `apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd-single
  namespace: default
spec:
  backendFramework: vllm
  services: {}`

			dgd, err := reconciler.extractDGDFromYAML([]byte(singleDocYAML))
			Expect(err).NotTo(HaveOccurred())
			Expect(dgd).NotTo(BeNil())
			Expect(dgd.Kind).Should(Equal("DynamoGraphDeployment"))
			Expect(dgd.Name).Should(Equal("test-dgd-single"))
		})

		It("Should handle DGD + ConfigMap order (DGD first)", func() {
			// Multi-document YAML with DGD first, then ConfigMap
			multiDocYAML := `---
apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd-first
  namespace: default
spec:
  backendFramework: vllm
  services: {}
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: test-config
  namespace: default
data:
  some-data: "value"`

			dgd, err := reconciler.extractDGDFromYAML([]byte(multiDocYAML))
			Expect(err).NotTo(HaveOccurred())
			Expect(dgd).NotTo(BeNil())
			Expect(dgd.Kind).Should(Equal("DynamoGraphDeployment"))
			Expect(dgd.Name).Should(Equal("test-dgd-first"))
		})

		It("Should return error when no DGD found", func() {
			// YAML with only ConfigMap
			configMapOnlyYAML := `---
apiVersion: v1
kind: ConfigMap
metadata:
  name: test-config
  namespace: default
data:
  some-data: "value"`

			_, err := reconciler.extractDGDFromYAML([]byte(configMapOnlyYAML))
			Expect(err).To(HaveOccurred())
			Expect(err.Error()).Should(ContainSubstring("no DynamoGraphDeployment found"))
		})

		It("Should handle YAML with leading separator", func() {
			// YAML starting with --- separator
			yamlWithLeadingSeparator := `---
apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd-leading
  namespace: default
spec:
  backendFramework: vllm
  services: {}`

			dgd, err := reconciler.extractDGDFromYAML([]byte(yamlWithLeadingSeparator))
			Expect(err).NotTo(HaveOccurred())
			Expect(dgd).NotTo(BeNil())
			Expect(dgd.Name).Should(Equal("test-dgd-leading"))
		})

		It("Should extract DGD and additional resources correctly", func() {
			// Multi-document YAML with ConfigMap and DGD
			multiDocYAML := `---
apiVersion: v1
kind: ConfigMap
metadata:
  name: model-config
  namespace: default
data:
  model.json: '{"name": "test-model"}'
---
apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd
  namespace: default
spec:
  backendFramework: vllm
  services: {}`

			dgd, additionalResources, err := reconciler.extractResourcesFromYAML([]byte(multiDocYAML))
			Expect(err).NotTo(HaveOccurred())
			Expect(dgd).NotTo(BeNil())
			Expect(dgd.Name).Should(Equal("test-dgd"))
			Expect(additionalResources).To(HaveLen(1))
			Expect(additionalResources[0].GetKind()).Should(Equal("ConfigMap"))
			Expect(additionalResources[0].GetName()).Should(Equal("model-config"))
		})

		It("Should handle multiple additional resources", func() {
			// Multi-document YAML with multiple ConfigMaps and DGD
			multiDocYAML := `---
apiVersion: v1
kind: ConfigMap
metadata:
  name: config1
data:
  key1: value1
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: config2
data:
  key2: value2
---
apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd
spec:
  backendFramework: vllm
  services: {}`

			dgd, additionalResources, err := reconciler.extractResourcesFromYAML([]byte(multiDocYAML))
			Expect(err).NotTo(HaveOccurred())
			Expect(dgd).NotTo(BeNil())
			Expect(additionalResources).To(HaveLen(2))
			Expect(additionalResources[0].GetName()).Should(Equal("config1"))
			Expect(additionalResources[1].GetName()).Should(Equal("config2"))
		})
	})

	Context("GPU Discovery Integration Tests", func() {
		It("Should use GPU discovery when nodes have GPU labels", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-gpu-discovery"
			namespace := defaultNamespace

			// Create a node with GPU labels (simulating GFD labels)
			gpuNode := &corev1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Name: "gpu-worker-1",
					Labels: map[string]string{
						"nvidia.com/gpu.count":   "8",
						"nvidia.com/gpu.product": "h100_sxm",
						"nvidia.com/gpu.memory":  "81920",
					},
				},
			}
			Expect(k8sClient.Create(ctx, gpuNode)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, gpuNode) }()

			// Create DGDR WITHOUT hardware config (should use GPU discovery)
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			mockGPU := &gpu.GPUInfo{
				GPUsPerNode:   8,
				VRAMPerGPU:    81920,
				System:        "h100_sxm",
				NodesWithGPUs: 1,
			}
			cache := gpu.NewGPUDiscoveryCache()
			cache.Set(mockGPU, 10*time.Minute)
			reconciler.GPUDiscoveryCache = cache
			reconciler.GPUDiscovery = gpu.NewGPUDiscovery(nil)
			reconciler.APIReader = k8sClient

			// Reconcile - should succeed with GPU discovery
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      dgdrName,
					Namespace: namespace,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Should transition to Pending (validation passed)
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))
		})

		It("Should respect manual hardware config over GPU discovery", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-manual-override"
			namespace := defaultNamespace

			// Create a node with H100 GPUs
			gpuNode := &corev1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Name: "gpu-worker-h100",
					Labels: map[string]string{
						"nvidia.com/gpu.count":   "8",
						"nvidia.com/gpu.product": "h100_sxm",
						"nvidia.com/gpu.memory":  "81920",
					},
				},
			}
			Expect(k8sClient.Create(ctx, gpuNode)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, gpuNode) }()

			// Create DGDR WITH manual hardware config (A100, not H100)
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](4),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeA100SXM,
						VRAMMB:         ptr.To(40960.0),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile - should succeed and use manual config
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      dgdrName,
					Namespace: namespace,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Should transition to Pending (validation passed with manual config)
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))
		})

		It("Should succeed with GPU discovery when cluster has GPU nodes", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-with-autodiscovery"
			namespace := defaultNamespace

			// Create a GPU node so GPU discovery can succeed
			node := &corev1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Name: "gpu-worker-autodiscovery",
					Labels: map[string]string{
						"nvidia.com/gpu.count":   "8",
						"nvidia.com/gpu.product": "h100_sxm",
						"nvidia.com/gpu.memory":  "81920",
					},
				},
			}
			Expect(k8sClient.Create(ctx, node)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, node) }()

			// Create DGDR WITHOUT hardware config - should use GPU discovery
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			mockGPU := &gpu.GPUInfo{
				GPUsPerNode:   8,
				VRAMPerGPU:    81920,
				System:        "h100_sxm",
				NodesWithGPUs: 1,
			}
			cache := gpu.NewGPUDiscoveryCache()
			cache.Set(mockGPU, 10*time.Minute)
			reconciler.GPUDiscoveryCache = cache
			reconciler.GPUDiscovery = gpu.NewGPUDiscovery(nil)
			reconciler.APIReader = k8sClient

			// Reconcile - should succeed with GPU discovery
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      dgdrName,
					Namespace: namespace,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Should transition to Pending
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))
		})

		It("Should pass validation with explicit GPU ranges without GPU discovery", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-explicit-ranges"
			namespace := defaultNamespace

			// Intentionally don't create GPU nodes to test that explicit ranges work without GPU discovery
			// Create DGDR with explicit minNumGpusPerEngine/maxNumGpusPerEngine
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile - should succeed (explicit ranges + minimal hardware bypass GPU discovery requirement)
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      dgdrName,
					Namespace: namespace,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Should transition to Pending
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))
		})

		It("Should use GPU discovery with heterogeneous nodes (picks best)", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-heterogeneous"
			namespace := defaultNamespace

			// Create nodes with different GPU configs
			nodeA100 := &corev1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Name: "gpu-worker-a100",
					Labels: map[string]string{
						"nvidia.com/gpu.count":   "4",
						"nvidia.com/gpu.product": "A100-SXM4-40GB",
						"nvidia.com/gpu.memory":  "40960",
					},
				},
			}
			nodeH100 := &corev1.Node{
				ObjectMeta: metav1.ObjectMeta{
					Name: "gpu-worker-h100",
					Labels: map[string]string{
						"nvidia.com/gpu.count":   "8",
						"nvidia.com/gpu.product": "h100_sxm",
						"nvidia.com/gpu.memory":  "81920",
					},
				},
			}
			Expect(k8sClient.Create(ctx, nodeA100)).Should(Succeed())
			Expect(k8sClient.Create(ctx, nodeH100)).Should(Succeed())
			defer func() {
				_ = k8sClient.Delete(ctx, nodeA100)
				_ = k8sClient.Delete(ctx, nodeH100)
			}()

			// Create DGDR without hardware config
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			mockGPU := &gpu.GPUInfo{
				GPUsPerNode:   8,
				VRAMPerGPU:    81920,
				System:        "h100_sxm",
				NodesWithGPUs: 1,
			}
			cache := gpu.NewGPUDiscoveryCache()
			cache.Set(mockGPU, 10*time.Minute)
			reconciler.GPUDiscoveryCache = cache
			reconciler.GPUDiscovery = gpu.NewGPUDiscovery(nil)
			reconciler.APIReader = k8sClient
			// Reconcile - should pick H100 (8 GPUs > 4 GPUs)
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{
					Name:      dgdrName,
					Namespace: namespace,
				},
			})
			Expect(err).NotTo(HaveOccurred())

			// Should transition to Pending (using H100 config)
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))
		})
	})

	Context("v1beta1-specific behavior", func() {
		It("Should transition to Deployed when DGD reaches Ready", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-deployed-phase"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Set DGDR to Deploying with a DGDName
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseDeploying
			dgdr.Status.DGDName = "test-dgd-deployed"
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create the DGD in Ready state
			dgd := &dgdv1alpha1.DynamoGraphDeployment{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test-dgd-deployed",
					Namespace: namespace,
					Labels: map[string]string{
						nvidiacomv1beta1.LabelDGDRName:      dgdrName,
						nvidiacomv1beta1.LabelDGDRNamespace: namespace,
					},
				},
				Spec: dgdv1alpha1.DynamoGraphDeploymentSpec{},
			}
			Expect(k8sClient.Create(ctx, dgd)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgd) }()

			// Set DGD to Successful state
			dgd.Status.State = dgdv1alpha1.DGDStateSuccessful
			Expect(k8sClient.Status().Update(ctx, dgd)).Should(Succeed())

			// Reconcile — should transition DGDR to Deployed
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseDeployed))
		})

		It("Should set Succeeded condition at each phase transition", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-succeeded-cond"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// First reconcile: initial validation → Pending
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())

			// Check that Succeeded condition exists with reason matching the phase
			succeededCond := meta.FindStatusCondition(updated.Status.Conditions, nvidiacomv1beta1.ConditionTypeSucceeded)
			Expect(succeededCond).NotTo(BeNil())
			Expect(succeededCond.Reason).Should(Equal(string(nvidiacomv1beta1.DGDRPhasePending)))
		})

		It("Should set ProfilingPhase on entry to Profiling and clear on exit", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-profiling-phase"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Transition through initial validation to Pending
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Reconcile again to start profiling (creates job, transitions to Profiling)
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Check ProfilingPhase is set
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseProfiling))
			Expect(updated.Status.ProfilingPhase).Should(Equal(nvidiacomv1beta1.ProfilingPhaseInitializing))

			// Simulate profiling completion
			jobName := getProfilingJobName(&updated)
			job := &batchv1.Job{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)).Should(Succeed())
			job.Status.Conditions = []batchv1.JobCondition{{
				Type:   batchv1.JobComplete,
				Status: corev1.ConditionTrue,
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			dgdYAML := `apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd-profphase
spec:
  services:
    Frontend:
      replicas: 1`

			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      getOutputConfigMapName(&updated),
					Namespace: namespace,
				},
				Data: map[string]string{
					ProfilingOutputFile: dgdYAML,
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Reconcile to complete profiling
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.ProfilingPhase).Should(BeEmpty(), "profilingPhase must be cleared after profiling completes")
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseDeploying),
				"phase must be Deploying after profiling completes with autoApply=true")
		})

		It("Should use spec.features.mocker.enabled to select mocker output", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-mocker"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:     "test-model",
					Backend:   "vllm",
					Image:     "test-profiler:latest",
					AutoApply: ptr.To(true),
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
					Features: &nvidiacomv1beta1.FeaturesSpec{
						Mocker: &nvidiacomv1beta1.MockerSpec{Enabled: true},
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Transition to Profiling
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			dgdr.Status.ObservedGeneration = dgdr.Generation
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create completed job
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{
				ObjectMeta: metav1.ObjectMeta{Name: jobName, Namespace: namespace},
				Spec: batchv1.JobSpec{
					Template: corev1.PodTemplateSpec{
						Spec: corev1.PodSpec{
							Containers:    []corev1.Container{{Name: "test", Image: "test"}},
							RestartPolicy: corev1.RestartPolicyNever,
						},
					},
				},
			}
			Expect(k8sClient.Create(ctx, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()

			job.Status.Conditions = []batchv1.JobCondition{{
				Type: batchv1.JobComplete, Status: corev1.ConditionTrue,
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Create output ConfigMap with mocker output file
			dgdYAML := `apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-dgd-mocker
spec:
  services:
    Frontend:
      replicas: 1`

			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      getOutputConfigMapName(dgdr),
					Namespace: namespace,
				},
				Data: map[string]string{
					ProfilingOutputFile: dgdYAML,
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Reconcile — should read from mocker output file
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify the generated spec came from the mocker file
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Annotations["nvidia.com/generated-dgd-spec"]).Should(ContainSubstring("test-dgd-mocker"))
		})

		It("Should populate profilingJobName in status", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-jobname"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile through initial validation
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Reconcile to create profiling job
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Check profilingJobName is set in status
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.ProfilingJobName).Should(Equal(getProfilingJobName(&updated)))

			// Clean up job
			job := &batchv1.Job{}
			_ = k8sClient.Get(ctx, types.NamespacedName{Name: updated.Status.ProfilingJobName, Namespace: namespace}, job)
			_ = k8sClient.Delete(ctx, job)
		})

		It("Should clear profilingPhase, set ProfilingCompleted condition, and populate profilingResults.selectedConfig after profiling completes", func() {
			// Regression test for: profilingPhase staying "Initializing", Profiling condition
			// not updated to ProfilingCompleted, and profilingResults never populated.
			// Root cause was that generateDGDSpec called r.Get() before r.Status().Update(),
			// overwriting the in-memory status changes made by handleProfilingPhase.
			ctx := context.Background()
			dgdrName := "test-dgdr-status-regression"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:     "test-model",
					Backend:   "vllm",
					Image:     "test-profiler:latest",
					AutoApply: ptr.To(false),
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](8),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(200.0),
						ITL:  ptr.To(30.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile: initial validation → Pending
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Reconcile: create profiling job → Profiling + ProfilingPhase=Initializing
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseProfiling))
			Expect(updated.Status.ProfilingPhase).Should(Equal(nvidiacomv1beta1.ProfilingPhaseInitializing))

			// Mark profiling job as complete
			jobName := getProfilingJobName(&updated)
			job := &batchv1.Job{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()
			job.Status.Conditions = []batchv1.JobCondition{{
				Type:   batchv1.JobComplete,
				Status: corev1.ConditionTrue,
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Create the output ConfigMap that the sidecar would have created
			dgdYAML := `apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-status-regression-dgd
spec:
  services:
    Frontend:
      replicas: 1
    VllmWorker:
      replicas: 2`

			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      getOutputConfigMapName(&updated),
					Namespace: namespace,
				},
				Data: map[string]string{
					ProfilingOutputFile: dgdYAML,
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Reconcile: profiling complete → should clear profilingPhase, set
			// ProfilingCompleted condition, populate profilingResults.selectedConfig
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())

			// profilingPhase must be cleared (was staying "Initializing" before fix)
			Expect(updated.Status.ProfilingPhase).Should(BeEmpty(),
				"profilingPhase should be cleared after profiling completes")

			// Profiling condition must be ProfilingCompleted (was never set before fix)
			profilingCond := meta.FindStatusCondition(updated.Status.Conditions, nvidiacomv1beta1.ConditionTypeProfiling)
			Expect(profilingCond).ShouldNot(BeNil(), "Profiling condition must exist")
			Expect(profilingCond.Status).Should(Equal(metav1.ConditionTrue),
				"Profiling condition status must be True")
			Expect(profilingCond.Reason).Should(Equal("ProfilingCompleted"),
				"Profiling condition reason must be ProfilingCompleted")

			// profilingResults.selectedConfig must be populated (was nil before fix)
			Expect(updated.Status.ProfilingResults).ShouldNot(BeNil(),
				"profilingResults must be populated after profiling")
			Expect(updated.Status.ProfilingResults.SelectedConfig).ShouldNot(BeNil(),
				"profilingResults.selectedConfig must be set")
			Expect(updated.Status.ProfilingResults.SelectedConfig.Raw).ShouldNot(BeEmpty(),
				"profilingResults.selectedConfig must not be empty JSON")

			// Phase must be Ready (autoApply=false → profiling complete, spec available)
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseReady),
				"phase must be Ready after profiling completes with autoApply=false")

			// status.dgdName must be preserved after profiling
			Expect(updated.Status.DGDName).ShouldNot(BeEmpty(), "status.dgdName must be preserved after profiling")
		})

		It("Should populate profilingResults.pareto from webui_data.json in output ConfigMap", func() {
			// Regression test for profilingResults.pareto never being populated.
			// The sidecar now includes webui_data.json in the output ConfigMap;
			// the controller reads it to populate status.profilingResults.pareto.
			ctx := context.Background()
			dgdrName := "test-dgdr-pareto"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:     "test-model",
					Backend:   "vllm",
					Image:     "test-profiler:latest",
					AutoApply: ptr.To(false),
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         nvidiacomv1beta1.GPUSKUTypeH100SXM,
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](8),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(200.0),
						ITL:  ptr.To(30.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Drive to Profiling phase
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseProfiling))

			// Mark job complete
			jobName := getProfilingJobName(&updated)
			job := &batchv1.Job{}
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()
			job.Status.Conditions = []batchv1.JobCondition{{
				Type:   batchv1.JobComplete,
				Status: corev1.ConditionTrue,
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Create output ConfigMap that includes both final_config.yaml and webui_data.json.
			// The webui_data.json format mirrors what the profiler writes; two rows in cost.table
			// represent two Pareto-optimal configurations.
			dgdYAML := `apiVersion: nvidia.com/v1alpha1
kind: DynamoGraphDeployment
metadata:
  name: test-pareto-dgd
spec:
  services:
    Frontend:
      replicas: 1`

			webUIDataJSON := `{
  "settings": {},
  "prefill": {"chart": {}, "table": {"columns": [], "data": []}},
  "decode":  {"chart": {}, "table": {"columns": [], "data": []}},
  "cost": {
    "chart": {},
    "index_mapping": {"0": [0, 0], "1": [0, 1]},
    "table": {
      "columns": ["TTFT (ms)", "Prefill Thpt", "ITL (ms)", "Decode Thpt", "Tokens/User", "GPU Hours", "Action"],
      "data": [
        [94.38, 7946.85, 7.09, 35.26, 141.04, 16.17,
          "# Prefill: 4 GPU(s), TP=4\n# Decode: 4 GPU(s), TP=4\napiVersion: nvidia.com/v1alpha1\nkind: DynamoGraphDeployment\nspec:\n  services:\n    PrefillWorker:\n      replicas: 1\n    DecodeWorker:\n      replicas: 1\n"],
        [94.38, 7946.85, 10.69, 46.77, 93.54, 6.36,
          "# Prefill: 4 GPU(s), TP=4\n# Decode: 2 GPU(s), TP=2\napiVersion: nvidia.com/v1alpha1\nkind: DynamoGraphDeployment\nspec:\n  services:\n    PrefillWorker:\n      replicas: 1\n    DecodeWorker:\n      replicas: 1\n"]
      ]
    }
  }
}`

			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      getOutputConfigMapName(&updated),
					Namespace: namespace,
				},
				Data: map[string]string{
					ProfilingOutputFile: dgdYAML,
					"webui_data.json":   webUIDataJSON,
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Reconcile to complete profiling and populate profilingResults
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())

			// profilingResults.pareto must contain both Pareto-optimal configurations
			Expect(updated.Status.ProfilingResults).ShouldNot(BeNil(),
				"profilingResults must be populated")
			Expect(updated.Status.ProfilingResults.Pareto).Should(HaveLen(2),
				"profilingResults.pareto should contain 2 entries from webui_data.json")

			// Each pareto entry must have non-empty Config
			for i, p := range updated.Status.ProfilingResults.Pareto {
				Expect(p.Config.Raw).ShouldNot(BeEmpty(),
					"pareto[%d].config must not be empty", i)
			}
		})

		It("Should validate typed hardware fields without blob parsing", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-typed-hw"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						GPUSKU: nvidiacomv1beta1.GPUSKUTypeA100SXM,
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Reconcile — partial hardware (GPUSKU only) should pass validation
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhasePending))
		})
	})
})

var _ = Describe("DGDR Profiling Phase Derivation Functions", func() {
	Context("profilingPhaseReason", func() {
		It("Should return phase string as reason (they are identical by design)", func() {
			tests := []struct {
				phase    nvidiacomv1beta1.ProfilingPhase
				expected string
			}{
				{nvidiacomv1beta1.ProfilingPhaseInitializing, "Initializing"},
				{nvidiacomv1beta1.ProfilingPhaseSweepingPrefill, "SweepingPrefill"},
				{nvidiacomv1beta1.ProfilingPhaseSweepingDecode, "SweepingDecode"},
				{nvidiacomv1beta1.ProfilingPhaseSelectingConfig, "SelectingConfig"},
				{nvidiacomv1beta1.ProfilingPhaseBuildingCurves, "BuildingCurves"},
				{nvidiacomv1beta1.ProfilingPhaseGeneratingDGD, "GeneratingDGD"},
			}
			for _, tt := range tests {
				Expect(profilingPhaseReason(tt.phase)).Should(Equal(tt.expected))
			}
		})

		It("Should return Completed for Done phase", func() {
			Expect(profilingPhaseReason(nvidiacomv1beta1.ProfilingPhaseDone)).Should(Equal(nvidiacomv1beta1.ProfilingReasonCompleted))
		})

		It("Should pass through unrecognized phases as-is", func() {
			Expect(profilingPhaseReason(nvidiacomv1beta1.ProfilingPhase("CustomPhase"))).Should(Equal("CustomPhase"))
		})
	})

	Context("profilingPhaseFailureReason", func() {
		It("Should derive failure reason as phase + Failed", func() {
			tests := []struct {
				phase    nvidiacomv1beta1.ProfilingPhase
				expected string
			}{
				{nvidiacomv1beta1.ProfilingPhaseInitializing, "InitializingFailed"},
				{nvidiacomv1beta1.ProfilingPhaseSweepingPrefill, "SweepingPrefillFailed"},
				{nvidiacomv1beta1.ProfilingPhaseSweepingDecode, "SweepingDecodeFailed"},
				{nvidiacomv1beta1.ProfilingPhaseSelectingConfig, "SelectingConfigFailed"},
				{nvidiacomv1beta1.ProfilingPhaseBuildingCurves, "BuildingCurvesFailed"},
				{nvidiacomv1beta1.ProfilingPhaseGeneratingDGD, "GeneratingDGDFailed"},
				{nvidiacomv1beta1.ProfilingPhaseDone, "DoneFailed"},
			}
			for _, tt := range tests {
				Expect(profilingPhaseFailureReason(tt.phase)).Should(Equal(tt.expected))
			}
		})

		It("Should return generic ProfilingFailed for empty phase", func() {
			Expect(profilingPhaseFailureReason(nvidiacomv1beta1.ProfilingPhase(""))).Should(Equal("ProfilingFailed"))
		})
	})
})

var _ = Describe("DGDR Output ConfigMap Naming", func() {
	Context("getOutputConfigMapName", func() {
		It("Should use ConfigMapOutputPrefix", func() {
			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name: "my-deploy",
				},
			}
			name := getOutputConfigMapName(dgdr)
			Expect(name).Should(HavePrefix(ConfigMapOutputPrefix))
			Expect(name).Should(Equal("dgdr-output-my-deploy"))
		})
	})
})

var _ = Describe("DGDR Profiling Failure Attribution", func() {
	var (
		reconciler *DynamoGraphDeploymentRequestReconciler
		recorder   *record.FakeRecorder
	)

	BeforeEach(func() {
		recorder = record.NewFakeRecorder(100)
		reconciler = &DynamoGraphDeploymentRequestReconciler{
			Client:    k8sClient,
			APIReader: k8sClient,
			Recorder:  recorder,
			Config: &configv1alpha1.OperatorConfiguration{
				Namespace: configv1alpha1.NamespaceConfiguration{
					Restricted: "",
				},
			},
			RuntimeConfig: &commonController.RuntimeConfig{},
			RBACManager:   &MockRBACManager{},
		}
	})

	Context("Profiling failure keeps profilingPhase", func() {
		It("Should preserve profilingPhase and use sub-phase failure reason on job failure", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-keep-phase"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         "h100_sxm",
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Set status to Profiling with SweepingDecode sub-phase
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			dgdr.Status.ProfilingPhase = nvidiacomv1beta1.ProfilingPhaseSweepingDecode
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create failed job
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{
				ObjectMeta: metav1.ObjectMeta{
					Name:      jobName,
					Namespace: namespace,
				},
				Spec: batchv1.JobSpec{
					Template: corev1.PodTemplateSpec{
						Spec: corev1.PodSpec{
							Containers: []corev1.Container{{
								Name:  ContainerNameProfiler,
								Image: "test",
							}},
							RestartPolicy: corev1.RestartPolicyNever,
						},
					},
				},
			}
			Expect(k8sClient.Create(ctx, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()

			// Update job status to failed
			job.Status.Conditions = []batchv1.JobCondition{{
				Type:    batchv1.JobFailed,
				Status:  corev1.ConditionTrue,
				Message: "BackoffLimitExceeded",
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Reconcile
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify DGDR is in Failed phase with profilingPhase preserved
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseFailed))
			Expect(updated.Status.ProfilingPhase).Should(Equal(nvidiacomv1beta1.ProfilingPhaseSweepingDecode))

			// Verify Profiling condition has sub-phase-specific failure reason
			profilingCond := meta.FindStatusCondition(updated.Status.Conditions, nvidiacomv1beta1.ConditionTypeProfiling)
			Expect(profilingCond).NotTo(BeNil())
			Expect(profilingCond.Reason).Should(Equal(nvidiacomv1beta1.ProfilingReasonSweepingDecodeFailed))

			// Verify Succeeded condition has sub-phase-specific failure reason
			succeededCond := meta.FindStatusCondition(updated.Status.Conditions, nvidiacomv1beta1.ConditionTypeSucceeded)
			Expect(succeededCond).NotTo(BeNil())
			Expect(succeededCond.Reason).Should(Equal(nvidiacomv1beta1.ProfilingReasonSweepingDecodeFailed))
		})

		It("Should use generic ProfilingFailed when no sub-phase info available", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-generic-fail"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         "h100_sxm",
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Set status to Profiling with empty sub-phase
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			dgdr.Status.ProfilingPhase = ""
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create failed job
			jobName := getProfilingJobName(dgdr)
			job := &batchv1.Job{
				ObjectMeta: metav1.ObjectMeta{
					Name:      jobName,
					Namespace: namespace,
				},
				Spec: batchv1.JobSpec{
					Template: corev1.PodTemplateSpec{
						Spec: corev1.PodSpec{
							Containers: []corev1.Container{{
								Name:  ContainerNameProfiler,
								Image: "test",
							}},
							RestartPolicy: corev1.RestartPolicyNever,
						},
					},
				},
			}
			Expect(k8sClient.Create(ctx, job)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, job) }()

			job.Status.Conditions = []batchv1.JobCondition{{
				Type:    batchv1.JobFailed,
				Status:  corev1.ConditionTrue,
				Message: "BackoffLimitExceeded",
			}}
			Expect(k8sClient.Status().Update(ctx, job)).Should(Succeed())

			// Reconcile
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify generic ProfilingFailed is used
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseFailed))

			profilingCond := meta.FindStatusCondition(updated.Status.Conditions, nvidiacomv1beta1.ConditionTypeProfiling)
			Expect(profilingCond).NotTo(BeNil())
			Expect(profilingCond.Reason).Should(Equal("ProfilingFailed"))
		})
	})

	Context("Profiling entry uses Initializing reason", func() {
		It("Should use Initializing reason when entering Profiling phase", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-init-reason"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         "h100_sxm",
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// First reconcile: validation → Pending
			_, err := reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Second reconcile: Pending → Profiling
			_, err = reconciler.Reconcile(ctx, reconcile.Request{
				NamespacedName: types.NamespacedName{Name: dgdrName, Namespace: namespace},
			})
			Expect(err).NotTo(HaveOccurred())

			// Verify Profiling condition uses Initializing reason (not generic ProfilingRunning)
			var updated nvidiacomv1beta1.DynamoGraphDeploymentRequest
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, &updated)).Should(Succeed())
			Expect(updated.Status.Phase).Should(Equal(nvidiacomv1beta1.DGDRPhaseProfiling))

			profilingCond := meta.FindStatusCondition(updated.Status.Conditions, nvidiacomv1beta1.ConditionTypeProfiling)
			Expect(profilingCond).NotTo(BeNil())
			Expect(profilingCond.Reason).Should(Equal(nvidiacomv1beta1.ProfilingReasonInitializing))

			// Clean up job
			jobName := getProfilingJobName(&updated)
			job := &batchv1.Job{}
			if err := k8sClient.Get(ctx, types.NamespacedName{Name: jobName, Namespace: namespace}, job); err == nil {
				_ = k8sClient.Delete(ctx, job)
			}
		})
	})

	Context("updateProfilingSubPhase", func() {
		It("Should update profilingPhase from output ConfigMap", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-subphase-update"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         "h100_sxm",
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			// Set initial status
			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			dgdr.Status.ProfilingPhase = nvidiacomv1beta1.ProfilingPhaseInitializing
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create output ConfigMap with updated phase and message from profiler
			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      getOutputConfigMapName(dgdr),
					Namespace: namespace,
				},
				Data: map[string]string{
					"phase":   "SweepingPrefill",
					"message": "Sweeping TP=4 DEP=2, measuring TTFT",
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Re-fetch to get latest resourceVersion
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, dgdr)).Should(Succeed())

			// Call updateProfilingSubPhase
			Expect(reconciler.updateProfilingSubPhase(ctx, dgdr)).Should(Succeed())

			// Verify in-memory status was updated
			Expect(dgdr.Status.ProfilingPhase).Should(Equal(nvidiacomv1beta1.ProfilingPhaseSweepingPrefill))

			// Verify conditions: reason derived from phase, message from profiler
			profilingCond := meta.FindStatusCondition(dgdr.Status.Conditions, nvidiacomv1beta1.ConditionTypeProfiling)
			Expect(profilingCond).NotTo(BeNil())
			Expect(profilingCond.Reason).Should(Equal("SweepingPrefill"))
			Expect(profilingCond.Message).Should(Equal("Sweeping TP=4 DEP=2, measuring TTFT"))
		})

		It("Should be a no-op when no progress ConfigMap exists", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-no-cm"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         "h100_sxm",
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			dgdr.Status.ProfilingPhase = nvidiacomv1beta1.ProfilingPhaseInitializing
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Re-fetch
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, dgdr)).Should(Succeed())

			// Call updateProfilingSubPhase — should not change anything
			Expect(reconciler.updateProfilingSubPhase(ctx, dgdr)).Should(Succeed())

			// ProfilingPhase should remain Initializing
			Expect(dgdr.Status.ProfilingPhase).Should(Equal(nvidiacomv1beta1.ProfilingPhaseInitializing))
		})

		It("Should skip update when phase has not changed", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-same-phase"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         "h100_sxm",
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			dgdr.Status.ProfilingPhase = nvidiacomv1beta1.ProfilingPhaseSweepingPrefill
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create output ConfigMap with same phase as status
			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      getOutputConfigMapName(dgdr),
					Namespace: namespace,
				},
				Data: map[string]string{
					"phase":   "SweepingPrefill",
					"message": "Sweeping TP=4 DEP=2, measuring TTFT",
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Re-fetch
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, dgdr)).Should(Succeed())

			// Call updateProfilingSubPhase — should not update since phase hasn't changed
			Expect(reconciler.updateProfilingSubPhase(ctx, dgdr)).Should(Succeed())

			// Should still be SweepingPrefill
			Expect(dgdr.Status.ProfilingPhase).Should(Equal(nvidiacomv1beta1.ProfilingPhaseSweepingPrefill))
		})

		It("Should return error for invalid phase value in ConfigMap", func() {
			ctx := context.Background()
			dgdrName := "test-dgdr-invalid-phase"
			namespace := defaultNamespace

			dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
				ObjectMeta: metav1.ObjectMeta{
					Name:      dgdrName,
					Namespace: namespace,
				},
				Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
					Model:   "test-model",
					Backend: "vllm",
					Image:   "test-profiler:latest",
					Hardware: &nvidiacomv1beta1.HardwareSpec{
						NumGPUsPerNode: ptr.To[int32](8),
						GPUSKU:         "h100_sxm",
						VRAMMB:         ptr.To(81920.0),
						TotalGPUs:      ptr.To[int32](128),
					},
					SLA: &nvidiacomv1beta1.SLASpec{
						TTFT: ptr.To(100.0),
						ITL:  ptr.To(1500.0),
					},
				},
			}

			Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()

			dgdr.Status.Phase = nvidiacomv1beta1.DGDRPhaseProfiling
			dgdr.Status.ProfilingPhase = nvidiacomv1beta1.ProfilingPhaseInitializing
			Expect(k8sClient.Status().Update(ctx, dgdr)).Should(Succeed())

			// Create output ConfigMap with invalid phase
			cm := &corev1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name:      getOutputConfigMapName(dgdr),
					Namespace: namespace,
				},
				Data: map[string]string{
					"phase":   "BogusPhase",
					"message": "this should not be accepted",
				},
			}
			Expect(k8sClient.Create(ctx, cm)).Should(Succeed())
			defer func() { _ = k8sClient.Delete(ctx, cm) }()

			// Re-fetch
			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: dgdrName, Namespace: namespace}, dgdr)).Should(Succeed())

			err := reconciler.updateProfilingSubPhase(ctx, dgdr)
			Expect(err).Should(HaveOccurred())
			Expect(err.Error()).Should(ContainSubstring("invalid profiling phase"))
			Expect(err.Error()).Should(ContainSubstring("BogusPhase"))

			// profilingPhase should remain unchanged
			Expect(dgdr.Status.ProfilingPhase).Should(Equal(nvidiacomv1beta1.ProfilingPhaseInitializing))
		})
	})
})