feat: dynamo deploy hello world example to k8s (#205)

deploy/dynamo/helm/deploy.sh

@@ -20,8 +20,9 @@ set -euo pipefail
 # Set default values only if not already set
 export NAMESPACE="${NAMESPACE:=cai-system}"  # Default namespace
 export NGC_TOKEN="${NGC_TOKEN:=<your-ngc-token>}"  # Default NGC token
-export CI_COMMIT_SHA="${CI_COMMIT_SHA:=6083324a0a5f310dcec38c6863f043cd9070ffcc}"  # Default commit SHA
+export CI_COMMIT_SHA="${CI_COMMIT_SHA:=250e2e0f93f7af3d83a4a0ff992e56956f7651f2}"  # Default commit SHA
 export RELEASE_NAME="${RELEASE_NAME:=dynamo-platform}"  # Default commit SHA
+export DYNAMO_INGRESS_SUFFIX="${DYNAMO_INGRESS_SUFFIX:=}"
 
 
 # Check if required variables are set
@@ -50,12 +51,13 @@ echo "NGC_TOKEN: [HIDDEN]"
 echo "RELEASE_NAME: $RELEASE_NAME"
 
 echo "generated file contents:"
-envsubst '${NAMESPACE} ${NGC_TOKEN} ${CI_COMMIT_SHA} ${RELEASE_NAME}' < dynamo-platform-values.yaml
+envsubst '${NAMESPACE} ${NGC_TOKEN} ${CI_COMMIT_SHA} ${RELEASE_NAME} ${DYNAMO_INGRESS_SUFFIX}' < dynamo-platform-values.yaml
 
-envsubst '${NAMESPACE} ${NGC_TOKEN} ${CI_COMMIT_SHA} ${RELEASE_NAME}' < dynamo-platform-values.yaml > generated-values.yaml
+envsubst '${NAMESPACE} ${NGC_TOKEN} ${CI_COMMIT_SHA} ${RELEASE_NAME} ${DYNAMO_INGRESS_SUFFIX}' < dynamo-platform-values.yaml > generated-values.yaml
 
 echo "Generated values file saved as generated-values.yaml"
 
+
 # Install/upgrade the helm chart
 echo "Installing/upgrading helm chart..."
 helm upgrade --install $RELEASE_NAME platform/ \

deploy/dynamo/helm/dynamo-platform-values.yaml

@@ -33,11 +33,9 @@ dynamo-operator:
       args:
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
-        - --leader-elect=false
-        - --natsAddr=nats://my-nats:4222
-        - --etcdAddr=etcd:2379
 
   dynamo:
+    dynamoIngressSuffix: ${DYNAMO_INGRESS_SUFFIX}
     yatai:
     # todo: only use dns name
       endpoint: http://dynamo-server
@@ -66,9 +64,9 @@ dynamo-operator:
       inClusterServer: ''
       username: '$oauthtoken'
       # Password will use global.NGC_API_KEY by default
-      password: ""
-      passwordExistingSecretName: ''
-      passwordExistingSecretKey: ''
+      # password: ""
+      passwordExistingSecretName: 'nvcrimagepullsecret'
+      # passwordExistingSecretKey: ''
       secure: true
       bentoRepositoryName: yatai-bentos
 
@@ -97,7 +95,11 @@ dynamo-api-server:
   image:
     repository: gitlab-master.nvidia.com:5005/aire/microservices/compoundai/dynamo-api-server
     tag: ${CI_COMMIT_SHA}
-    pullPolicy: Always
+    pullPolicy: IfNotPresent
+  storeImage:
+    repository: gitlab-master.nvidia.com:5005/aire/microservices/compoundai/dynamo-api-store
+    tag: ${CI_COMMIT_SHA}
+    pullPolicy: IfNotPresent
   imagePullSecrets:
     - name: nvcrimagepullsecret
     - name: gitlab-imagepull
@@ -108,8 +110,8 @@ dynamo-api-server:
   minio:
     persistence:
       enabled: true
-      size: 10Gi
-      storageClass: ""
+      size: 5Gi
+      storageClass: "local-path"
     apiIngress:
       enabled: false
       ingressClassName: nginx
@@ -118,6 +120,17 @@ dynamo-api-server:
 
 etcd:
   enabled: true
+  persistence:
+    enabled: true
+    storageClass: "local-path"
+    size: 1Gi
 
 nats:
   enabled: true
+  config:
+    jetstream:
+      enabled: true
+      fileStore:
+        pvc:
+          size: 1Gi
+          storageClassName: "local-path"
\ No newline at end of file

deploy/dynamo/helm/platform/components/api-server/templates/deployment.yaml → deploy/dynamo/helm/platform/components/api-server/templates/deployment-api-server.yaml

@@ -55,7 +55,7 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           ports:
             - name: http
-              containerPort: {{ .Values.dynamo.port }}
+              containerPort: {{ .Values.dynamo.apiServer.port }}
               protocol: TCP
           livenessProbe:
             {{- toYaml .Values.livenessProbe | nindent 12 }}
@@ -84,25 +84,11 @@ spec:
             - name: API_DATABASE_PORT
               value: "8001"
             - name: API_BACKEND_URL
-              value: "http://127.0.0.1:8001"
+              value: "http://dynamo-store"
             - name: RESOURCE_SCOPE
               value: {{ .Values.dynamo.env.resource_scope | quote }}
             - name: DEFAULT_KUBE_NAMESPACE
               value: {{ .Release.Namespace }}
-            - name: DYNAMO_CONTAINER_NAME
-              value: "dynamo-storage"
-            - name: S3_ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-                  name: "{{ .Release.Name }}-minio"
-                  key: root-user
-            - name: S3_SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: "{{ .Release.Name }}-minio"
-                  key: root-password
-            - name: S3_ENDPOINT_URL
-              value: "http://{{ .Release.Name }}-minio:9000"
           envFrom:
           - secretRef:
               name: dynamo-deployment-env

deploy/dynamo/helm/platform/components/api-server/templates/deployment-api-store.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dynamo-api-store
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dynamo-api-store
+  template:
+    metadata:
+      labels:
+        app: dynamo-api-store
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "helm.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      initContainers:
+        - name: wait-for-postgres
+          image: busybox
+          command: [ 'sh', '-c', 'until nc -z {{ .Release.Name }}-postgresql 5432; do echo "PostgreSQL is unavailable. Sleeping for 5 seconds"; sleep 5; done;' ]
+      containers:
+        - name: "api-store"
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          imagePullPolicy: {{ .Values.storeImage.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.dynamo.apiStore.port }}
+              protocol: TCP
+          livenessProbe:
+            {{- toYaml .Values.livenessProbe | nindent 12 }}
+          readinessProbe:
+            {{- toYaml .Values.readinessProbe | nindent 12 }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          {{- with .Values.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          env:
+            - name: DB_HOST
+              value:  "{{ .Release.Name }}-postgresql"
+            - name: DB_USER
+              value: {{ .Values.postgresql.auth.username | quote }}
+            - name: DB_NAME
+              value: {{ .Values.postgresql.auth.database | quote }}
+            - name: DB_PORT
+              value: "5432"
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ .Release.Name }}-postgresql"
+                  key: password
+            - name: SERVICE_PORT
+              value: "8000"
+            - name: RESOURCE_SCOPE
+              value: {{ .Values.dynamo.env.resource_scope | quote }}
+            - name: DEFAULT_KUBE_NAMESPACE
+              value: {{ .Release.Namespace }}
+            - name: DYN_OBJECT_STORE_BUCKET
+              value: "dynamo-storage"
+            - name: DYN_OBJECT_STORE_ID
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ .Release.Name }}-minio"
+                  key: root-user
+            - name: DYN_OBJECT_STORE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ .Release.Name }}-minio"
+                  key: root-password
+            - name: DYN_OBJECT_STORE_ENDPOINT
+              value: "http://{{ .Release.Name }}-minio:9000"
+          envFrom:
+          - secretRef:
+              name: dynamo-deployment-env
+          image: "{{ .Values.storeImage.repository }}:{{ .Values.storeImage.tag | default .Chart.AppVersion }}"
+      {{- with .Values.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      imagePullSecrets: {{ .Values.imagePullSecrets | default list | toJson }}
\ No newline at end of file

deploy/dynamo/helm/platform/components/api-server/templates/deployment-ui.yaml

@@ -36,7 +36,7 @@ spec:
     spec:
       containers:
       - image: nvcr.io/nvidian/nim-llm-dev/compoundai-ui:0.0.11
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent
         livenessProbe:
           failureThreshold: 3
           httpGet:

deploy/dynamo/helm/platform/components/api-server/templates/server-rbac.yaml

@@ -27,6 +27,18 @@ rules:
 - apiGroups: [""]
   resources: ["pods"]
   verbs: ["get", "list", "watch"]
+- apiGroups:
+  - nvidia.com
+  resources:
+  - dynamodeployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 {{- if .Values.namespaceRestriction.enabled }}
@@ -44,6 +56,10 @@ subjects:
   name: {{ include "helm.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
 roleRef:
+  {{- if .Values.namespaceRestriction.enabled }}
+  kind: Role
+  {{- else }}
   kind: ClusterRole
+  {{- end }}
   name: {{ include "helm.fullname" . }}-role
   apiGroup: rbac.authorization.k8s.io
\ No newline at end of file

deploy/dynamo/helm/platform/components/api-server/templates/service-store.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: Service
+metadata:
+  name: "dynamo-store"
+  labels:
+    {{- include "helm.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: {{ .Values.dynamo.apiStore.port }}
+      protocol: TCP
+      name: http
+  selector:
+    app: dynamo-api-store

deploy/dynamo/helm/platform/components/api-server/templates/service.yaml

@@ -22,7 +22,7 @@ spec:
   type: {{ .Values.service.type }}
   ports:
     - port: {{ .Values.service.port }}
-      targetPort: {{ .Values.dynamo.port }}
+      targetPort: {{ .Values.dynamo.apiServer.port }}
       protocol: TCP
       name: http
   selector:

deploy/dynamo/helm/platform/components/api-server/values.yaml

@@ -19,7 +19,7 @@ replicaCount: 1
 image:
   repository: gitlab-master.nvidia.com:5005/aire/microservices/dynamo/api-server
   # This sets the pull policy for images.
-  pullPolicy: Always
+  pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
   tag: ""
 
@@ -105,7 +105,7 @@ ingress:
 
 # Istio settings
 istio:
-  host: cai.dev.aire.nvidia.com
+  host: ""
 
 ## In-cluster minio deployment configuration
 ## ref: https://github.com/bitnami/charts/blob/minio/13.3.1/bitnami/minio/values.yaml
@@ -145,7 +145,10 @@ resources: {}
   #   memory: 128Mi
 
 dynamo:
+  apiServer:
     port: 8181
+  apiStore:
+    port: 8000
 
 postgresql:
   auth:

deploy/dynamo/helm/platform/components/operator/charts/dynamo-crds/crds/nvidia.com_dynamonimdeployments.yaml

@@ -391,6 +391,8 @@ spec:
                 type: object
               dynamoNim:
                 type: string
+              dynamoTag:
+                type: string
               envFromSecret:
                 type: string
               envs:
@@ -2784,6 +2786,7 @@ spec:
                 type: string
             required:
             - dynamoNim
+            - dynamoTag
             type: object
           status:
             properties:

deploy/dynamo/helm/platform/components/operator/templates/_helpers.tpl

@@ -96,9 +96,18 @@ Generate docker config json for registry credentials
 {{- $username := .Values.dynamo.dockerRegistry.username -}}
 {{- $password := default .Values.global.NGC_API_KEY .Values.dynamo.dockerRegistry.password -}}
 {{- if .Values.dynamo.dockerRegistry.passwordExistingSecretName -}}
-{{- $password = .Values.dynamo.dockerRegistry.passwordExistingSecretKey -}}
-{{- end -}}
-{
+  {{- $secretName := .Values.dynamo.dockerRegistry.passwordExistingSecretName -}}
+  {{- $secretKey := .Values.dynamo.dockerRegistry.passwordExistingSecretKey -}}
+  {{- $dockerconfigjson := lookup "v1" "Secret" .Release.Namespace $secretName }}
+
+  {{- if $dockerconfigjson -}}
+    {{- if eq $dockerconfigjson.type "kubernetes.io/dockerconfigjson" -}}
+      {{/* If the secret is already of type kubernetes.io/dockerconfigjson, use its .dockerconfigjson value directly */}}
+      {{- index $dockerconfigjson.data ".dockerconfigjson" | b64dec }}
+    {{- else -}}
+      {{/* If the secret is not of the correct type, extract password from the secret and build a new one */}}
+      {{- $password = index $dockerconfigjson.data $secretKey | b64dec }}
+      {
         "auths": {
           "{{ $server }}": {
             "username": "{{ $username }}",
@@ -106,5 +115,64 @@ Generate docker config json for registry credentials
             "auth": "{{ printf "%s:%s" $username $password | b64enc }}"
           }
         }
-}
+      }
+    {{- end -}}
+  {{- else -}}
+    {{/* If no secret is found, use the default password */}}
+    {{- $password = .Values.dynamo.dockerRegistry.password | default .Values.global.NGC_API_KEY }}
+    {
+      "auths": {
+        "{{ $server }}": {
+          "username": "{{ $username }}",
+          "password": "{{ $password }}",
+          "auth": "{{ printf "%s:%s" $username $password | b64enc }}"
+        }
+      }
+    }
+  {{- end -}}
+{{- else -}}
+  {{/* Build a new dockerconfigjson if passwordExistingSecretName is not set */}}
+  {
+    "auths": {
+      "{{ $server }}": {
+        "username": "{{ $username }}",
+        "password": "{{ $password }}",
+        "auth": "{{ printf "%s:%s" $username $password | b64enc }}"
+      }
+    }
+  }
 {{- end -}}
+{{- end -}}
+
+
+{{/*
+Extract username and password from docker registry configuration
+*/}}
+{{- define "dynamo-operator.extractDockerCredentials" -}}
+{{- $server := .Values.dynamo.dockerRegistry.server -}}
+{{- $username := .Values.dynamo.dockerRegistry.username -}}
+{{- $password := default .Values.global.NGC_API_KEY .Values.dynamo.dockerRegistry.password -}}
+{{- $result := dict "username" $username "password" $password }}
+
+{{- if .Values.dynamo.dockerRegistry.passwordExistingSecretName }}
+  {{- $secretName := .Values.dynamo.dockerRegistry.passwordExistingSecretName }}
+  {{- $secretKey := .Values.dynamo.dockerRegistry.passwordExistingSecretKey }}
+  {{- $dockerconfigjson := lookup "v1" "Secret" .Release.Namespace $secretName }}
+
+  {{- if $dockerconfigjson }}
+    {{- if eq $dockerconfigjson.type "kubernetes.io/dockerconfigjson" }}
+      {{- $decodedConfig := index $dockerconfigjson.data ".dockerconfigjson" | b64dec | fromJson }}
+      {{- range $registry, $authConfig := $decodedConfig.auths }}
+        {{- $_ := set $result "username" $authConfig.username }}
+        {{- $_ := set $result "password" $authConfig.password }}
+        {{- break }}
+      {{- end }}
+    {{- else if hasKey $dockerconfigjson.data $secretKey }}
+      {{- $_ := set $result "password" (index $dockerconfigjson.data $secretKey | b64dec) }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+
+{{- toYaml $result }}
+
+{{- end }}
\ No newline at end of file

deploy/dynamo/helm/platform/components/operator/templates/compoundai-secret-env.yaml

@@ -42,12 +42,15 @@ stringData:
   ENABLE_RESTRICTED_SECURITY_CONTEXT: "true"
   {{- end }}
 
+  {{- if .Values.dynamo.dynamoIngressSuffix }}
+  DYNAMO_INGRESS_SUFFIX: {{ .Values.dynamo.dynamoIngressSuffix | quote }}
+  {{- end }}
 
   DOCKER_REGISTRY_SERVER: {{ .Values.dynamo.dockerRegistry.server | quote }}
   DOCKER_REGISTRY_IN_CLUSTER_SERVER: {{ .Values.dynamo.dockerRegistry.inClusterServer | quote }}
-  DOCKER_REGISTRY_USERNAME: {{ .Values.dynamo.dockerRegistry.username | quote }}
-  {{- if not .Values.dynamo.dockerRegistry.passwordExistingSecretName }}
-  DOCKER_REGISTRY_PASSWORD: {{ default .Values.global.NGC_API_KEY .Values.dynamo.dockerRegistry.password | quote }}
+  {{- with include "dynamo-operator.extractDockerCredentials" . | fromYaml }}
+  DOCKER_REGISTRY_USERNAME: {{ .username | quote }}
+  DOCKER_REGISTRY_PASSWORD: {{ .password | quote }}
   {{- end }}
   DOCKER_REGISTRY_SECURE: {{ .Values.dynamo.dockerRegistry.secure | quote }}
   DOCKER_REGISTRY_BENTO_REPOSITORY_NAME: {{ .Values.dynamo.dockerRegistry.bentoRepositoryName | quote }}

deploy/dynamo/helm/platform/components/operator/templates/deployment.yaml

@@ -61,6 +61,10 @@ spec:
         {{- end }}
         {{- if .Values.namespaceRestriction.enabled }}
           - --restrictedNamespace={{ default .Release.Namespace .Values.namespaceRestriction.targetNamespace }}
+          - --leader-elect=false
+        {{- else }}
+          - --leader-elect
+          - --leader-election-id=dynamo.nvidia.com
         {{- end }}
         {{- if .Values.natsAddr }}
           - --natsAddr={{ .Values.natsAddr }}

deploy/dynamo/helm/platform/components/operator/templates/manager-rbac.yaml

@@ -329,6 +329,7 @@ rules:
   - nvidia.com
   resources:
   - dynamonimdeployments
+  - dynamodeployments
   verbs:
   - create
   - delete
@@ -341,12 +342,14 @@ rules:
   - nvidia.com
   resources:
   - dynamonimdeployments/finalizers
+  - dynamodeployments/finalizers
   verbs:
   - update
 - apiGroups:
   - nvidia.com
   resources:
   - dynamonimdeployments/status
+  - dynamodeployments/status
   verbs:
   - get
   - patch

deploy/dynamo/operator/api/v1alpha1/dynamoinimdeployment_types.go

@@ -41,6 +41,8 @@ type DynamoNimDeploymentSpec struct {
 	Labels      map[string]string `json:"labels,omitempty"`
 
 	DynamoNim string `json:"dynamoNim"`
+	// contains the tag of the DynamoNim: for example, "my_package:MyService"
+	DynamoTag string `json:"dynamoTag"`
 
 	// contains the name of the service
 	ServiceName string `json:"serviceName,omitempty"`

deploy/dynamo/operator/api/v1alpha1/zz_generated.deepcopy.go

 //go:build !ignore_autogenerated
 
+/*
+SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 /*
 Copyright 2024.
 

deploy/dynamo/operator/config/crd/bases/nvidia.com_dynamodeployments.yaml

@@ -392,6 +392,8 @@ spec:
                           type: object
                         dynamoNim:
                           type: string
+                        dynamoTag:
+                          type: string
                         envFromSecret:
                           type: string
                         envs:
@@ -2785,6 +2787,7 @@ spec:
                           type: string
                       required:
                       - dynamoNim
+                      - dynamoTag
                       type: object
                     status:
                       properties:

deploy/dynamo/operator/config/crd/bases/nvidia.com_dynamonimdeployments.yaml

@@ -391,6 +391,8 @@ spec:
                 type: object
               dynamoNim:
                 type: string
+              dynamoTag:
+                type: string
               envFromSecret:
                 type: string
               envs:
@@ -2784,6 +2786,7 @@ spec:
                 type: string
             required:
             - dynamoNim
+            - dynamoTag
             type: object
           status:
             properties:

deploy/dynamo/operator/internal/controller/dynamodeployment_controller.go

@@ -187,7 +187,9 @@ func (r *DynamoDeploymentReconciler) getSecret(ctx context.Context, namespace, n
 // SetupWithManager sets up the controller with the Manager.
 func (r *DynamoDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&nvidiacomv1alpha1.DynamoDeployment{}).
+		For(&nvidiacomv1alpha1.DynamoDeployment{}, builder.WithPredicates(
+			predicate.GenerationChangedPredicate{},
+		)).
 		Named("dynamodeployment").
 		Owns(&nvidiacomv1alpha1.DynamoNimDeployment{}, builder.WithPredicates(predicate.Funcs{
 			// ignore creation cause we don't want to be called again after we create the deployment