feat: simplify CRD management (#6466)

Signed-off-by: Julien Mancuso <jmancuso@nvidia.com>

.github/workflows/pr.yaml

@@ -332,6 +332,7 @@ jobs:
           --set dynamo-operator.controllerManager.manager.image.repository=${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo \
           --set dynamo-operator.controllerManager.manager.image.tag=${{ steps.operator-tag.outputs.tag }} \
           --set dynamo-operator.gpuDiscovery.enabled=false \
+          --set dynamo-operator.upgradeCRD=false \
           --debug
         # Wait for all deployments to be ready
         timeout 300s kubectl rollout status deployment -n $NAMESPACE --watch

deploy/helm/README.md

@@ -17,7 +17,13 @@ limitations under the License.
 
 # Dynamo Kubernetes Helm Charts
 
-There are two Helm charts available for the Dynamo Kubernetes Platform:
+The following Helm chart is available for the Dynamo Kubernetes Platform:
 
 - [platform](./charts/platform/README.md) - This chart installs the complete Dynamo Kubernetes Platform, including the Dynamo Operator, NATS, etcd, Grove, and Kai Scheduler.
-- [crds](./charts/crds/README.md) - This chart installs the CRDs for the Dynamo.
+
\ No newline at end of file
+## CRD Management
+
+CRDs are bundled in the operator subchart's `crds/` directory and managed automatically:
+
+- **Initial install**: Helm natively installs CRDs from the `crds/` directory during `helm install`.
+- **Upgrades**: A `pre-upgrade` hook Job applies CRDs using server-side apply from the operator image. This is necessary because Helm does not update CRDs from the `crds/` directory on `helm upgrade`. This can be disabled by setting `upgradeCRD: false`.
\ No newline at end of file

deploy/helm/charts/crds/Chart.yaml

-# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-apiVersion: v2
-name: dynamo-crds
-description: A Helm chart for dynamo CRDs
-type: application
-version: 0.9.0
-dependencies: []
\ No newline at end of file

deploy/helm/charts/crds/README.md

-<!--
-SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-SPDX-License-Identifier: Apache-2.0
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Dynamo Kubernetes Platform CRDs Helm Chart
-
-This chart installs the [CRDs](../../../../docs/pages/kubernetes/api-reference.md) for the Dynamo Kubernetes Platform.
\ No newline at end of file

deploy/helm/charts/platform/README.md

@@ -109,6 +109,7 @@ The chart includes built-in validation to prevent all operator conflicts:
 |-----|------|---------|-------------|
 | global.etcd.install | bool | `false` | Whether this chart should install the bundled etcd subchart. When true, deploys etcd and auto-configures the operator with its address. When false, etcd is not deployed. Use dynamo-operator.etcdAddr to point at an external instance if you are bringing your own etcd. |
 | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment |
+| dynamo-operator.upgradeCRD | bool | `true` | Whether to manage CRDs via a pre-install/pre-upgrade hook Job. The Job runs the operator image with the crd-apply tool to apply CRDs via server-side apply. |
 | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:port" |
 | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: "http://hostname:port" or "https://hostname:port" |
 | dynamo-operator.nats.enabled | bool | `true` | Whether the NATS is enabled |

deploy/helm/charts/crds/templates/nvidia.com_dynamocheckpoints.yaml → deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamocheckpoints.yaml

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 ---
 apiVersion: apiextensions.k8s.io/v1

deploy/helm/charts/crds/templates/nvidia.com_dynamocomponentdeployments.yaml → deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamocomponentdeployments.yaml

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 ---
 apiVersion: apiextensions.k8s.io/v1

deploy/helm/charts/crds/templates/nvidia.com_dynamographdeploymentrequests.yaml → deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeploymentrequests.yaml

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 ---
 apiVersion: apiextensions.k8s.io/v1

deploy/helm/charts/crds/templates/nvidia.com_dynamographdeployments.yaml → deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 ---
 apiVersion: apiextensions.k8s.io/v1

deploy/helm/charts/crds/templates/nvidia.com_dynamographdeploymentscalingadapters.yaml → deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeploymentscalingadapters.yaml

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 ---
 apiVersion: apiextensions.k8s.io/v1

deploy/helm/charts/crds/templates/nvidia.com_dynamomodels.yaml → deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamomodels.yaml

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 ---
 apiVersion: apiextensions.k8s.io/v1

deploy/helm/charts/crds/templates/nvidia.com_dynamoworkermetadatas.yaml → deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamoworkermetadatas.yaml

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   name: dynamoworkermetadatas.nvidia.com
+  annotations:
+    helm.sh/resource-policy: keep
 spec:
   group: nvidia.com
   names:
@@ -59,4 +49,3 @@ spec:
         - name: Age
           type: date
           jsonPath: .metadata.creationTimestamp
-

deploy/helm/charts/platform/components/operator/templates/gpu-discovery-preflight.yaml

@@ -33,7 +33,7 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-gpu-discovery-preflight
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-gpu-discovery-preflight
   labels:
     {{- include "dynamo-operator.labels" . | nindent 4 }}
   annotations:
@@ -48,7 +48,7 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-gpu-discovery-preflight
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-gpu-discovery-preflight
   labels:
     {{- include "dynamo-operator.labels" . | nindent 4 }}
   annotations:
@@ -58,7 +58,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: {{ include "dynamo-operator.fullname" . }}-gpu-discovery-preflight
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-gpu-discovery-preflight
 subjects:
   - kind: ServiceAccount
     name: {{ include "dynamo-operator.fullname" . }}-gpu-discovery-preflight

deploy/helm/charts/platform/components/operator/templates/gpu-discovery-rbac.yaml

@@ -20,7 +20,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-gpu-discovery
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-gpu-discovery
   labels:
     {{- include "dynamo-operator.labels" . | nindent 4 }}
 rules:
@@ -31,13 +31,13 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-gpu-discovery
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-gpu-discovery
   labels:
     {{- include "dynamo-operator.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: {{ include "dynamo-operator.fullname" . }}-gpu-discovery
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-gpu-discovery
 subjects:
   - kind: ServiceAccount
     name: {{ include "dynamo-operator.fullname" . }}-controller-manager

deploy/helm/charts/platform/components/operator/templates/upgrade-crd.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if .Values.upgradeCRD }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-apply-sa
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
+    helm.sh/hook-weight: "0"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-apply-role
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
+    helm.sh/hook-weight: "0"
+rules:
+  - apiGroups:
+      - apiextensions.k8s.io
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - patch
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-apply-binding
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
+    helm.sh/hook-weight: "0"
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-apply-sa
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-apply-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "dynamo-operator.fullname" . }}-crd-apply
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "1"
+    helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
+  labels:
+    {{- include "dynamo-operator.labels" . | nindent 4 }}
+    app.kubernetes.io/component: crd-apply
+spec:
+  template:
+    metadata:
+      name: {{ include "dynamo-operator.fullname" . }}-crd-apply
+      labels:
+        {{- include "dynamo-operator.labels" . | nindent 8 }}
+        app.kubernetes.io/component: crd-apply
+    spec:
+      serviceAccountName: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-apply-sa
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controllerManager.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controllerManager.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+        runAsNonRoot: true
+      containers:
+        - name: crd-apply
+          image: {{ .Values.controllerManager.manager.image.repository }}:{{ .Values.controllerManager.manager.image.tag | default .Chart.AppVersion }}
+          imagePullPolicy: {{ .Values.controllerManager.manager.image.pullPolicy | default "IfNotPresent" }}
+          command: ["/crd-apply"]
+          args:
+            - "--crds-dir=/opt/dynamo-operator/crds/"
+            - "--version={{ .Chart.AppVersion }}"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+          resources:
+            limits:
+              cpu: 500m
+              memory: 256Mi
+            requests:
+              cpu: 100m
+              memory: 128Mi
+      restartPolicy: OnFailure
+{{- end }}

deploy/helm/charts/platform/components/operator/templates/webhook-ca-inject-job.yaml

@@ -79,7 +79,7 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-webhook-ca-inject
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-webhook-ca-inject
   labels:
     app.kubernetes.io/component: webhook
     app.kubernetes.io/created-by: dynamo-operator
@@ -114,7 +114,7 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-webhook-ca-inject
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-webhook-ca-inject
   labels:
     app.kubernetes.io/component: webhook
     app.kubernetes.io/created-by: dynamo-operator
@@ -127,7 +127,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: {{ include "dynamo-operator.fullname" . }}-webhook-ca-inject
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-webhook-ca-inject
 subjects:
 - kind: ServiceAccount
   name: {{ include "dynamo-operator.fullname" . }}-webhook-ca-inject

deploy/helm/charts/platform/components/operator/templates/webhook-crd-conversion-certmanager.yaml

@@ -35,7 +35,7 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-crd-conversion-patch
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-conversion-patch
   labels:
     app.kubernetes.io/component: webhook
     app.kubernetes.io/created-by: dynamo-operator
@@ -54,7 +54,7 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: {{ include "dynamo-operator.fullname" . }}-crd-conversion-patch
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-conversion-patch
   labels:
     app.kubernetes.io/component: webhook
     app.kubernetes.io/created-by: dynamo-operator
@@ -67,7 +67,7 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: {{ include "dynamo-operator.fullname" . }}-crd-conversion-patch
+  name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-crd-conversion-patch
 subjects:
 - kind: ServiceAccount
   name: {{ include "dynamo-operator.fullname" . }}-crd-conversion-patch

deploy/helm/charts/platform/components/operator/values.yaml

@@ -16,6 +16,10 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
+# Whether to manage CRDs via a pre-install/pre-upgrade hook Job.
+# The Job runs the operator image with the crd-apply tool to apply CRDs via server-side apply.
+upgradeCRD: true
+
 # Namespace restriction configuration for the operator
 # If enabled: true and targetNamespace is empty, the operator will be restricted to the release namespace
 # If enabled: true and targetNamespace is set, the operator will be restricted to the specified namespace

deploy/helm/charts/platform/values.yaml

@@ -28,6 +28,10 @@ dynamo-operator:
   # -- Whether to enable the Dynamo Kubernetes operator deployment
   enabled: true
 
+  # -- Whether to manage CRDs via a pre-install/pre-upgrade hook Job.
+  # The Job runs the operator image with the crd-apply tool to apply CRDs via server-side apply.
+  upgradeCRD: true
+
   # -- NATS server address for operator communication (leave empty to use the bundled NATS chart). Format: "nats://hostname:port"
   natsAddr: ""
 

deploy/operator/Dockerfile

@@ -47,13 +47,15 @@ RUN make test
 # Build stage - depends on successful lint and test
 FROM base AS builder
 
-# Build the binary
-RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cmd/main.go && \
+    CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o crd-apply ./cmd/crd-apply/main.go
 
 # Runtime stage
 FROM nvcr.io/nvidia/distroless/go:v3.1.13
 WORKDIR /
 COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/crd-apply .
+COPY --from=builder /workspace/config/crd/bases/ /opt/dynamo-operator/crds/
 USER 65532:65532
 
 ENTRYPOINT ["./manager"]