Unverified Commit 5277fb9b authored by Julien Mancuso's avatar Julien Mancuso Committed by GitHub
Browse files

feat: simplify CRD management (#6466)


Signed-off-by: default avatarJulien Mancuso <jmancuso@nvidia.com>
parent 0f48837d
...@@ -88,18 +88,6 @@ manifests: controller-gen ensure-yq ## Generate WebhookConfiguration, ClusterRol ...@@ -88,18 +88,6 @@ manifests: controller-gen ensure-yq ## Generate WebhookConfiguration, ClusterRol
{ printf '%s\n' \ { printf '%s\n' \
'# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.' \ '# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.' \
'# SPDX-License-Identifier: Apache-2.0' \ '# SPDX-License-Identifier: Apache-2.0' \
'#' \
'# Licensed under the Apache License, Version 2.0 (the "License");' \
'# you may not use this file except in compliance with the License.' \
'# You may obtain a copy of the License at' \
'#' \
'# http://www.apache.org/licenses/LICENSE-2.0' \
'#' \
'# Unless required by applicable law or agreed to in writing, software' \
'# distributed under the License is distributed on an "AS IS" BASIS,' \
'# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.' \
'# See the License for the specific language governing permissions and' \
'# limitations under the License.' \
''; \ ''; \
cat "$$file"; \ cat "$$file"; \
} > "$$file.tmp" && mv "$$file.tmp" "$$file"; \ } > "$$file.tmp" && mv "$$file.tmp" "$$file"; \
...@@ -111,29 +99,15 @@ manifests: controller-gen ensure-yq ## Generate WebhookConfiguration, ClusterRol ...@@ -111,29 +99,15 @@ manifests: controller-gen ensure-yq ## Generate WebhookConfiguration, ClusterRol
yq eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \ yq eval '.metadata.annotations."helm.sh/resource-policy" = "keep"' -i "$$file"; \
fi; \ fi; \
done done
if [ ! -d "../helm/charts/crds/templates/" ]; then \ echo "Copying CRD files to operator Helm chart crds/ directory"
echo "Creating directory ../helm/charts/crds/templates/"; \ mkdir -p ../helm/charts/platform/components/operator/crds/
mkdir -p ../helm/charts/crds/templates/; \ cp config/crd/bases/*.yaml ../helm/charts/platform/components/operator/crds/
fi
cp config/crd/bases/*.yaml ../helm/charts/crds/templates/
echo "Adding NVIDIA header to RBAC files" echo "Adding NVIDIA header to RBAC files"
for file in config/rbac/*.yaml; do \ for file in config/rbac/*.yaml; do \
if [ -f "$$file" ] && ! head -20 "$$file" | grep -q "NVIDIA CORPORATION"; then \ if [ -f "$$file" ] && ! head -20 "$$file" | grep -q "NVIDIA CORPORATION"; then \
{ printf '%s\n' \ { printf '%s\n' \
'# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.' \ '# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.' \
'# SPDX-License-Identifier: Apache-2.0' \ '# SPDX-License-Identifier: Apache-2.0' \
'#' \
'# Licensed under the Apache License, Version 2.0 (the "License");' \
'# you may not use this file except in compliance with the License.' \
'# You may obtain a copy of the License at' \
'#' \
'# http://www.apache.org/licenses/LICENSE-2.0' \
'#' \
'# Unless required by applicable law or agreed to in writing, software' \
'# distributed under the License is distributed on an "AS IS" BASIS,' \
'# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.' \
'# See the License for the specific language governing permissions and' \
'# limitations under the License.' \
''; \ ''; \
cat "$$file"; \ cat "$$file"; \
} > "$$file.tmp" && mv "$$file.tmp" "$$file"; \ } > "$$file.tmp" && mv "$$file.tmp" "$$file"; \
......
/*
* SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/
package main
import (
"context"
"flag"
"fmt"
"os"
"path/filepath"
"strings"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/yaml"
)
const (
fieldManager = "dynamo-crd-apply"
versionAnnotation = "dynamo.nvidia.com/operator-version"
)
func main() {
crdsDir := flag.String("crds-dir", "/opt/dynamo-operator/crds/", "Directory containing CRD YAML files")
version := flag.String("version", "", "Operator version to stamp on CRDs")
flag.Parse()
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
log := ctrl.Log.WithName("crd-apply")
config, err := ctrl.GetConfig()
if err != nil {
log.Error(err, "unable to get kubernetes config")
os.Exit(1)
}
client, err := apiextensionsclient.NewForConfig(config)
if err != nil {
log.Error(err, "unable to create apiextensions client")
os.Exit(1)
}
entries, err := os.ReadDir(*crdsDir)
if err != nil {
log.Error(err, "unable to read CRDs directory", "dir", *crdsDir)
os.Exit(1)
}
ctx := context.Background()
var applied int
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") {
continue
}
filePath := filepath.Join(*crdsDir, entry.Name())
data, err := os.ReadFile(filePath)
if err != nil {
log.Error(err, "unable to read CRD file", "file", filePath)
os.Exit(1)
}
crd := &apiextensionsv1.CustomResourceDefinition{}
if err := yaml.Unmarshal(data, crd); err != nil {
log.Error(err, "unable to unmarshal CRD", "file", filePath)
os.Exit(1)
}
if *version != "" {
if crd.Annotations == nil {
crd.Annotations = make(map[string]string)
}
crd.Annotations[versionAnnotation] = *version
}
patchData, err := yaml.Marshal(crd)
if err != nil {
log.Error(err, "unable to marshal CRD for patch", "crd", crd.Name)
os.Exit(1)
}
_, err = client.ApiextensionsV1().CustomResourceDefinitions().Patch(
ctx,
crd.Name,
types.ApplyPatchType,
patchData,
metav1.PatchOptions{
FieldManager: fieldManager,
Force: ptr.To(true),
},
)
if err != nil {
log.Error(err, "unable to apply CRD", "crd", crd.Name)
os.Exit(1)
}
log.Info("Applied CRD", "crd", crd.Name)
applied++
}
if applied == 0 {
fmt.Fprintf(os.Stderr, "WARNING: no CRD files found in %s\n", *crdsDir)
os.Exit(1)
}
log.Info("CRD apply complete", "applied", applied)
}
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
   
--- ---
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
......
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
   
--- ---
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
......
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
   
--- ---
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
......
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
   
--- ---
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
......
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
--- ---
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
......
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
--- ---
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
......
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: dynamoworkermetadatas.nvidia.com
annotations:
helm.sh/resource-policy: keep
spec:
group: nvidia.com
names:
kind: DynamoWorkerMetadata
listKind: DynamoWorkerMetadataList
plural: dynamoworkermetadatas
singular: dynamoworkermetadata
shortNames:
- dwm
scope: Namespaced
versions:
- name: v1alpha1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
description: DynamoWorkerMetadata stores discovery metadata for a Dynamo worker pod
properties:
apiVersion:
type: string
description: APIVersion defines the versioned schema of this representation
kind:
type: string
description: Kind is a string value representing the REST resource
metadata:
type: object
spec:
type: object
description: Spec contains the worker metadata
required:
- data
properties:
data:
type: object
description: Raw JSON blob containing DiscoveryMetadata
x-kubernetes-preserve-unknown-fields: true
additionalPrinterColumns:
- name: Age
type: date
jsonPath: .metadata.creationTimestamp
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
......
...@@ -55,18 +55,14 @@ This validates kubectl connectivity, StorageClass configuration, and GPU availab ...@@ -55,18 +55,14 @@ This validates kubectl connectivity, StorageClass configuration, and GPU availab
export NAMESPACE=dynamo-system export NAMESPACE=dynamo-system
export RELEASE_VERSION=0.x.x # any version of Dynamo 0.3.2+ listed at https://github.com/ai-dynamo/dynamo/releases export RELEASE_VERSION=0.x.x # any version of Dynamo 0.3.2+ listed at https://github.com/ai-dynamo/dynamo/releases
# 2. Install CRDs (skip if on shared cluster where CRDs already exist) # 2. Install Platform (CRDs are automatically installed by the chart)
helm fetch https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-crds-${RELEASE_VERSION}.tgz
helm install dynamo-crds dynamo-crds-${RELEASE_VERSION}.tgz --namespace default
# 3. Install Platform
helm fetch https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-${RELEASE_VERSION}.tgz helm fetch https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-${RELEASE_VERSION}.tgz
helm install dynamo-platform dynamo-platform-${RELEASE_VERSION}.tgz --namespace ${NAMESPACE} --create-namespace helm install dynamo-platform dynamo-platform-${RELEASE_VERSION}.tgz --namespace ${NAMESPACE} --create-namespace
``` ```
**For Shared/Multi-Tenant Clusters:** **For Shared/Multi-Tenant Clusters:**
If your cluster has namespace-restricted Dynamo operators, add this flag to step 3: If your cluster has namespace-restricted Dynamo operators, add this flag to step 2:
```bash ```bash
--set dynamo-operator.namespaceRestriction.enabled=true --set dynamo-operator.namespaceRestriction.enabled=true
``` ```
......
...@@ -191,11 +191,8 @@ docker build -t $DOCKER_SERVER/dynamo-operator:$IMAGE_TAG . ...@@ -191,11 +191,8 @@ docker build -t $DOCKER_SERVER/dynamo-operator:$IMAGE_TAG .
docker push $DOCKER_SERVER/dynamo-operator:$IMAGE_TAG docker push $DOCKER_SERVER/dynamo-operator:$IMAGE_TAG
cd - cd -
# Install CRDs # Install platform with custom operator image (CRDs are automatically installed by the chart)
cd deploy/helm/charts cd deploy/helm/charts
helm install dynamo-crds ./crds/ --namespace default
# Install platform with custom operator image
helm install dynamo-platform ./platform/ \ helm install dynamo-platform ./platform/ \
--namespace ${NAMESPACE} \ --namespace ${NAMESPACE} \
--create-namespace \ --create-namespace \
......
...@@ -128,11 +128,7 @@ Install from [NGC published artifacts](https://catalog.ngc.nvidia.com/orgs/nvidi ...@@ -128,11 +128,7 @@ Install from [NGC published artifacts](https://catalog.ngc.nvidia.com/orgs/nvidi
export NAMESPACE=dynamo-system export NAMESPACE=dynamo-system
export RELEASE_VERSION=0.x.x # any version of Dynamo 0.3.2+ listed at https://github.com/ai-dynamo/dynamo/releases export RELEASE_VERSION=0.x.x # any version of Dynamo 0.3.2+ listed at https://github.com/ai-dynamo/dynamo/releases
# 2. Install CRDs (skip if on shared cluster where CRDs already exist) # 2. Install Platform (CRDs are automatically installed by the chart)
helm fetch https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-crds-${RELEASE_VERSION}.tgz
helm install dynamo-crds dynamo-crds-${RELEASE_VERSION}.tgz --namespace default
# 3. Install Platform
helm fetch https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-${RELEASE_VERSION}.tgz helm fetch https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-${RELEASE_VERSION}.tgz
helm install dynamo-platform dynamo-platform-${RELEASE_VERSION}.tgz --namespace ${NAMESPACE} --create-namespace helm install dynamo-platform dynamo-platform-${RELEASE_VERSION}.tgz --namespace ${NAMESPACE} --create-namespace
``` ```
...@@ -255,10 +251,7 @@ kubectl create secret docker-registry docker-imagepullsecret \ ...@@ -255,10 +251,7 @@ kubectl create secret docker-registry docker-imagepullsecret \
cd deploy/helm/charts cd deploy/helm/charts
# 4. Install CRDs # 4. Install Platform (CRDs are automatically installed by the chart)
helm upgrade --install dynamo-crds ./crds/ --namespace default
# 5. Install Platform
helm dep build ./platform/ helm dep build ./platform/
# To install cluster-wide instead, set NS_RESTRICT_FLAGS="" (empty) or omit that line entirely. # To install cluster-wide instead, set NS_RESTRICT_FLAGS="" (empty) or omit that line entirely.
......
...@@ -164,8 +164,7 @@ For version-specific artifact details, installation commands, and release histor ...@@ -164,8 +164,7 @@ For version-specific artifact details, installation commands, and release histor
- [Kubernetes Operator](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/containers/kubernetes-operator) - [Kubernetes Operator](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/containers/kubernetes-operator)
- **Helm Charts**: [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/collections/ai-dynamo) hosts the helm charts supporting Kubernetes deployments of Dynamo: - **Helm Charts**: [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/collections/ai-dynamo) hosts the helm charts supporting Kubernetes deployments of Dynamo:
- [Dynamo CRDs](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/helm-charts/dynamo-crds) - [Dynamo Platform](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/helm-charts/dynamo-platform) (includes CRDs)
- [Dynamo Platform](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/helm-charts/dynamo-platform)
- [Dynamo Graph](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/helm-charts/dynamo-graph) - [Dynamo Graph](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/helm-charts/dynamo-graph)
- **Rust Crates**: - **Rust Crates**:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment