feat: refactor docker registry secret management in operator (#1337)

deploy/cloud/helm/dynamo-platform-values.yaml

@@ -37,6 +37,7 @@ dynamo-operator:
       gateway: ${ISTIO_GATEWAY}
     ingressHostSuffix: ${DYNAMO_INGRESS_SUFFIX}
     dockerRegistry:
+      useKubernetesSecret: true
       server: ${PIPELINES_DOCKER_SERVER}
       username: ${PIPELINES_DOCKER_USERNAME}
       password: ${PIPELINES_DOCKER_PASSWORD}

deploy/cloud/helm/platform/components/operator/templates/component-serviceaccount.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "dynamo-operator.fullname" . }}-component
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: dynamo-operator
+    app.kubernetes.io/part-of: dynamo-operator
+    nvidia.com/dynamo-component-pod: "true"
+  {{- include "dynamo-operator.labels" . | nindent 4 }}
+  annotations:
+    {{- toYaml .Values.dynamo.components.serviceAccount.annotations | nindent 4 }}
+{{- if .Values.dynamo.dockerRegistry.useKubernetesSecret }}
+imagePullSecrets:
+- name: dynamo-regcred
+{{- end }}

deploy/cloud/helm/platform/components/operator/templates/deployment.yaml

@@ -96,12 +96,15 @@ spec:
         {{- if .Values.dynamo.enableLWS }}
           - --enable-lws
         {{- end }}
-
         command:
         - /manager
         env:
         - name: KUBERNETES_CLUSTER_DOMAIN
           value: {{ quote .Values.kubernetesClusterDomain }}
+        {{if .Values.dynamo.dockerRegistry.useKubernetesSecret}}
+        - name: DOCKER_CONFIG
+          value: /docker/.docker
+        {{end}}
         envFrom:
         - secretRef:
             name: dynamo-deployment-env
@@ -124,6 +127,19 @@ spec:
           10 }}
         securityContext: {{- toYaml .Values.controllerManager.manager.containerSecurityContext
           | nindent 10 }}
+      {{if .Values.dynamo.dockerRegistry.useKubernetesSecret}}
+        volumeMounts:
+        - name: docker-config
+          mountPath: /docker/.docker
+          readOnly: true
+      volumes:
+      - name: docker-config
+        secret:
+          secretName: dynamo-regcred
+          items:
+          - key: .dockerconfigjson
+            path: config.json
+      {{end}}
       securityContext:
         runAsNonRoot: true
       serviceAccountName: {{ include "dynamo-operator.fullname" . }}-controller-manager

deploy/cloud/helm/platform/components/operator/templates/image-builder-serviceaccount.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "dynamo-operator.fullname" . }}-image-builder
+  labels:
+    app.kubernetes.io/component: rbac
+    app.kubernetes.io/created-by: dynamo-operator
+    app.kubernetes.io/part-of: dynamo-operator
+    nvidia.com/dynamo-image-builder-pod: "true"
+  {{- include "dynamo-operator.labels" . | nindent 4 }}
+  annotations:
+    {{- toYaml .Values.dynamo.imageBuilder.serviceAccount.annotations | nindent 4 }}
\ No newline at end of file

deploy/cloud/helm/platform/components/operator/templates/planner.yaml

@@ -18,6 +18,10 @@ kind: ServiceAccount
 metadata:
   name: planner-serviceaccount
   namespace: {{ .Values.namespace }}
+{{- if .Values.dynamo.dockerRegistry.useKubernetesSecret }}
+imagePullSecrets:
+- name: dynamo-regcred
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role

deploy/cloud/helm/platform/components/operator/templates/regcred-secret.yaml

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+{{- if .Values.dynamo.dockerRegistry.useKubernetesSecret }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -21,3 +22,4 @@ metadata:
 type: kubernetes.io/dockerconfigjson
 data:
   .dockerconfigjson: {{ include "dynamo-operator.dockerconfig" . | b64enc }}
+{{- end }}
\ No newline at end of file

deploy/cloud/helm/platform/components/operator/templates/secret-env.yaml

@@ -38,7 +38,9 @@ stringData:
   {{- end }}
 
   DOCKER_REGISTRY_SERVER: {{ .Values.dynamo.dockerRegistry.server | quote }}
+  {{- if .Values.dynamo.dockerRegistry.useKubernetesSecret }}
   DOCKER_REGISTRY_SECRET_NAME: "dynamo-regcred"
+  {{- end }}
   DOCKER_REGISTRY_SECURE: {{ .Values.dynamo.dockerRegistry.secure | quote }}
   DOCKER_REGISTRY_DYNAMO_COMPONENTS_REPOSITORY_NAME: {{ .Values.dynamo.dockerRegistry.dynamoComponentsRepositoryName | quote }}
 

deploy/cloud/helm/platform/components/operator/values.yaml

@@ -73,6 +73,13 @@ controllerManager:
     annotations: {}
 
 dynamo:
+  imageBuilder:
+    serviceAccount:
+      annotations: {}
+  components:
+    serviceAccount:
+      annotations: {}
+
   enableLWS: false
   apiStore:
     endpoint: http://dynamo-server.dynamo-system.svc.cluster.local
@@ -89,7 +96,9 @@ dynamo:
 
   dockerRegistry:
     server: 'nvcr.io/nvidian/nim-llm-dev'
-    inClusterServer: ''
+    # set to true if you want to use the kubernetes secret for the registry credentials
+    # if false, no secret will be created and used. Allows to use cloud provider mechanisms for authentication (e.g. Workload Identity for GKE, ...)
+    useKubernetesSecret: false
     username: '$oauthtoken'
     password: ""
     passwordExistingSecretName: ''

deploy/cloud/helm/platform/values.yaml

@@ -43,8 +43,8 @@ dynamo-operator:
       debugger: python:3.12-slim
     enableRestrictedSecurityContext: false
     dockerRegistry:
+      useKubernetesSecret: false
       server: ""
-      inClusterServer: ""
       username: ""
       password: ""
       secure: true

deploy/cloud/operator/api/dynamo/schemas/schemas.go

@@ -73,8 +73,6 @@ const (
 type DockerRegistrySchema struct {
 	DynamoRepositoryURI string `json:"dynamoRepositoryURI"`
 	Server              string `json:"server"`
-	Username            string `json:"username"`
-	Password            string `json:"password"`
 	SecretName          string `json:"secretName"`
 	Secure              bool   `json:"secure"`
 }

deploy/cloud/operator/go.mod

@@ -8,14 +8,16 @@ require (
 	dario.cat/mergo v1.0.1
 	emperror.dev/errors v0.8.1
 	github.com/apparentlymart/go-shquot v0.0.1
+	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.9.1
 	github.com/bsm/gomega v1.27.10
+	github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589
 	github.com/google/go-cmp v0.7.0
+	github.com/google/go-containerregistry v0.20.5
 	github.com/huandu/xstrings v1.4.0
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/onsi/ginkgo/v2 v2.23.4
 	github.com/onsi/gomega v1.37.0
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.71.2
-	github.com/prune998/docker-registry-client v0.0.0-20200114164314-f8cd511a014c
 	github.com/rs/xid v1.4.0
 	github.com/sergeymakinen/go-quote v1.1.0
 	github.com/sirupsen/logrus v1.9.3
@@ -35,14 +37,41 @@ require (
 )
 
 require (
+	cloud.google.com/go/compute/metadata v0.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go v46.4.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.28 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.36.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.29.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.59 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecr v1.40.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.31.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/distribution/reference v0.6.0 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/docker/cli v28.1.1+incompatible // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
-	github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
+	github.com/docker/docker-credential-helpers v0.9.3 // indirect
 	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
@@ -55,50 +84,58 @@ require (
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.20.2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/vbatts/tar-split v0.12.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.16 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/net v0.37.0 // indirect
-	golang.org/x/oauth2 v0.23.0 // indirect
-	golang.org/x/sync v0.12.0 // indirect
-	golang.org/x/sys v0.32.0 // indirect
-	golang.org/x/term v0.30.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sync v0.14.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
 	golang.org/x/time v0.7.0 // indirect
-	golang.org/x/tools v0.31.0 // indirect
+	golang.org/x/tools v0.33.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
-	google.golang.org/grpc v1.65.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect
+	google.golang.org/grpc v1.72.1 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.5.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect

deploy/cloud/operator/internal/consts/consts.go

@@ -56,7 +56,7 @@ const (
 	KubeLabelValueTrue  = "true"
 
 	KubeLabelDynamoImageBuilderPod = "nvidia.com/dynamo-image-builder-pod"
-	KubeLabelDynamoDeploymentPod   = "nvidia.com/dynamo-deployment-pod"
+	KubeLabelDynamoComponentPod    = "nvidia.com/dynamo-component-pod"
 
 	KubeAnnotationDynamoRepository             = "nvidia.com/dynamo-repository"
 	KubeAnnotationDynamoVersion                = "nvidia.com/dynamo-version"

deploy/cloud/operator/internal/controller/dynamocomponent_controller.go

@@ -23,9 +23,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/md5"
-	"encoding/base64"
 	"encoding/hex"
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -40,9 +38,14 @@ import (
 	commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
 	"github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/controller_common"
 	"github.com/apparentlymart/go-shquot/shquot"
+	"github.com/awslabs/amazon-ecr-credential-helper/ecr-login"
+	"github.com/chrismellard/docker-credential-acr-env/pkg/credhelper"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/google"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/huandu/xstrings"
 	"github.com/mitchellh/hashstructure/v2"
-	"github.com/prune998/docker-registry-client/registry"
 	"github.com/rs/xid"
 	"github.com/sergeymakinen/go-quote/unix"
 	"github.com/sirupsen/logrus"
@@ -69,10 +72,6 @@ import (
 	nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/v1alpha1"
 )
 
-const (
-	dockerConfigSecretKey = ".dockerconfigjson"
-)
-
 // DynamoComponentReconciler reconciles a DynamoComponent object
 type DynamoComponentReconciler struct {
 	client.Client
@@ -236,13 +235,9 @@ func (r *DynamoComponentReconciler) ensureImageExists(ctx context.Context, opt e
 	DynamoComponent = opt.DynamoComponent
 	req := opt.req
 
-	imageInfo, err = r.getImageInfo(ctx, GetImageInfoOption{
+	imageInfo = r.getImageInfo(GetImageInfoOption{
 		DynamoComponent: DynamoComponent,
 	})
-	if err != nil {
-		err = errors.Wrap(err, "get image info")
-		return
-	}
 
 	imageExistsCheckedCondition := meta.FindStatusCondition(DynamoComponent.Status.Conditions, nvidiacomv1alpha1.DynamoComponentConditionTypeImageExistsChecked)
 	imageExistsCondition := meta.FindStatusCondition(DynamoComponent.Status.Conditions, nvidiacomv1alpha1.DynamoComponentConditionTypeImageExists)
@@ -264,7 +259,7 @@ func (r *DynamoComponentReconciler) ensureImageExists(ctx context.Context, opt e
 			return
 		}
 		r.Recorder.Eventf(DynamoComponent, corev1.EventTypeNormal, "CheckingImage", "Checking image exists: %s", imageInfo.ImageName)
-		imageExists, err = checkImageExists(DynamoComponent, imageInfo.DockerRegistry, imageInfo.ImageName)
+		imageExists, err = checkImageExists(DynamoComponent, imageInfo.ImageName)
 		if err != nil {
 			err = errors.Wrapf(err, "check image %s exists", imageInfo.ImageName)
 			return
@@ -594,67 +589,8 @@ func (r *DynamoComponentReconciler) getApiStoreClient(ctx context.Context) (*api
 	return apiStoreClient, apiStoreConf, nil
 }
 
-func (r *DynamoComponentReconciler) RetrieveDockerRegistrySecret(ctx context.Context, secretName string, namespace string, dockerRegistry *schemas.DockerRegistrySchema) error {
-	secret := &corev1.Secret{}
-	err := r.Get(ctx, types.NamespacedName{
-		Namespace: namespace,
-		Name:      secretName,
-	}, secret)
-	if err != nil {
-		err = errors.Wrapf(err, "get docker config json secret %s", secretName)
-		return err
-	}
-	configJSON, ok := secret.Data[dockerConfigSecretKey]
-	if !ok {
-		err = errors.Errorf("docker config json secret %s does not have %s key", secretName, dockerConfigSecretKey)
-		return err
-	}
-	var configObj struct {
-		Auths map[string]struct {
-			Auth string `json:"auth"`
-		} `json:"auths"`
-	}
-	err = json.Unmarshal(configJSON, &configObj)
-	if err != nil {
-		err = errors.Wrapf(err, "unmarshal docker config json secret %s", secretName)
-		return err
-	}
-	var server string
-	var auth string
-	if dockerRegistry.Server != "" {
-		for k, v := range configObj.Auths {
-			if k == dockerRegistry.Server {
-				server = k
-				auth = v.Auth
-				break
-			}
-		}
-		if server == "" {
-			for k, v := range configObj.Auths {
-				if strings.Contains(k, dockerRegistry.Server) {
-					server = k
-					auth = v.Auth
-					break
-				}
-			}
-		}
-	}
-	if server == "" {
-		err = errors.Errorf("no auth in docker config json secret %s for server %s", secretName, dockerRegistry.Server)
-		return err
-	}
-	var credentials []byte
-	credentials, err = base64.StdEncoding.DecodeString(auth)
-	if err != nil {
-		err = errors.Wrapf(err, "cannot base64 decode auth in docker config json secret %s", secretName)
-		return err
-	}
-	dockerRegistry.Username, _, dockerRegistry.Password = xstrings.Partition(string(credentials), ":")
-	return nil
-}
-
 //nolint:nakedret
-func (r *DynamoComponentReconciler) getDockerRegistry(ctx context.Context, DynamoComponent *nvidiacomv1alpha1.DynamoComponent) (dockerRegistry *schemas.DockerRegistrySchema, err error) {
+func (r *DynamoComponentReconciler) getDockerRegistry(DynamoComponent *nvidiacomv1alpha1.DynamoComponent) *schemas.DockerRegistrySchema {
 
 	dockerRegistryConfig := commonconfig.GetDockerRegistryConfig()
 
@@ -668,20 +604,12 @@ func (r *DynamoComponentReconciler) getDockerRegistry(ctx context.Context, Dynam
 		dockerRegistryConfig.SecretName = DynamoComponent.Spec.DockerConfigJSONSecretName
 	}
 
-	dockerRegistry = &schemas.DockerRegistrySchema{
+	return &schemas.DockerRegistrySchema{
 		Server:              dockerRegistryConfig.Server,
 		Secure:              dockerRegistryConfig.Secure,
 		DynamoRepositoryURI: dynamoRepositoryURI,
 		SecretName:          dockerRegistryConfig.SecretName,
 	}
-
-	err = r.RetrieveDockerRegistrySecret(ctx, dockerRegistryConfig.SecretName, DynamoComponent.Namespace, dockerRegistry)
-	if err != nil {
-		err = errors.Wrap(err, "retrieve docker registry secret")
-		return
-	}
-
-	return
 }
 
 func isAddNamespacePrefix() bool {
@@ -725,41 +653,32 @@ func getDynamoComponentImageName(DynamoComponent *nvidiacomv1alpha1.DynamoCompon
 	return fmt.Sprintf("%s:%s", uri, tag)
 }
 
-func checkImageExists(DynamoComponent *nvidiacomv1alpha1.DynamoComponent, dockerRegistry schemas.DockerRegistrySchema, imageName string) (bool, error) {
+func checkImageExists(DynamoComponent *nvidiacomv1alpha1.DynamoComponent, imageName string) (bool, error) {
 	if DynamoComponent.Annotations["nvidia.com/force-build-image"] == commonconsts.KubeLabelValueTrue {
 		return false, nil
 	}
-
-	server, _, imageName := xstrings.Partition(imageName, "/")
-	if strings.Contains(server, "docker.io") {
-		server = "index.docker.io"
-	}
-	if dockerRegistry.Secure {
-		server = fmt.Sprintf("https://%s", server)
-	} else {
-		server = fmt.Sprintf("http://%s", server)
-	}
-	hub, err := registry.New(server, dockerRegistry.Username, dockerRegistry.Password, logrus.Debugf)
+	ref, err := name.ParseReference(imageName)
 	if err != nil {
-		err = errors.Wrapf(err, "create docker registry client for %s", server)
-		return false, err
-	}
-	imageName, _, tag := xstrings.LastPartition(imageName, ":")
-	tags, err := hub.Tags(imageName)
-	isNotFound := err != nil && strings.Contains(err.Error(), "404")
-	if isNotFound {
-		return false, nil
-	}
+		return false, fmt.Errorf("parsing image reference: %w", err)
+	}
+	keychain := authn.NewMultiKeychain(
+		// This picks up auth from DOCKER_CONFIG env var
+		authn.DefaultKeychain,
+		// This picks up auth from GCR
+		google.Keychain,
+		// This picks up auth from ECR
+		authn.NewKeychainFromHelper(ecr.NewECRHelper()),
+		// This picks up auth from ACR
+		authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper()),
+	)
+	_, err = remote.Head(ref, remote.WithAuthFromKeychain(keychain))
 	if err != nil {
-		err = errors.Wrapf(err, "get tags for docker image %s", imageName)
-		return false, err
-	}
-	for _, tag_ := range tags {
-		if tag_ == tag {
-			return true, nil
+		if strings.Contains(err.Error(), "404") {
+			return false, nil
 		}
+		return false, fmt.Errorf("checking image: %w", err)
 	}
-	return false, nil
+	return true, nil
 }
 
 type ImageInfo struct {
@@ -774,20 +693,16 @@ type GetImageInfoOption struct {
 }
 
 //nolint:nakedret
-func (r *DynamoComponentReconciler) getImageInfo(ctx context.Context, opt GetImageInfoOption) (imageInfo ImageInfo, err error) {
+func (r *DynamoComponentReconciler) getImageInfo(opt GetImageInfoOption) ImageInfo {
 	dynamoComponentRepositoryName, _, dynamoComponentVersion := xstrings.Partition(opt.DynamoComponent.Spec.DynamoComponent, ":")
-	dockerRegistry, err := r.getDockerRegistry(ctx, opt.DynamoComponent)
-	if err != nil {
-		err = errors.Wrap(err, "get docker registry")
-		return
-	}
-	imageInfo.DockerRegistry = *dockerRegistry
-	imageInfo.ImageName = getDynamoComponentImageName(opt.DynamoComponent, *dockerRegistry, dynamoComponentRepositoryName, dynamoComponentVersion)
-
-	imageInfo.DockerConfigJSONSecretName = dockerRegistry.SecretName
-
-	imageInfo.DockerRegistryInsecure = opt.DynamoComponent.Annotations[commonconsts.KubeAnnotationDynamoDockerRegistryInsecure] == "true"
-	return
+	dockerRegistry := r.getDockerRegistry(opt.DynamoComponent)
+	imageInfo := ImageInfo{
+		DockerRegistry:             *dockerRegistry,
+		ImageName:                  getDynamoComponentImageName(opt.DynamoComponent, *dockerRegistry, dynamoComponentRepositoryName, dynamoComponentVersion),
+		DockerConfigJSONSecretName: dockerRegistry.SecretName,
+		DockerRegistryInsecure:     opt.DynamoComponent.Annotations[commonconsts.KubeAnnotationDynamoDockerRegistryInsecure] == "true",
+	}
+	return imageInfo
 }
 
 func (r *DynamoComponentReconciler) getImageBuilderJobName() string {
@@ -1192,16 +1107,19 @@ echo "Done"
 
 	builderContainerEnvFrom := make([]corev1.EnvFromSource, 0)
 	builderContainerEnvs := []corev1.EnvVar{
-		{
-			Name:  "DOCKER_CONFIG",
-			Value: "/kaniko/.docker/",
-		},
 		{
 			Name:  "IFS",
 			Value: "''",
 		},
 	}
 
+	if dockerConfigJSONSecretName != "" {
+		builderContainerEnvs = append(builderContainerEnvs, corev1.EnvVar{
+			Name:  "DOCKER_CONFIG",
+			Value: "/kaniko/.docker/",
+		})
+	}
+
 	kanikoCacheRepo := os.Getenv("KANIKO_CACHE_REPO")
 	if kanikoCacheRepo == "" {
 		kanikoCacheRepo = opt.ImageInfo.DockerRegistry.DynamoRepositoryURI

deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller.go

@@ -38,7 +38,6 @@ import (
 	dynamoCommon "github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/common"
 	"github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/schemas"
 	"github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/v1alpha1"
-	"github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/config"
 	commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
 	"github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/controller_common"
 	commonController "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/controller_common"
@@ -1697,17 +1696,18 @@ func (r *DynamoComponentDeploymentReconciler) generatePodTemplateSpec(ctx contex
 		Volumes:    volumes,
 	}
 
-	podSpec.ImagePullSecrets = []corev1.LocalObjectReference{
-		{
-			Name: config.GetDockerRegistryConfig().SecretName,
-		},
-	}
+	imagePullSecrets := []corev1.LocalObjectReference{}
+
 	if opt.dynamoComponent.Spec.DockerConfigJSONSecretName != "" {
-		podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, corev1.LocalObjectReference{
+		imagePullSecrets = append(imagePullSecrets, corev1.LocalObjectReference{
 			Name: opt.dynamoComponent.Spec.DockerConfigJSONSecretName,
 		})
 	}
-	podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, opt.dynamoComponent.Spec.ImagePullSecrets...)
+	imagePullSecrets = append(imagePullSecrets, opt.dynamoComponent.Spec.ImagePullSecrets...)
+
+	if len(imagePullSecrets) > 0 {
+		podSpec.ImagePullSecrets = imagePullSecrets
+	}
 
 	extraPodMetadata := opt.dynamoComponentDeployment.Spec.ExtraPodMetadata
 
@@ -1736,7 +1736,7 @@ func (r *DynamoComponentDeploymentReconciler) generatePodTemplateSpec(ctx contex
 	if podSpec.ServiceAccountName == "" {
 		serviceAccounts := &corev1.ServiceAccountList{}
 		err = r.List(ctx, serviceAccounts, client.InNamespace(opt.dynamoComponentDeployment.Namespace), client.MatchingLabels{
-			commonconsts.KubeLabelDynamoDeploymentPod: commonconsts.KubeLabelValueTrue,
+			commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue,
 		})
 		if err != nil {
 			err = errors.Wrapf(err, "failed to list service accounts in namespace %s", opt.dynamoComponentDeployment.Namespace)

deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller_test.go

@@ -893,7 +893,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 							Name:      "default-test-sa", // Name it will be resolved to
 							Namespace: "default",         // Must match dynamoComponentDeployment.Namespace
 							Labels: map[string]string{
-								commonconsts.KubeLabelDynamoDeploymentPod: commonconsts.KubeLabelValueTrue,
+								commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue,
 							},
 						},
 					},
@@ -964,8 +964,8 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 									},
 								},
 								Volumes:            []corev1.Volume{{Name: "shared-memory", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: limit}}}},
-								ImagePullSecrets:   []corev1.LocalObjectReference{{Name: ""}}, // Assuming default config gives empty secret name
-								ServiceAccountName: "default-test-sa",                         // Updated to reflect mocked SA
+								ImagePullSecrets:   nil,               // Assuming default config gives empty secret name
+								ServiceAccountName: "default-test-sa", // Updated to reflect mocked SA
 							},
 						},
 						WorkerTemplate: corev1.PodTemplateSpec{
@@ -1002,7 +1002,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 									},
 								},
 								Volumes:            []corev1.Volume{{Name: "shared-memory", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: limit}}}},
-								ImagePullSecrets:   []corev1.LocalObjectReference{{Name: ""}},
+								ImagePullSecrets:   nil,
 								ServiceAccountName: "default-test-sa", // Updated to reflect mocked SA
 							},
 						},
@@ -1043,7 +1043,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 					&corev1.ServiceAccount{
 						ObjectMeta: metav1.ObjectMeta{
 							Name: "default-test-sa", Namespace: "default", // Match namespace
-							Labels: map[string]string{commonconsts.KubeLabelDynamoDeploymentPod: commonconsts.KubeLabelValueTrue},
+							Labels: map[string]string{commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue},
 						},
 					},
 				},
@@ -1084,7 +1084,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 					&corev1.ServiceAccount{
 						ObjectMeta: metav1.ObjectMeta{
 							Name: "default-test-sa", Namespace: "default", // Match namespace
-							Labels: map[string]string{commonconsts.KubeLabelDynamoDeploymentPod: commonconsts.KubeLabelValueTrue},
+							Labels: map[string]string{commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue},
 						},
 					},
 				},