feat: add option to configure separate docker registry for pipelines docker images (#744)

deploy/dynamo/helm/deploy.sh

@@ -26,24 +26,32 @@ export RELEASE_NAME="${RELEASE_NAME:=${NAMESPACE}}"  # Default release name is s
 export DOCKER_USERNAME="${DOCKER_USERNAME:=<your-docker-username>}"  # Default docker username
 export DOCKER_PASSWORD="${DOCKER_PASSWORD:=<your-docker-password>}"  # Default docker password
 export DOCKER_SERVER="${DOCKER_SERVER:=<your-docker-server>}"  # Default docker server
+export PIPELINES_DOCKER_SERVER="${PIPELINES_DOCKER_SERVER:=${DOCKER_SERVER}}"
+export PIPELINES_DOCKER_USERNAME="${PIPELINES_DOCKER_USERNAME:=${DOCKER_USERNAME}}"
+export PIPELINES_DOCKER_PASSWORD="${PIPELINES_DOCKER_PASSWORD:=${DOCKER_PASSWORD}}"
 export IMAGE_TAG="${IMAGE_TAG:=latest}"  # Default image tag
 export DYNAMO_INGRESS_SUFFIX="${DYNAMO_INGRESS_SUFFIX:=dynamo-cloud.com}"
-
+export DOCKER_SECRET_NAME="${DOCKER_SECRET_NAME:=docker-imagepullsecret}"
 # Check if required variables are set
-if [ "$DOCKER_USERNAME" = "<your-docker-username>" ]; then
-    echo "Error: Please set your DOCKER_USERNAME in the script or via environment variable"
+if [ "$DOCKER_SERVER" = "<your-docker-server>" ]; then
+    echo "Error: Please set your DOCKER_SERVER in the script or via environment variable"
     exit 1
 fi
 
-if [ "$DOCKER_PASSWORD" = "<your-docker-password>" ]; then
-    echo "Error: Please set your DOCKER_PASSWORD in the script or via environment variable"
-    exit 1
+# Creates a docker registry secret. Only proceed if both username and password are set
+if [[ -n "${DOCKER_USERNAME:-}" && -n "${DOCKER_PASSWORD:-}" ]]; then
+  echo "Creating/updating Docker registry secret '$DOCKER_SECRET_NAME' in namespace '$NAMESPACE'..."
+
+  kubectl create secret docker-registry "$DOCKER_SECRET_NAME" \
+    --docker-username="$DOCKER_USERNAME" \
+    --docker-password="$DOCKER_PASSWORD" \
+    --docker-server="$DOCKER_SERVER" \
+    --namespace "$NAMESPACE" \
+    --dry-run=client -o yaml | kubectl apply -f -
+else
+  echo "DOCKER_USERNAME and/or DOCKER_PASSWORD not set — skipping docker secret creation."
 fi
 
-if [ "$DOCKER_SERVER" = "<your-docker-server>" ]; then
-    echo "Error: Please set your DOCKER_SERVER in the script or via environment variable"
-    exit 1
-fi
 
 # Function to retry commands
 retry_command() {
@@ -87,11 +95,14 @@ echo "IMAGE_TAG: $IMAGE_TAG"
 echo "DOCKER_USERNAME: $DOCKER_USERNAME"
 echo "DOCKER_SERVER: $DOCKER_SERVER"
 echo "DOCKER_PASSWORD: [HIDDEN]"
+echo "PIPELINES_DOCKER_SERVER: $PIPELINES_DOCKER_SERVER"
+echo "PIPELINES_DOCKER_USERNAME: $PIPELINES_DOCKER_USERNAME"
+echo "PIPELINES_DOCKER_PASSWORD: [HIDDEN]"
+echo "DOCKER_SECRET_NAME: $DOCKER_SECRET_NAME"
 
+envsubst '${NAMESPACE} ${RELEASE_NAME} ${DOCKER_USERNAME} ${DOCKER_PASSWORD} ${DOCKER_SERVER} ${IMAGE_TAG} ${DYNAMO_INGRESS_SUFFIX} ${PIPELINES_DOCKER_SERVER} ${PIPELINES_DOCKER_USERNAME} ${PIPELINES_DOCKER_PASSWORD} ${DOCKER_SECRET_NAME}' < dynamo-platform-values.yaml > generated-values.yaml
 echo "generated file contents:"
-envsubst '${NAMESPACE} ${RELEASE_NAME} ${DOCKER_USERNAME} ${DOCKER_PASSWORD} ${DOCKER_SERVER} ${IMAGE_TAG} ${DYNAMO_INGRESS_SUFFIX}' < dynamo-platform-values.yaml
-
-envsubst '${NAMESPACE} ${RELEASE_NAME} ${DOCKER_USERNAME} ${DOCKER_PASSWORD} ${DOCKER_SERVER} ${IMAGE_TAG} ${DYNAMO_INGRESS_SUFFIX}' < dynamo-platform-values.yaml > generated-values.yaml
+cat generated-values.yaml
 
 echo ""
 echo "Generated values file saved as generated-values.yaml"

deploy/dynamo/helm/dynamo-platform-values.yaml

@@ -13,10 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Global values used across charts
-global:
-  NGC_API_KEY: ${DOCKER_PASSWORD}
-
 dynamo-operator:
   natsAddr: "nats://${RELEASE_NAME}-nats:4222"
   etcdAddr: "${RELEASE_NAME}-etcd:2379"
@@ -29,13 +25,15 @@ dynamo-operator:
       image:
         repository: ${DOCKER_SERVER}/dynamo-operator
         tag: ${IMAGE_TAG}
+  imagePullSecrets:
+    - name: ${DOCKER_SECRET_NAME}
 
   dynamo:
     dynamoIngressSuffix: ${DYNAMO_INGRESS_SUFFIX}
     dockerRegistry:
-      server: ${DOCKER_SERVER}
-      username: ${DOCKER_USERNAME}
-      password: ${DOCKER_PASSWORD}
+      server: ${PIPELINES_DOCKER_SERVER}
+      username: ${PIPELINES_DOCKER_USERNAME}
+      password: ${PIPELINES_DOCKER_PASSWORD}
 
 dynamo-api-store:
   namespaceRestriction:
@@ -48,7 +46,7 @@ dynamo-api-store:
     tag: ${IMAGE_TAG}
     pullPolicy: IfNotPresent
   imagePullSecrets:
-    - name: docker-imagepullsecret
+    - name: ${DOCKER_SECRET_NAME}
   ingress:
     hosts:
       - host: ${NAMESPACE}.${DYNAMO_INGRESS_SUFFIX}

deploy/dynamo/helm/platform/components/operator/templates/_helpers.tpl

@@ -94,7 +94,7 @@ Generate docker config json for registry credentials
 {{- define "dynamo-operator.dockerconfig" -}}
 {{- $server := .Values.dynamo.dockerRegistry.server -}}
 {{- $username := .Values.dynamo.dockerRegistry.username -}}
-{{- $password := default .Values.global.NGC_API_KEY .Values.dynamo.dockerRegistry.password -}}
+{{- $password := .Values.dynamo.dockerRegistry.password -}}
 {{- if .Values.dynamo.dockerRegistry.passwordExistingSecretName -}}
   {{- $secretName := .Values.dynamo.dockerRegistry.passwordExistingSecretName -}}
   {{- $secretKey := .Values.dynamo.dockerRegistry.passwordExistingSecretKey -}}
@@ -119,7 +119,7 @@ Generate docker config json for registry credentials
     {{- end -}}
   {{- else -}}
     {{/* If no secret is found, use the default password */}}
-    {{- $password = .Values.dynamo.dockerRegistry.password | default .Values.global.NGC_API_KEY }}
+    {{- $password = .Values.dynamo.dockerRegistry.password }}
     {
       "auths": {
         "{{ $server }}": {
@@ -151,7 +151,7 @@ Extract username and password from docker registry configuration
 {{- define "dynamo-operator.extractDockerCredentials" -}}
 {{- $server := .Values.dynamo.dockerRegistry.server -}}
 {{- $username := .Values.dynamo.dockerRegistry.username -}}
-{{- $password := default .Values.global.NGC_API_KEY .Values.dynamo.dockerRegistry.password -}}
+{{- $password := .Values.dynamo.dockerRegistry.password -}}
 {{- $result := dict "username" $username "password" $password }}
 
 {{- if .Values.dynamo.dockerRegistry.passwordExistingSecretName }}

deploy/dynamo/helm/platform/components/operator/values.yaml

@@ -16,10 +16,6 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-global:
-  # NGC API Key to use as default for docker registry password
-  NGC_API_KEY: ""
-
 # Namespace restriction configuration for the operator
 # If enabled: true and targetNamespace is empty, the operator will be restricted to the release namespace
 # If enabled: true and targetNamespace is set, the operator will be restricted to the specified namespace
@@ -103,12 +99,11 @@ dynamo:
     server: 'nvcr.io/nvidian/nim-llm-dev'
     inClusterServer: ''
     username: '$oauthtoken'
-    # If not set, will use global.NGC_API_KEY
     password: ""
     passwordExistingSecretName: ''
     passwordExistingSecretKey: ''
     secure: true
-    bentoRepositoryName: yatai-bentos
+    bentoRepositoryName: dynamo-pipelines
 
   bentoImageBuildEngine: buildkit # options: kaniko, buildkit, buildkit-rootless
   addNamespacePrefixToImageName: false

deploy/dynamo/helm/platform/values.yaml

@@ -60,7 +60,7 @@ dynamo-operator:
       username: ""
       password: ""
       secure: true
-      bentoRepositoryName: yatai-bentos
+      bentoRepositoryName: dynamo-pipelines
     bentoImageBuildEngine: buildkit
     addNamespacePrefixToImageName: false
     estargz:

docs/guides/dynamo_deploy/dynamo_cloud.md

@@ -121,18 +121,26 @@ export IMAGE_TAG=<TAG>  # Use the same tag you used when building the images
 export NAMESPACE=dynamo-cloud    # change this to whatever you want!
 ```
 
-2. [One-time Action] Create a new kubernetes namespace and set it as your default. Create image pull secrets if needed.
+> [!NOTE]
+> DOCKER_USERNAME and DOCKER_PASSWORD are optional and only needed if you want to pull docker images from a private registry.
+> A docker image pull secret will be created automatically if these variables are set. Its name will be `docker-imagepullsecret` unless overridden by the `DOCKER_SECRET_NAME` environment variable.
+
+The Dynamo Cloud Platform auto-generates docker images for pipelines and pushes them to a container registry.
+By default, the platform will use the same container registry as the platform components (specified by `DOCKER_SERVER`).
+However, you can specify a different container registry for pipelines by additionally setting the following environment variables:
+
+```bash
+export PIPELINES_DOCKER_SERVER=<your-docker-server>
+export PIPELINES_DOCKER_USERNAME=<your-docker-username>
+export PIPELINES_DOCKER_PASSWORD=<your-docker-password>
+```
+
+2. [One-time Action] Create a new kubernetes namespace and set it as your default.
 
 ```bash
 cd deploy/dynamo/helm
 kubectl create namespace $NAMESPACE
 kubectl config set-context --current --namespace=$NAMESPACE
-
-kubectl create secret docker-registry docker-imagepullsecret \
-  --docker-server=$DOCKER_SERVER \
-  --docker-username=$DOCKER_USERNAME \
-  --docker-password=$DOCKER_PASSWORD \
-  --namespace=$NAMESPACE
 ```
 
 3. Deploy the helm chart using the deploy script:

docs/guides/dynamo_deploy/operator_deployment.md

@@ -61,6 +61,7 @@ dynamo cloud login --api-token TEST-TOKEN --endpoint $DYNAMO_CLOUD
 
 > [!NOTE]
 > For instructions on building and pushing the Dynamo base image, see the [Building the Dynamo Base Image](../../README.md#building-the-dynamo-base-image) section in the main README.
+> The Dynamo base image must be pushed to the same registry as the pipelines images configured in the [Dynamo Cloud](dynamo_cloud.md) deployment.
 
 ```bash
 # Set your runtime image name
@@ -70,7 +71,7 @@ export DYNAMO_IMAGE=<dynamo_docker_image_name>
 cd $PROJECT_ROOT/examples/hello_world
 
 # Build the service and capture the tag
-DYNAMO_TAG=$(dynamo build hello_world:Frontend | grep "Successfully built" | awk -F"\"" '{ print $2 }')
+DYNAMO_TAG=$(dynamo build hello_world:Frontend | grep "Successfully built" | awk '{ print $3 }' | sed 's/\.$//')
 ```
 
 ### 3. Deploy to Kubernetes