ci: Accelerate dynamo frontend image build (#6479)

.github/actions/docker-remote-build/action.yml

@@ -19,9 +19,6 @@ inputs:
   image_tag:
     description: 'Custom image tag'
     required: true
-  ci_token:
-    description: 'CI Token'
-    required: false
   aws_default_region:
     description: 'AWS Default Region'
     required: false
@@ -56,10 +53,10 @@ inputs:
     description: 'Use SCCache for caching'
     required: false
     default: 'true'
-  ci:
-    description: 'CI mode: for frontend target, uses existing buildx builder and pushes EPP image to ECR'
+  extra_build_args:
+    description: 'Additional Docker build args (newline-separated list of KEY=VALUE pairs)'
     required: false
-    default: 'false'
+    default: ''
 outputs:
   image_tag:
     description: 'Image Tag'
@@ -72,7 +69,6 @@ runs:
       id: build
       shell: bash
       env:
-        GITHUB_TOKEN: ${{ inputs.ci_token }}
         AWS_DEFAULT_REGION: ${{ inputs.aws_default_region }}
         SCCACHE_S3_BUCKET:  ${{ inputs.sccache_s3_bucket }}
         AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
@@ -104,27 +100,27 @@ runs:
 
         # Collect optional overrides provided by the workflow
         # Set base cache args and set --cache-to if this is a main commit
-        # TODO: Fix this - Skip cache for frontend target - a different docker driver is used for the EPP build, which causes issues with cache export
         CACHE_ARGS=""
         if [[ "${{ inputs.target }}" != "frontend" ]]; then
-          CACHE_ARGS="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:${{ inputs.framework }}-cuda${CUDA_VERSION_MAJOR}-${PLATFORM}-cache "
-          CACHE_ARGS+="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${{ inputs.framework }}-cuda${CUDA_VERSION_MAJOR}-${PLATFORM} "
-          CACHE_ARGS+="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${{ inputs.framework }}-cuda${CUDA_VERSION_MAJOR}-${PLATFORM}-cache "
+          CACHE_TAG="${{ inputs.framework }}-cuda${CUDA_VERSION_MAJOR}-${PLATFORM}-cache"
+          CACHE_ARGS="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG} "
+          CACHE_ARGS+="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG} "
           if [[ "$GITHUB_REF_NAME" =~ ^release ]]; then
             # Release branches also use release cache
-            CACHE_ARGS+="--cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${{ inputs.framework }}-cuda${CUDA_VERSION_MAJOR}-${PLATFORM}-cache,mode=max "
+            CACHE_ARGS+="--cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG},mode=max "
           elif [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            CACHE_ARGS+="--cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:${{ inputs.framework }}-cuda${CUDA_VERSION_MAJOR}-${PLATFORM}-cache,mode=max "
+            CACHE_ARGS+="--cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG},mode=max "
+          fi
+        else
+          CACHE_TAG="${{ inputs.target }}-cuda${CUDA_VERSION_MAJOR}-${PLATFORM}-cache"
+          CACHE_ARGS="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG} "
+          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
+            CACHE_ARGS+="--cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG},mode=max "
           fi
         fi
+
         echo "$CACHE_ARGS"
         # Collect optional overrides provided by the workflow
-
-        if [[ "${{ inputs.ci }}" == "true" ]]; then
-          # CI mode for frontend: use existing buildx builder, push EPP to registry
-          EXTRA_ARGS+=" --ci"
-        fi
-
         if [ "${{ inputs.no_cache }}" == "true" ]; then
           EXTRA_ARGS+=" --no-cache"
         fi
@@ -149,6 +145,16 @@ runs:
           done <<< "$EXTRA_TAGS"
         fi
 
+        # Add extra build args (each as a separate --build-arg)
+        EXTRA_BUILD_ARGS="${{ inputs.extra_build_args }}"
+        if [ -n "$EXTRA_BUILD_ARGS" ]; then
+          while IFS= read -r BUILD_ARG; do
+            if [ -n "$BUILD_ARG" ]; then
+              EXTRA_ARGS+=" --build-arg ${BUILD_ARG}"
+            fi
+          done <<< "$EXTRA_BUILD_ARGS"
+        fi
+
         # Pass AWS credentials as build secrets for sccache S3 access.
         # Dockerfile steps reference these via --mount=type=secret,id=aws-key-id,env=...
         SECRET_ARGS=""
@@ -162,7 +168,7 @@ runs:
           --tag "$IMAGE_TAG" \
           --platform linux/${{ inputs.platform }} \
           -f ./container/rendered.Dockerfile \
-          $CACHE_ARGS $EXTRA_ARGS $SECRET_ARGS $EPP_IMAGE_ARG . 2>&1 | tee "${BUILD_LOG_FILE}"
+          $CACHE_ARGS $EXTRA_ARGS $SECRET_ARGS . 2>&1 | tee "${BUILD_LOG_FILE}"
 
         BUILD_EXIT_CODE=${PIPESTATUS[0]}
 

.github/workflows/build-frontend-image.yaml

@@ -25,8 +25,6 @@ on:
         required: true
       AZURE_ACR_PASSWORD:
         required: true
-      CI_TOKEN:
-        required: true
       SCCACHE_S3_BUCKET:
         required: true
 
@@ -34,11 +32,15 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name == 'main' && github.run_id || github.ref_name }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
+env:
+  BUILDER_NAME: b-${{ github.run_id }}-${{ github.run_attempt }}
+
 jobs:
   changed-files:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
     outputs:
       frontend: ${{ steps.changes.outputs.frontend }}
+      builder_name: ${{ steps.export-builder-name.outputs.builder_name }}
     steps:
       - name: Checkout code
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
@@ -49,25 +51,114 @@ jobs:
         uses: ./.github/actions/changed-files
         with:
           gh_token: ${{ github.token }}
+      - name: Export builder name
+        id: export-builder-name
+        run: |
+          echo "builder_name=${{ env.BUILDER_NAME }}" >> $GITHUB_OUTPUT
 
-  build-frontend-image:
-    name: Build Frontend Image (${{ matrix.platform.arch }})
+  build-epp-image:
+    name: Build EPP Image
     needs: changed-files
     if: needs.changed-files.outputs.frontend == 'true'
+    runs-on: prod-builder-v3
+    outputs:
+      epp_image_ref: ${{ steps.build-epp-image.outputs.epp_image_ref }}
+    env:
+      ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
+      IMAGE_REGISTRY: ai-dynamo
+      IMAGE_REPOSITORY: dynamo/dynamo-epp
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
+      - name: Initialize Dynamo Builder
+        uses: ./.github/actions/init-dynamo-builder
+        with:
+          builder_name: ${{ needs.changed-files.outputs.builder_name }}
+          flavor: general
+          all_arch: 'true'
+      - name: Docker Login
+        uses: ./.github/actions/docker-login
+        with:
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
+          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
+          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+      - name: Build EPP Image
+        id: build-epp-image
+        shell: bash
+        timeout-minutes: 20
+        run: |
+          set -x
+          EPP_IMAGE_REPO="${{ env.ECR_HOSTNAME }}/${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_REPOSITORY }}"
+          EPP_IMAGE_TAG="${{ github.sha }}"
+          EPP_IMAGE_REF="${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_REPOSITORY }}:${EPP_IMAGE_TAG}"
+          echo "epp_image_ref=${EPP_IMAGE_REF}" >> $GITHUB_OUTPUT
+
+          CACHE_ARGS=""
+          CACHE_ARGS+="--cache-from type=registry,ref=${ECR_HOSTNAME}/${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:cache "
+          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
+            CACHE_ARGS+="--cache-to type=registry,ref=${ECR_HOSTNAME}/${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}:cache,mode=max "
+          fi
+
+          make -C deploy/inference-gateway/epp image-multiarch-push \
+            IMAGE_REPO="${EPP_IMAGE_REPO}" \
+            GIT_TAG="${EPP_IMAGE_TAG}" \
+            DOCKER_PROXY="${ECR_HOSTNAME}/dockerhub/" \
+            EXTRA_BUILD_ARGS="${CACHE_ARGS}"
+
+
+  build-frontend-image:
+    name: Build Frontend Image
+    needs: [changed-files, build-epp-image]
+    if: needs.changed-files.outputs.frontend == 'true'
     strategy:
       fail-fast: false
       matrix:
-        platform:
-          - { arch: amd64, runner: prod-builder-amd-v1  }
-          - { arch: arm64, runner: prod-builder-arm-v1 }
-    runs-on: ${{ matrix.platform.runner }}
+        arch: [amd64, arm64]
+    runs-on: prod-builder-v3
+    env:
+      IMAGE_REGISTRY: ai-dynamo
+      IMAGE_REPOSITORY: dynamo
+      ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
+      TARGET: frontend
+      CUDA_VERSION: "12.9"
+      FRAMEWORK: dynamo
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.24'
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
+      - name: Generate Dockerfile
+        shell: bash
+        run: |
+          echo "Generating Dockerfile for target: ${{ env.TARGET }} and framework: ${{ env.FRAMEWORK }} "
+          python ./container/render.py \
+              --target=${{ env.TARGET }} \
+              --framework=${{ env.FRAMEWORK }} \
+              --platform=${{ matrix.arch }} \
+              --cuda-version=${{ env.CUDA_VERSION }} \
+              --show-result \
+              --output-short-filename
+      - name: Calculate target tag
+        id: calculate-target-tag
+        shell: bash
+        run: |
+          TARGET_TAG="${{ github.sha }}-${{ env.TARGET }}-${{ matrix.arch }}"
+          DEFAULT_TARGET_IMAGE_URI="${{ env.ECR_HOSTNAME }}/${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_REPOSITORY }}:${TARGET_TAG}"
+          echo "default_target_image_uri=${DEFAULT_TARGET_IMAGE_URI}" >> $GITHUB_OUTPUT
+          echo "epp_image_uri=${{ env.ECR_HOSTNAME }}/${{ needs.build-epp-image.outputs.epp_image_ref }}" >> $GITHUB_OUTPUT
+          echo "target_tag=${TARGET_TAG}" >> $GITHUB_OUTPUT
+          echo "azure_target_image_uri=${{ secrets.AZURE_ACR_HOSTNAME }}/${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_REPOSITORY }}:${TARGET_TAG}" >> $GITHUB_OUTPUT
+      - name: Print Build Container parameters
+        run: |
+          echo "=== Build Container Parameters ==="
+          echo "image_tag: ${{ steps.calculate-target-tag.outputs.default_target_image_uri }}"
+          echo "framework: ${{ env.FRAMEWORK }}"
+          echo "target: ${{ env.TARGET }}"
+          echo "platform: ${{ matrix.arch }}"
+          echo "cuda_version: ${{ env.CUDA_VERSION }}"
+          echo "extra_tags: ${{ steps.calculate-target-tag.outputs.azure_target_image_uri }}"
+          echo "epp_image_uri: ${{ steps.calculate-target-tag.outputs.epp_image_uri }}"
+          echo "push_image: true"
       - name: Docker Login
         uses: ./.github/actions/docker-login
         with:
@@ -76,41 +167,70 @@ jobs:
           azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
           azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
           azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+      - name: Initialize Dynamo Builder
+        uses: ./.github/actions/init-dynamo-builder
+        with:
+          builder_name: ${{ needs.changed-files.outputs.builder_name }}
+          flavor: general
+          arch: ${{ matrix.arch }}
       - name: Build Frontend Container
         id: build-image
-        uses: ./.github/actions/docker-build
-        env:
-          PLATFORMS: ${{ matrix.platform.arch }}
-          TARGETARCH: ${{ matrix.platform.arch }}
+        timeout-minutes: 30
+        uses: ./.github/actions/docker-remote-build
         with:
-          framework: dynamo
-          target: frontend
-          platform: ${{ env.PLATFORMS }}
-          ci_token: ${{ secrets.CI_TOKEN }}
-          cuda_version: "12.9"
+          image_tag: ${{ steps.calculate-target-tag.outputs.default_target_image_uri }}
+          framework: ${{ env.FRAMEWORK }}
+          target: ${{ env.TARGET }}
+          platform: ${{ matrix.arch }}
+          cuda_version: ${{ env.CUDA_VERSION }}
           aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
           sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }}
           aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - name: Docker Tag and Push Frontend Image
-        uses: ./.github/actions/docker-tag-push
-        with:
-          local_image: ${{ steps.build-image.outputs.image_tag }}
-          push_tags: ai-dynamo/dynamo:${{ github.sha }}-frontend-${{ matrix.platform.arch }}
-          aws_push: 'true'
-          azure_push: 'true'
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
-          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+          push_image: true
+          extra_build_args: |
+            EPP_IMAGE=${{ steps.calculate-target-tag.outputs.epp_image_uri }}
+          extra_tags: |
+            ${{ steps.calculate-target-tag.outputs.azure_target_image_uri }}
+            ${{ github.ref_name == 'main' && format('main-frontend-{0}', matrix.arch) || '' }}
+            ${{ github.ref_name == 'main' && format('main-frontend-{0}-{1}', github.sha, matrix.arch) || '' }}
+
+      - name: Show summary
+        shell: bash
+        run: |
+          echo "### 🐳 ${{ env.FRAMEWORK }}-cuda${{ env.CUDA_VERSION }}-${{ matrix.arch }} Default Image" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Image URI |" >> $GITHUB_STEP_SUMMARY
+          echo "|-----|" >> $GITHUB_STEP_SUMMARY
+          echo "| \`${{ steps.calculate-target-tag.outputs.default_target_image_uri }}\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| \`${{ steps.calculate-target-tag.outputs.azure_target_image_uri }}\` |" >> $GITHUB_STEP_SUMMARY
 
   frontend-status-check:
     runs-on: ubuntu-latest
-    needs: [changed-files, build-frontend-image]
+    needs: [changed-files, build-frontend-image, build-epp-image]
     if: always()
     steps:
       - name: "Check all dependent jobs"
         run: |
           echo '${{ toJson(needs) }}' | jq -e 'to_entries | map(.value.result) | all(. as $result | ["success", "skipped"] | any($result == .))'
+
+  clean-k8s-builder:
+    name: Clean K8s builder if exists
+    runs-on: prod-default-small-v2
+    if: always()
+    needs: [build-frontend-image, changed-files]
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
+    - name: Create K8s builders (skip bootstrap)
+      uses: ./.github/actions/bootstrap-buildkit
+      continue-on-error: true
+      with:
+        builder_name: ${{ needs.changed-files.outputs.builder_name }}
+        buildkit_worker_addresses: '' # k8s builder
+        skip_bootstrap: true
+    - name: Builder Cleanup in case of k8s builder
+      shell: bash
+      run: |
+        docker buildx rm ${{ needs.changed-files.outputs.builder_name }} || true

deploy/inference-gateway/epp/Dockerfile

@@ -68,7 +68,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
     cargo build --release -p libdynamo_llm && \
     mkdir -p /out && \
     cp target/release/libdynamo_llm_capi.a /out/ && \
-    cp lib/bindings/c/include/nvidia/dynamo_llm/llm_engine.h /out/
+    HEADER=$(find target/release/build -name llm_engine.h -path "*/out/*" | head -1) && \
+    [ -n "$HEADER" ] && cp "$HEADER" /out/ || { echo "ERROR: llm_engine.h not found in target/"; exit 1; }
 
 # =============================================================================
 # Stage 2: Build Go EPP binary

deploy/inference-gateway/epp/Makefile

@@ -25,6 +25,7 @@ endif
 MULTIARCH_PLATFORMS ?= linux/amd64,linux/arm64
 # Docker proxy for avoiding rate limits (e.g., ECR mirror)
 DOCKER_PROXY ?=
+EXTRA_BUILD_ARGS ?=
 
 DOCKER_BUILDX_CMD ?= docker buildx
 IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build
@@ -89,7 +90,7 @@ image-build: ## Build the Docker image (self-contained, no host prerequisites)
 		--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
 		--build-arg COMMIT_SHA=$(GIT_COMMIT_SHA) \
 		--build-arg BUILD_REF=$(GIT_TAG) \
-		$(PUSH) $(LOAD) .
+		$(EXTRA_BUILD_ARGS) $(PUSH) $(LOAD) .
 
 .PHONY: image-push
 image-push: PUSH=--push ## Build and push the Docker image
@@ -115,7 +116,7 @@ image-multiarch: ## Build multi-arch image (requires --push ; --load not support
 		--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
 		--build-arg COMMIT_SHA=$(GIT_COMMIT_SHA) \
 		--build-arg BUILD_REF=$(GIT_TAG) \
-		$(PUSH) .
+		$(EXTRA_BUILD_ARGS) $(PUSH) .
 
 .PHONY: image-multiarch-push
 image-multiarch-push: PUSH=--push ## Build and push multi-arch image to registry

lib/bindings/c/build.rs

@@ -2,18 +2,27 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::env;
+use std::fs;
 use std::path::Path;
 
 fn main() {
     let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+    let out_dir = env::var("OUT_DIR").unwrap();
 
-    let header_path = Path::new(&crate_dir)
+    let bindings = cbindgen::generate(&crate_dir).expect("Unable to generate bindings");
+
+    // Primary output: write to OUT_DIR (inside target/) so the header survives
+    // Docker BuildKit cache mounts across rebuilds.
+    let out_dir_header = Path::new(&out_dir).join("llm_engine.h");
+    bindings.write_to_file(&out_dir_header);
+
+    // Convenience copy: write to source tree for local development workflows
+    // (e.g. `make build` which expects the header under the crate directory).
+    let src_tree_header = Path::new(&crate_dir)
         .join("include")
         .join("nvidia")
         .join("dynamo_llm")
         .join("llm_engine.h");
-
-    cbindgen::generate(crate_dir)
-        .expect("Unable to generate bindings")
-        .write_to_file(header_path);
+    fs::create_dir_all(src_tree_header.parent().unwrap()).ok();
+    fs::copy(&out_dir_header, &src_tree_header).ok();
 }