ci: switch EPP rust build from target cache mount to sccache (#8604)

Signed-off-by: Anant Sharma <anants@nvidia.com>

.github/workflows/build-frontend-image.yaml

@@ -103,6 +103,9 @@ jobs:
         id: build-epp-image
         shell: bash
         timeout-minutes: 30
+        env:
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          SCCACHE_S3_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
         run: |
           set -x
           EPP_REPOSITORY="${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_REPOSITORY }}/dynamo-epp"
@@ -121,6 +124,9 @@ jobs:
             IMAGE_REPO="${EPP_IMAGE_REPO}" \
             GIT_TAG="${EPP_IMAGE_TAG}" \
             DOCKER_PROXY="${ECR_HOSTNAME}/dockerhub/" \
+            USE_SCCACHE=true \
+            SCCACHE_BUCKET="${SCCACHE_S3_BUCKET}" \
+            SCCACHE_REGION="${AWS_DEFAULT_REGION}" \
             EXTRA_BUILD_ARGS="${CACHE_ARGS}"
       - name: Generate Dockerfile
         shell: bash

deploy/inference-gateway/epp/Dockerfile

@@ -40,9 +40,17 @@ ARG BASE_IMAGE=ubuntu:24.04
 FROM ${RUST_IMAGE} AS rust-builder
 
 # TARGETARCH is provided automatically by buildx for multi-platform builds.
-# Used here only for per-platform cache isolation.
 ARG TARGETARCH
 
+# sccache configuration (content-addressed S3 cache keyed by source hash).
+# When USE_SCCACHE=true, the build authenticates to S3 via IRSA secrets
+# mounted on the cargo-build RUN step. If the sccache server fails to start
+# (e.g. missing creds), the build continues without cache -- never with a
+# stale one, unlike a persistent target/ mount.
+ARG USE_SCCACHE
+ARG SCCACHE_BUCKET=""
+ARG SCCACHE_REGION=""
+
 # etcd-client crate requires protoc to compile proto files
 RUN apt-get update && apt-get install -y --no-install-recommends \
     protobuf-compiler \
@@ -51,6 +59,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 WORKDIR /dynamo
 
+# Install sccache using the shared helper so this build matches the rest
+# of the repo's Rust builds (see container/templates/wheel_builder.Dockerfile).
+COPY --from=dynamo container/use-sccache.sh /tmp/use-sccache.sh
+RUN if [ "$USE_SCCACHE" = "true" ]; then \
+        /tmp/use-sccache.sh install; \
+    fi
+
+ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \
+    SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION}}
+
 # Copy Cargo workspace manifests, lockfile, and README (some crates inherit
 # readme.workspace = true, so cargo needs README.md at the workspace root)
 COPY --from=dynamo .cargo/ .cargo/
@@ -59,18 +77,25 @@ COPY --from=dynamo Cargo.toml Cargo.lock README.md ./
 # Copy all workspace crates (libdynamo_llm depends transitively on many)
 COPY --from=dynamo lib/ lib/
 
-# Build the static library; use BuildKit cache mounts for cargo registry and
-# build artifacts so incremental Docker rebuilds are fast.
-# Cache IDs are keyed by TARGETARCH so multi-platform builds don't collide.
-# Artifacts are copied to /out/ because the target/ cache mount is ephemeral.
+# Build libdynamo_llm with sccache for cross-run caching. The registry and
+# git caches are content-addressed (safe to persist); no target/ mount --
+# sccache caches compilations in S3 where stale artifacts can't be linked
+# against newer source.
 RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETARCH} \
     --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETARCH} \
-    --mount=type=cache,target=/dynamo/target,id=cargo-target-${TARGETARCH} \
+    --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \
+    --mount=type=secret,id=aws-role-arn,env=AWS_ROLE_ARN \
+    export AWS_WEB_IDENTITY_TOKEN_FILE=/run/secrets/aws-token && \
+    export SCCACHE_S3_KEY_PREFIX="${SCCACHE_S3_KEY_PREFIX:-epp-${TARGETARCH}}" && \
+    if [ "$USE_SCCACHE" = "true" ]; then \
+        eval $(/tmp/use-sccache.sh setup-env); \
+    fi && \
     cargo build --release -p libdynamo_llm && \
     mkdir -p /out && \
     cp target/release/libdynamo_llm_capi.a /out/ && \
     HEADER=$(find target/release/build -name llm_engine.h -path "*/out/*" | head -1) && \
-    [ -n "$HEADER" ] && cp "$HEADER" /out/ || { echo "ERROR: llm_engine.h not found in target/"; exit 1; }
+    [ -n "$HEADER" ] && cp "$HEADER" /out/ || { echo "ERROR: llm_engine.h not found in target/"; exit 1; } && \
+    if [ "$USE_SCCACHE" = "true" ]; then /tmp/use-sccache.sh show-stats "libdynamo_llm" || true; fi
 
 # =============================================================================
 # Stage 2: Build Go EPP binary

deploy/inference-gateway/epp/Makefile

@@ -27,6 +27,12 @@ MULTIARCH_PLATFORMS ?= linux/amd64,linux/arm64
 DOCKER_PROXY ?=
 EXTRA_BUILD_ARGS ?=
 
+# sccache configuration for Rust compilation caching (CI only).
+# Leave USE_SCCACHE unset locally to build without S3 cache.
+USE_SCCACHE ?=
+SCCACHE_BUCKET ?=
+SCCACHE_REGION ?=
+
 DOCKER_BUILDX_CMD ?= docker buildx
 IMAGE_BUILD_CMD ?= $(DOCKER_BUILDX_CMD) build
 RUST_IMAGE ?= $(DOCKER_PROXY)rust:1.93.1
@@ -106,6 +112,22 @@ image-kind: image-load ## Build and load the image into kind cluster
 
 ##@ Multi-Architecture Builds
 
+# Collect sccache args only when USE_SCCACHE=true. IRSA secrets come from the
+# runner env (AWS_WEB_IDENTITY_TOKEN_FILE + AWS_ROLE_ARN); buildx mounts them
+# into the RUN step that invokes cargo so sccache can authenticate to S3.
+SCCACHE_ARGS =
+ifeq ($(USE_SCCACHE),true)
+  SCCACHE_ARGS += --build-arg USE_SCCACHE=true
+  SCCACHE_ARGS += --build-arg SCCACHE_BUCKET=$(SCCACHE_BUCKET)
+  SCCACHE_ARGS += --build-arg SCCACHE_REGION=$(SCCACHE_REGION)
+  ifneq ($(AWS_WEB_IDENTITY_TOKEN_FILE),)
+    ifneq ($(AWS_ROLE_ARN),)
+      SCCACHE_ARGS += --secret id=aws-web-identity-token,src=$(AWS_WEB_IDENTITY_TOKEN_FILE)
+      SCCACHE_ARGS += --secret id=aws-role-arn,env=AWS_ROLE_ARN
+    endif
+  endif
+endif
+
 .PHONY: image-multiarch
 image-multiarch: ## Build multi-arch image (requires --push ; --load not supported)
 	$(IMAGE_BUILD_CMD) -t $(IMAGE_TAG) \
@@ -116,7 +138,7 @@ image-multiarch: ## Build multi-arch image (requires --push ; --load not support
 		--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
 		--build-arg COMMIT_SHA=$(GIT_COMMIT_SHA) \
 		--build-arg BUILD_REF=$(GIT_TAG) \
-		$(EXTRA_BUILD_ARGS) $(PUSH) .
+		$(SCCACHE_ARGS) $(EXTRA_BUILD_ARGS) $(PUSH) .
 
 .PHONY: image-multiarch-push
 image-multiarch-push: PUSH=--push ## Build and push multi-arch image to registry