ci: remote builders sccache support (#6369)

.github/actions/docker-remote-build/action.yml

@@ -55,7 +55,7 @@ inputs:
   use_sccache:
     description: 'Use SCCache for caching'
     required: false
-    default: 'false'
+    default: 'true'
   ci:
     description: 'CI mode: for frontend target, uses existing buildx builder and pushes EPP image to ECR'
     required: false
@@ -129,7 +129,9 @@ runs:
           EXTRA_ARGS+=" --no-cache"
         fi
         if [ "${{ inputs.use_sccache }}" == "true" ]; then
-          EXTRA_ARGS+=" --build-arg CARGO_BUILD_JOBS=4 --use-sccache"
+          EXTRA_ARGS+=" --build-arg USE_SCCACHE=true"
+          EXTRA_ARGS+=" --build-arg SCCACHE_BUCKET=${SCCACHE_S3_BUCKET}"
+          EXTRA_ARGS+=" --build-arg SCCACHE_REGION=${AWS_DEFAULT_REGION}"
         fi
         if [ "${{ inputs.push_image }}" == "true" ]; then
           EXTRA_ARGS+=" --push"
@@ -147,14 +149,20 @@ runs:
           done <<< "$EXTRA_TAGS"
         fi
 
+        # Pass AWS credentials as build secrets for sccache S3 access.
+        # Dockerfile steps reference these via --mount=type=secret,id=aws-key-id,env=...
+        SECRET_ARGS=""
+        if [ "${{ inputs.use_sccache }}" == "true" ] && [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
+          SECRET_ARGS+=" --secret id=aws-key-id,env=AWS_ACCESS_KEY_ID"
+          SECRET_ARGS+=" --secret id=aws-secret-id,env=AWS_SECRET_ACCESS_KEY"
+        fi
+
         docker buildx build \
           --progress=plain \
           --tag "$IMAGE_TAG" \
           --platform linux/${{ inputs.platform }} \
           -f ./container/rendered.Dockerfile \
-          $CACHE_ARGS \
-          $EXTRA_ARGS \
-          $EPP_IMAGE_ARG . 2>&1 | tee "${BUILD_LOG_FILE}"
+          $CACHE_ARGS $EXTRA_ARGS $SECRET_ARGS $EPP_IMAGE_ARG . 2>&1 | tee "${BUILD_LOG_FILE}"
 
         BUILD_EXIT_CODE=${PIPESTATUS[0]}
 

container/templates/wheel_builder.Dockerfile

@@ -139,10 +139,10 @@ RUN if [ "$USE_SCCACHE" = "true" ]; then \
         /tmp/use-sccache.sh install; \
     fi
 
-# Set SCCACHE environment variables
+# Set SCCACHE environment variables (RUSTC_WRAPPER is set dynamically by
+# setup-env only when the sccache server starts successfully)
 ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET}} \
-    SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION}} \
-    RUSTC_WRAPPER=${USE_SCCACHE:+sccache}
+    SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION}}
 
 # Build FFmpeg from source
 # Do not delete the source tarball for legal reasons
@@ -153,9 +153,7 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
 if [ "$ENABLE_MEDIA_FFMPEG" = "true" ]; then \
     export SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX:-${ARCH}} && \
     if [ "$USE_SCCACHE" = "true" ]; then \
-        export CMAKE_C_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CXX_COMPILER_LAUNCHER="sccache" && \
-        export RUSTC_WRAPPER="sccache"; \
+        eval $(/tmp/use-sccache.sh setup-env); \
     fi && \
     dnf install -y pkg-config && \
     cd /tmp && \
@@ -191,9 +189,7 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
     --mount=type=secret,id=aws-secret-id,env=AWS_SECRET_ACCESS_KEY \
     export SCCACHE_S3_KEY_PREFIX="${SCCACHE_S3_KEY_PREFIX:-${ARCH}}" && \
     if [ "$USE_SCCACHE" = "true" ]; then \
-        export CMAKE_C_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CXX_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"; \
+        eval $(/tmp/use-sccache.sh setup-env); \
     fi && \
     cd /usr/local/src && \
      git clone https://github.com/openucx/ucx.git && \
@@ -226,9 +222,7 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
     --mount=type=secret,id=aws-secret-id,env=AWS_SECRET_ACCESS_KEY \
     export SCCACHE_S3_KEY_PREFIX="${SCCACHE_S3_KEY_PREFIX:-${ARCH}}" && \
     if [ "$USE_SCCACHE" = "true" ]; then \
-        export CMAKE_C_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CXX_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"; \
+        eval $(/tmp/use-sccache.sh setup-env); \
     fi && \
     cd /usr/local/src && \
     git clone https://github.com/ofiwg/libfabric.git && \
@@ -258,6 +252,9 @@ ARG AWS_SDK_CPP_VERSION=1.11.581
 RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
     --mount=type=secret,id=aws-secret-id,env=AWS_SECRET_ACCESS_KEY \
     export SCCACHE_S3_KEY_PREFIX="${SCCACHE_S3_KEY_PREFIX:-${ARCH}}" && \
+    if [ "$USE_SCCACHE" = "true" ]; then \
+        eval $(/tmp/use-sccache.sh setup-env cmake); \
+    fi && \
     git clone --recurse-submodules --depth 1 --branch ${AWS_SDK_CPP_VERSION} \
         https://github.com/aws/aws-sdk-cpp.git /tmp/aws-sdk-cpp && \
     mkdir -p /tmp/aws-sdk-cpp/build && \
@@ -282,9 +279,7 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
     --mount=type=secret,id=aws-secret-id,env=AWS_SECRET_ACCESS_KEY \
     export SCCACHE_S3_KEY_PREFIX="${SCCACHE_S3_KEY_PREFIX:-${ARCH}}" && \
     if [ "$USE_SCCACHE" = "true" ]; then \
-        export CMAKE_C_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CXX_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"; \
+        eval $(/tmp/use-sccache.sh setup-env); \
     fi && \
     source ${VIRTUAL_ENV}/bin/activate && \
     git clone "https://github.com/ai-dynamo/nixl.git" && \
@@ -318,9 +313,7 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
     export UV_CACHE_DIR=/root/.cache/uv && \
     export SCCACHE_S3_KEY_PREFIX="${SCCACHE_S3_KEY_PREFIX:-${ARCH}}" && \
     if [ "$USE_SCCACHE" = "true" ]; then \
-        export CMAKE_C_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CXX_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"; \
+        eval $(/tmp/use-sccache.sh setup-env); \
     fi && \
     cd /workspace/nixl && \
     uv build . --wheel --out-dir /opt/dynamo/dist/nixl --python $PYTHON_VERSION
@@ -340,10 +333,9 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
     export UV_CACHE_DIR=/root/.cache/uv && \
     export SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX:-${ARCH}} && \
     if [ "$USE_SCCACHE" = "true" ]; then \
-        export CMAKE_C_COMPILER_LAUNCHER="sccache" && \
-        export CMAKE_CXX_COMPILER_LAUNCHER="sccache" && \
-        export RUSTC_WRAPPER="sccache"; \
+        eval $(/tmp/use-sccache.sh setup-env cmake); \
     fi && \
+    mkdir -p ${CARGO_TARGET_DIR} && \
     source ${VIRTUAL_ENV}/bin/activate && \
     cd /opt/dynamo && \
     uv build --wheel --out-dir /opt/dynamo/dist && \

container/use-sccache.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 # sccache management script
 # This script handles sccache installation, environment setup, and statistics display
 
-SCCACHE_VERSION="v0.8.2"
+SCCACHE_VERSION="v0.14.0"
 
 
 usage() {
@@ -16,9 +16,18 @@ Usage: $0 [COMMAND] [OPTIONS]
 
 Commands:
     install         Install sccache binary (requires ARCH_ALT environment variable)
+    setup-env       Print export statements to configure sccache for compilation
     show-stats      Display sccache statistics with optional build name
     help            Show this help message
 
+setup-env modes:
+    setup-env           Wraps CC/CXX with sccache + sets RUSTC_WRAPPER.
+                        Works for autotools (make) and Meson builds.
+    setup-env cmake     Also sets CMAKE_C/CXX/CUDA_COMPILER_LAUNCHER.
+                        Use for CMake-based builds.
+
+    Usage: eval \$($0 setup-env [cmake])
+
 Environment variables:
     USE_SCCACHE             Set to 'true' to enable sccache
     SCCACHE_BUCKET          S3 bucket name (fallback if not passed as parameter)
@@ -27,9 +36,9 @@ Environment variables:
     ARCH_ALT                Alternative architecture name for downloads (e.g., x86_64, aarch64)
 
 Examples:
-    # Install sccache (requires ARCH_ALT to be set)
     ARCH_ALT=x86_64 $0 install
-    # Show stats with build name
+    eval \$($0 setup-env)          # autotools / Meson
+    eval \$($0 setup-env cmake)    # CMake builds
     $0 show-stats "UCX"
 EOF
 }
@@ -47,9 +56,70 @@ install_sccache() {
     mv "sccache-${SCCACHE_VERSION}-${ARCH_ALT}-unknown-linux-musl/sccache" /usr/local/bin/
     # Cleanup
     rm -rf sccache*
+
+    # Create compiler wrapper scripts for autotools/Meson compatibility.
+    # Autoconf breaks with CC="sccache gcc" (multi-word value), so we provide
+    # single-binary wrappers that autoconf sees as a regular compiler.
+    # The real compiler path is passed at runtime via SCCACHE_CC_REAL / SCCACHE_CXX_REAL.
+    cat > /usr/local/bin/sccache-cc <<'WRAPPER'
+#!/bin/sh
+# Only use sccache for pure compilations (-c flag).
+# Autoconf tests the compiler with combined compile+link (no -c), and sccache
+# can interfere with those tests. sccache only caches compilations anyway,
+# so routing linking/other invocations directly to gcc loses nothing.
+case " $* " in
+    *" -c "*) exec sccache "${SCCACHE_CC_REAL:-gcc}" "$@" ;;
+    *)        exec "${SCCACHE_CC_REAL:-gcc}" "$@" ;;
+esac
+WRAPPER
+    chmod +x /usr/local/bin/sccache-cc
+
+    cat > /usr/local/bin/sccache-cxx <<'WRAPPER'
+#!/bin/sh
+case " $* " in
+    *" -c "*) exec sccache "${SCCACHE_CXX_REAL:-g++}" "$@" ;;
+    *)        exec "${SCCACHE_CXX_REAL:-g++}" "$@" ;;
+esac
+WRAPPER
+    chmod +x /usr/local/bin/sccache-cxx
+
     echo "sccache installed successfully"
 }
 
+setup_env() {
+    local mode="${1:-default}"
+
+    # Output a conditional block: only configure sccache if the server starts
+    # successfully. The server needs working S3 credentials (mounted via
+    # --mount=type=secret); if they're missing or invalid, we skip sccache
+    # entirely so the build continues with normal compilers.
+    #
+    # Use a per-step Unix domain socket so concurrent builds on the same
+    # buildkit worker don't collide on the default TCP port 4226.
+    echo 'export SCCACHE_SERVER_UDS="/tmp/sccache-$(mktemp -u XXXXXX).sock";'
+    echo 'if sccache --start-server; then'
+    echo '  export SCCACHE_IDLE_TIMEOUT=0;'
+    echo '  export RUSTC_WRAPPER="sccache";'
+
+    if [ "$mode" = "cmake" ]; then
+        echo '  export CMAKE_C_COMPILER_LAUNCHER="sccache";'
+        echo '  export CMAKE_CXX_COMPILER_LAUNCHER="sccache";'
+        echo '  export CMAKE_CUDA_COMPILER_LAUNCHER="sccache";'
+    else
+        # Wrapper scripts (installed during sccache install) route only pure
+        # compilations (-c flag) through sccache; linking goes directly to
+        # the real compiler so autoconf's link tests pass.
+        echo '  export SCCACHE_CC_REAL="${CC:-gcc}";'
+        echo '  export SCCACHE_CXX_REAL="${CXX:-g++}";'
+        echo '  export CC="/usr/local/bin/sccache-cc";'
+        echo '  export CXX="/usr/local/bin/sccache-cxx";'
+    fi
+
+    echo 'else'
+    echo '  echo "WARNING: sccache server failed to start, building without cache";'
+    echo 'fi'
+}
+
 show_stats() {
     if command -v sccache >/dev/null 2>&1; then
         # Generate timestamp in ISO 8601 format
@@ -83,9 +153,9 @@ main() {
         install)
             install_sccache
             ;;
-        generate-env)
-            shift  # Remove the command from arguments
-            generate_env_file "$@"  # Pass all remaining arguments
+        setup-env)
+            shift
+            setup_env "$@"
             ;;
         show-stats)
             shift  # Remove the command from arguments