chore: optimize dockerfiles cache (#6371)

container/templates/aws.Dockerfile

@@ -25,7 +25,9 @@ USER root
 #   --skip-kmod: Skip kernel module installation (handled by host)
 #   --skip-limit-conf: Skip ulimit configuration (handled by container runtime)
 #   --no-verify: Skip GPG verification (optional, can be removed if verification is needed)
-RUN mkdir -p /tmp/efa && \
+# Cache apt downloads; sharing=locked avoids apt/dpkg races with concurrent builds.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    mkdir -p /tmp/efa && \
     cd /tmp/efa && \
     curl --retry 3 --retry-delay 2 -fsSL -o aws-efa-installer-${EFA_VERSION}.tar.gz \
         https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_VERSION}.tar.gz && \

container/templates/dynamo_base.Dockerfile

@@ -20,7 +20,7 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 # Install NATS server
 ARG NATS_VERSION
-RUN --mount=type=cache,target=/var/cache/apt \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     wget --tries=3 --waitretry=5 https://github.com/nats-io/nats-server/releases/download/${NATS_VERSION}/nats-server-${NATS_VERSION}-${ARCH}.deb && \
     dpkg -i nats-server-${NATS_VERSION}-${ARCH}.deb && rm nats-server-${NATS_VERSION}-${ARCH}.deb
 

container/templates/trtllm_framework.Dockerfile

@@ -92,7 +92,9 @@ COPY --from=pytorch_base /usr/local/lib/python${PYTHON_VERSION}/dist-packages/fl
 COPY --from=pytorch_base /usr/local/lib/python${PYTHON_VERSION}/dist-packages/torch_tensorrt ${VIRTUAL_ENV}/lib/python${PYTHON_VERSION}/site-packages/torch_tensorrt
 COPY --from=pytorch_base /usr/local/lib/python${PYTHON_VERSION}/dist-packages/torch_tensorrt-${TORCH_TENSORRT_VER}.dist-info ${VIRTUAL_ENV}/lib/python${PYTHON_VERSION}/site-packages/torch_tensorrt-${TORCH_TENSORRT_VER}.dist-info
 
-RUN uv pip install flashinfer-python==${FLASHINFER_PYTHON_VER}
+RUN --mount=type=cache,target=/root/.cache/uv \
+    export UV_CACHE_DIR=/root/.cache/uv UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 && \
+    uv pip install flashinfer-python==${FLASHINFER_PYTHON_VER}
 
 # Install TensorRT-LLM and related dependencies
 ARG HAS_TRTLLM_CONTEXT

container/templates/trtllm_runtime.Dockerfile

@@ -89,8 +89,10 @@ RUN userdel -r ubuntu > /dev/null 2>&1 || true \
     && mkdir -p /etc/profile.d && echo 'umask 002' > /etc/profile.d/00-umask.sh
 
 # Install Python, build-essential and python3-dev as apt dependencies
+# Cache apt downloads; sharing=locked avoids apt/dpkg races with concurrent builds.
 ARG PYTHON_VERSION
-RUN if [ ${ARCH_ALT} = "x86_64" ]; then \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    if [ ${ARCH_ALT} = "x86_64" ]; then \
         ARCH_FOR_GPG=${ARCH_ALT}; \
     else \
         ARCH_FOR_GPG="sbsa"; \
@@ -230,10 +232,10 @@ RUN --mount=type=cache,target=/home/dynamo/.cache/uv,uid=1000,gid=0,mode=0775 \
     chmod -R g+w /workspace/benchmarks
 
 # Install common and test dependencies
+# --no-cache is intentional: mixed indexes (PyPI + PyTorch CUDA wheels) risk serving stale/wrong-variant cached wheels
 RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requirements.txt \
     --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.test.txt \
-    --mount=type=cache,target=/home/dynamo/.cache/uv,uid=1000,gid=0,mode=0775 \
-    export UV_CACHE_DIR=/home/dynamo/.cache/uv UV_GIT_LFS=1 UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 && \
+    export UV_GIT_LFS=1 UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 && \
     uv pip install \
         --no-cache \
         --index-strategy unsafe-best-match \

container/templates/wheel_builder.Dockerfile

@@ -35,7 +35,9 @@ COPY --from=dynamo_base $RUSTUP_HOME $RUSTUP_HOME
 COPY --from=dynamo_base $CARGO_HOME $CARGO_HOME
 
 # Install system dependencies
-RUN dnf install -y almalinux-release-synergy && \
+# Cache dnf downloads; sharing=locked avoids dnf/rpm races with concurrent builds.
+RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked \
+    dnf install -y almalinux-release-synergy && \
     dnf config-manager --set-enabled powertools && \
     dnf install -y \
         # Autotools (required for UCX, libfabric ./autogen.sh and ./configure)
@@ -75,8 +77,7 @@ RUN dnf install -y almalinux-release-synergy && \
         libcurl-devel \
         openssl-devel \
         libuuid-devel \
-        zlib-devel && \
-    dnf clean all && rm -rf /var/cache/dnf/
+        zlib-devel
 
 # Set GCC toolset 14 as the default compiler (CUDA requires GCC <= 14)
 ENV PATH="/opt/rh/gcc-toolset-14/root/usr/bin:${PATH}" \
@@ -369,7 +370,9 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
 
 # Build gpu_memory_service wheel (C++ extension only needs Python headers, no CUDA/torch)
 ARG ENABLE_GPU_MEMORY_SERVICE
-RUN if [ "$ENABLE_GPU_MEMORY_SERVICE" = "true" ]; then \
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ "$ENABLE_GPU_MEMORY_SERVICE" = "true" ]; then \
+        export UV_CACHE_DIR=/root/.cache/uv && \
         source ${VIRTUAL_ENV}/bin/activate && \
         uv build --wheel --out-dir /opt/dynamo/dist /opt/dynamo/lib/gpu_memory_service; \
     fi