feat: split monolithic requirements.txt and remove test deps from runtime image (#6656)

Signed-off-by: Anant Sharma <anants@nvidia.com>

.github/filters.yaml

@@ -67,6 +67,7 @@ ci: &ci
   - '.github/actions/**'
   - '.github/scripts/parse_buildkit_output.py'
   - '.github/scripts/route_buildkit.sh'
+  - 'container/Dockerfile.test'
 
 core:
   # Exclude documentation files first, then include patterns override for specific paths

.github/workflows/build-test-distribute-flavor.yml

@@ -162,6 +162,7 @@ jobs:
     runs-on: prod-builder-v3
     outputs:
       target_tag_plain: ${{ steps.calculate-target-tag.outputs.target_tag_plain }}
+      test_tag_plain: ${{ steps.calculate-target-tag.outputs.test_tag_plain }}
     env:
       FRAMEWORK: ${{ inputs.framework }}
     steps:
@@ -188,9 +189,13 @@ jobs:
             EFA_SUFFIX="-efa"
           fi
           TARGET_TAG_PLAIN="${{ github.sha }}-${{ inputs.framework }}${EFA_SUFFIX}"
+          TEST_TAG_PLAIN="${{ github.sha }}-${{ inputs.framework }}${EFA_SUFFIX}-test"
           DEFAULT_TARGET_IMAGE_URI="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${TARGET_TAG_PLAIN}-cuda${CUDA_VERSION}-${{ inputs.platform }}"
+          TEST_IMAGE_URI="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${TEST_TAG_PLAIN}-cuda${CUDA_VERSION}-${{ inputs.platform }}"
           echo "default_target_image_uri=${DEFAULT_TARGET_IMAGE_URI}" >> $GITHUB_OUTPUT
+          echo "test_image_uri=${TEST_IMAGE_URI}" >> $GITHUB_OUTPUT
           echo "target_tag_plain=${TARGET_TAG_PLAIN}" >> $GITHUB_OUTPUT
+          echo "test_tag_plain=${TEST_TAG_PLAIN}" >> $GITHUB_OUTPUT
           echo "cuda_version_plain=${CUDA_VERSION}" >> $GITHUB_OUTPUT
       - name: Initialize Dynamo Builder
         uses: ./.github/actions/init-dynamo-builder
@@ -258,7 +263,7 @@ jobs:
               ${MAKE_EFA_FLAG} \
               --show-result \
               --output-short-filename
-      - name: Build Container
+      - name: Build and Push Runtime Image
         id: build-image
         timeout-minutes: ${{ inputs.build_timeout_minutes }}
         uses: ./.github/actions/docker-remote-build
@@ -279,6 +284,36 @@ jobs:
           no_load: ${{ inputs.no_load }}
           extra_build_args: |
             DYNAMO_COMMIT_SHA=${{ github.sha }}
+      - name: Build and Push Test Image
+        shell: bash
+        env:
+          ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
+        run: |
+          CUDA_MAJOR=${{ steps.calculate-target-tag.outputs.cuda_version_plain }}
+          CACHE_TAG="test-${{ inputs.framework }}-cuda${CUDA_MAJOR}-${{ inputs.platform }}-cache"
+          CACHE_ARGS="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG}"
+          CACHE_ARGS+=" --cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG}"
+          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
+            CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG},mode=max"
+          elif [[ "$GITHUB_REF_NAME" =~ ^release ]]; then
+            CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG},mode=max"
+          fi
+
+          PUSH_ARGS=""
+          if [ "${{ inputs.push_image }}" == "true" ]; then
+            PUSH_ARGS="--push"
+          elif [ "${{ inputs.no_load }}" == "false" ]; then
+            PUSH_ARGS="--load"
+          fi
+
+          docker buildx build \
+            --progress=plain \
+            ${PUSH_ARGS} \
+            --platform linux/${{ inputs.platform }} \
+            -f container/Dockerfile.test \
+            --build-arg BASE_IMAGE=${{ steps.calculate-target-tag.outputs.default_target_image_uri }} \
+            ${CACHE_ARGS} \
+            -t ${{ steps.calculate-target-tag.outputs.test_image_uri }} .
       - name: Show summary
         shell: bash
         if: ${{ inputs.push_image && inputs.show_summary }}
@@ -326,7 +361,9 @@ jobs:
           CUDA_VERSION_RAW=${{ inputs.cuda_version }}
           CUDA_VERSION=${CUDA_VERSION_RAW%%.*}
           echo "cuda_version_plain=${CUDA_VERSION}" >> $GITHUB_OUTPUT
-          TEST_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.target_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
+          RUNTIME_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.target_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
+          echo "runtime_image=${RUNTIME_IMAGE}" >> $GITHUB_OUTPUT
+          TEST_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
           echo "test_image=${TEST_IMAGE}" >> $GITHUB_OUTPUT
       - name: Docker Login
         uses: ./.github/actions/docker-login
@@ -341,6 +378,7 @@ jobs:
         run: |
           source ./.github/scripts/retry_docker.sh
           start_time=$(date +%s)
+          retry_pull ${{ steps.calculate-target-tag.outputs.runtime_image }}
           retry_pull ${{ steps.calculate-target-tag.outputs.test_image }}
           retry_pull quay.io/minio/minio
           end_time=$(date +%s)
@@ -350,14 +388,14 @@ jobs:
       - name: Run Sanity Check on Runtime Image
         shell: bash
         run: |
-          echo "Running sanity check on image: ${{ steps.calculate-target-tag.outputs.test_image }}"
+          echo "Running sanity check on image: ${{ steps.calculate-target-tag.outputs.runtime_image }}"
 
           # Run the sanity check script inside the container
           # The script is located in /workspace/deploy/sanity_check.py in runtime containers
           export WORKSPACE=/workspace
 
           set +e
-          docker run --rm "${{ steps.calculate-target-tag.outputs.test_image }}" python ${WORKSPACE}/deploy/sanity_check.py --runtime-check --no-gpu-check
+          docker run --rm "${{ steps.calculate-target-tag.outputs.runtime_image }}" python ${WORKSPACE}/deploy/sanity_check.py --runtime-check --no-gpu-check
           SANITY_CHECK_EXIT_CODE=$?
           set -e
           if [ ${SANITY_CHECK_EXIT_CODE} -ne 0 ]; then
@@ -426,7 +464,9 @@ jobs:
           CUDA_VERSION_RAW=${{ inputs.cuda_version }}
           CUDA_VERSION=${CUDA_VERSION_RAW%%.*}
           echo "cuda_version_plain=${CUDA_VERSION}" >> $GITHUB_OUTPUT
-          TEST_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.target_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
+          RUNTIME_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.target_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
+          echo "runtime_image=${RUNTIME_IMAGE}" >> $GITHUB_OUTPUT
+          TEST_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
           echo "test_image=${TEST_IMAGE}" >> $GITHUB_OUTPUT
       - name: Docker Login
         uses: ./.github/actions/docker-login
@@ -490,8 +530,6 @@ jobs:
           CUDA_VERSION_RAW=${{ inputs.cuda_version }}
           CUDA_VERSION=${CUDA_VERSION_RAW%%.*}
           echo "cuda_version_plain=${CUDA_VERSION}" >> $GITHUB_OUTPUT
-          TEST_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.target_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
-          echo "test_image=${TEST_IMAGE}" >> $GITHUB_OUTPUT
 
       - name: Copy image to target registry
         timeout-minutes: ${{ inputs.copy_timeout_minutes }}

.github/workflows/container-validation-dynamo.yml

@@ -62,7 +62,8 @@ jobs:
     timeout-minutes: 60
     outputs:
       # Only pass the non-secret tag suffix between jobs (GitHub blanks outputs containing secrets)
-      image_tag_suffix: ${{ steps.define_image_tag.outputs.image_tag_suffix }}
+      runtime_tag_suffix: ${{ steps.define_image_tag.outputs.runtime_tag_suffix }}
+      test_tag_suffix: ${{ steps.define_image_tag.outputs.test_tag_suffix }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
@@ -90,9 +91,12 @@ jobs:
           ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
         run: |
           CUDA_MAJOR=${CUDA_VERSION%%.*}
-          IMAGE_TAG_SUFFIX="${{ github.sha }}-dynamo-runtime-cuda${CUDA_MAJOR}-amd64"
-          echo "image_tag_suffix=${IMAGE_TAG_SUFFIX}" >> $GITHUB_OUTPUT
-          echo "remote_tag=${ECR_HOSTNAME}/ai-dynamo/dynamo:${IMAGE_TAG_SUFFIX}" >> $GITHUB_OUTPUT
+          RUNTIME_TAG_SUFFIX="${{ github.sha }}-dynamo-runtime-cuda${CUDA_MAJOR}-amd64"
+          TEST_TAG_SUFFIX="${{ github.sha }}-dynamo-test-cuda${CUDA_MAJOR}-amd64"
+          echo "runtime_tag_suffix=${RUNTIME_TAG_SUFFIX}" >> $GITHUB_OUTPUT
+          echo "test_tag_suffix=${TEST_TAG_SUFFIX}" >> $GITHUB_OUTPUT
+          echo "runtime_remote_tag=${ECR_HOSTNAME}/ai-dynamo/dynamo:${RUNTIME_TAG_SUFFIX}" >> $GITHUB_OUTPUT
+          echo "test_remote_tag=${ECR_HOSTNAME}/ai-dynamo/dynamo:${TEST_TAG_SUFFIX}" >> $GITHUB_OUTPUT
       - name: Generate Dockerfile
         shell: bash
         run: |
@@ -104,10 +108,10 @@ jobs:
               --cuda-version=${{ env.CUDA_VERSION }} \
               --show-result \
               --output-short-filename
-      - name: Build and Push Container
+      - name: Build and Push Runtime Image
         uses: ./.github/actions/docker-remote-build
         with:
-          image_tag: ${{ steps.define_image_tag.outputs.remote_tag }}
+          image_tag: ${{ steps.define_image_tag.outputs.runtime_remote_tag }}
           framework: dynamo
           target: runtime
           platform: amd64
@@ -119,6 +123,28 @@ jobs:
           aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           push_image: 'true'
+      - name: Build and Push Test Image
+        env:
+          ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com
+        shell: bash
+        run: |
+          CACHE_TAG="test-dynamo-cuda${CUDA_VERSION%%.*}-amd64-cache"
+          CACHE_ARGS="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG}"
+          CACHE_ARGS+=" --cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG}"
+          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
+            CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG},mode=max"
+          elif [[ "$GITHUB_REF_NAME" =~ ^release ]]; then
+            CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG},mode=max"
+          fi
+
+          docker buildx build \
+            --progress=plain \
+            --push \
+            --platform linux/amd64 \
+            -f container/Dockerfile.test \
+            --build-arg BASE_IMAGE=${{ steps.define_image_tag.outputs.runtime_remote_tag }} \
+            ${CACHE_ARGS} \
+            -t ${{ steps.define_image_tag.outputs.test_remote_tag }} .
 
   rust-checks:
     needs: [changed-files, build]
@@ -128,7 +154,7 @@ jobs:
     timeout-minutes: 30
     env:
       CONTAINER_ID: test_${{ github.run_id }}_${{ github.run_attempt }}_rust_dynamo
-      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.image_tag_suffix }}
+      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.runtime_tag_suffix }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
@@ -137,7 +163,7 @@ jobs:
         with:
           aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
           aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-      - name: Pull image
+      - name: Pull runtime image
         run: |
           source ./.github/scripts/retry_docker.sh
           retry_pull ${{ env.IMAGE_TAG }}
@@ -167,7 +193,7 @@ jobs:
     name: Pytest (parallel)
     timeout-minutes: 30
     env:
-      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.image_tag_suffix }}
+      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_suffix }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
@@ -176,7 +202,7 @@ jobs:
         with:
           aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
           aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-      - name: Pull image
+      - name: Pull test image
         run: |
           source ./.github/scripts/retry_docker.sh
           retry_pull ${{ env.IMAGE_TAG }}
@@ -200,7 +226,7 @@ jobs:
     name: Pytest (sequential)
     timeout-minutes: 30
     env:
-      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.image_tag_suffix }}
+      IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_suffix }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
@@ -209,7 +235,7 @@ jobs:
         with:
           aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
           aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-      - name: Pull image
+      - name: Pull test image
         run: |
           source ./.github/scripts/retry_docker.sh
           retry_pull ${{ env.IMAGE_TAG }}

container/Dockerfile.test

+# syntax=docker/dockerfile:1.10.0-labs
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Standalone test image — layers test deps on top of a pre-built runtime image.
+# Usage: docker build -f container/Dockerfile.test --build-arg BASE_IMAGE=<runtime-image> .
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+# Install test dependencies on top of runtime image
+USER root
+RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.test.txt \
+    --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=cache,target=/root/.cache/pip,sharing=locked \
+    export UV_CACHE_DIR=/root/.cache/uv UV_GIT_LFS=1 UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 && \
+    export PIP_CACHE_DIR=/root/.cache/pip && \
+    if [ -n "$VIRTUAL_ENV" ]; then \
+        uv pip install \
+            --requirement /tmp/requirements.test.txt; \
+    else \
+        pip install --break-system-packages \
+            --requirement /tmp/requirements.test.txt; \
+    fi
+
+USER dynamo

container/README.md

@@ -122,7 +122,13 @@ docker build -t dynamo:latest-vllm-runtime -f container/rendered.Dockerfile .
 container/run.sh --image dynamo:latest-vllm-runtime -it
 ```
 
-### 2. local-dev + `run.sh` (runs as dynamo user with matched host UID/GID):
+### 2. test image (layers test deps on top of runtime):
+```bash
+# Build test image from a runtime image (for running tests locally)
+docker build -f container/Dockerfile.test --build-arg BASE_IMAGE=dynamo:latest-vllm-runtime -t dynamo:latest-vllm-test .
+```
+
+### 3. local-dev + `run.sh` (runs as dynamo user with matched host UID/GID):
 ```bash
 run.sh --mount-workspace -it --image dynamo:latest-vllm-local-dev ...
 ```

container/deps/README.md

+# Container Python Dependencies
+
+Python dependency files for Dynamo container images, split by component
+so each image only installs what it needs.
+
+## Files
+
+| File | Purpose |
+|------|---------|
+| `requirements.common.txt` | Core deps shared by all containers |
+| `requirements.planner.txt` | Planner, profiler, global_planner, deploy utils |
+| `requirements.frontend.txt` | Frontend deps |
+| `requirements.vllm.txt` | vLLM-specific deps |
+| `requirements.benchmark.txt` | Benchmark and profiling tools |
+| `requirements.test.txt` | Test-only deps |
+| `requirements.dev.txt` | Dev-only tools |
+
+## Version Pinning Strategy
+
+- Use `==` for packages that are pure Python and well-tested.
+- Use `<=` or `<` for packages that may have platform-specific builds
+  (CUDA, system packages) — the max available version can differ across
+  x86_64 / aarch64 and CUDA versions.
+- **Never use `>=`** as it allows untested future versions that may introduce
+  breaking changes, create non-reproducible builds, and cause dependency
+  conflicts. Every installed version should be explicitly tested.

container/deps/requirements.benchmark.txt

+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Benchmark and profiling tool dependencies.
+
+aiperf @ git+https://github.com/ai-dynamo/aiperf.git@0746a7cdeb259f365e1124ffa94b2a976207b668
+genai-perf==0.0.15

container/deps/requirements.common.txt

+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Core runtime dependencies shared by ALL Dynamo containers.
+# See README.md in this directory for version pinning strategy.
+
+fastapi==0.120.1
+grpcio-tools<=1.76.0  # May have platform-specific builds; pins grpcio ecosystem version
+httpx==0.28.1
+msgpack==1.1.2
+msgspec==0.19.0
+nvidia-ml-py<=13.580.65  # NVIDIA/CUDA related, may vary by driver version
+opentelemetry-api<=1.38.0  # Pinned for tracing support
+opentelemetry-exporter-otlp<=1.38.0
+opentelemetry-sdk<=1.38.0
+prometheus_client==0.23.1
+protobuf>=5.29.5,<7.0.0
+pydantic>=2.11.4,<2.13
+pydantic-settings<2.13.0
+PyYAML==6.0.3
+tensorboard>=2.19.0,<2.21.0
+tensorboardX==2.6.2.2
+# Transformers version constraint for container builds
+# - vLLM 0.11.0: >=4.55.2, vLLM 0.11.2: >=4.56.0,<5
+# - TensorRT-LLM 1.3.0rc5: ==4.57.1
+# - SGLang 0.5.8: ==4.57.1
+# Using >=4.56.0 to satisfy all frameworks
+transformers>=4.56.0
+uvicorn==0.38.0

container/deps/requirements.dev.txt

+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Development-only tools — NOT for runtime or test containers.
+
+mypy==1.18.2
+pre-commit==4.5.0
+pyright==1.1.407

container/deps/requirements.frontend.txt

+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# tritonclient<=2.62.0 requires grpcio<1.68 and protobuf<6.0dev; pin here so the
+# resolver picks compatible versions when installed alongside runtime deps.
+grpcio<1.68,>=1.63.0
+protobuf>=5.29.5,<6.0dev
+# Frontend-specific dependencies — used by the frontend runtime and any test
+# image that needs to run gRPC frontend tests (e.g. dynamo test image).
+
+# Triton client for gRPC frontend tests
+tritonclient[grpc]<=2.62.0  # May have platform-specific builds

container/deps/requirements.planner.txt

+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Dependencies for planner, profiler, global_planner, and deploy utils.
+
+aiconfigurator[webapp] @ git+https://github.com/ai-dynamo/aiconfigurator.git@c59086c9e4fd88159db1f4c4259600ce2ca5435a
+aiofiles<=25.1.0
+filterpy==1.4.5
+kubernetes==32.0.1
+kubernetes_asyncio==32.0.0
+matplotlib==3.10.7
+pmdarima==2.1.1
+prometheus-api-client==0.6.0
+prophet==1.2.1
+scikit-learn==1.7.2
+scipy<1.14.0  # Upper bound for pmdarima compatibility

container/deps/requirements.standard.txt

-# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-# Version Pinning Strategy:
-# - Use == for packages that are pure Python and well-tested
-# - Use <= or < for packages that may have platform-specific versions
-# - Never use >= as it allows untested future versions that may introduce breaking changes,
-#   create non-reproducible builds, and cause dependency conflicts. Every installed version
-#   should be explicitly tested, not an unknown future release.
-
-# CUDA-specific packages that may have platform variations
-ucx-py-cu12<=0.45.0  # CUDA 12 variant, max version may differ by platform/architecture

container/deps/requirements.test.txt

 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# Version Pinning Strategy:
-# - Use == for packages that are pure Python and well-tested
-# - Use <= or < for packages that may have platform-specific versions
-# - Never use >= as it allows untested future versions that may introduce breaking changes,
-#   create non-reproducible builds, and cause dependency conflicts. Every installed version
-#   should be explicitly tested, not an unknown future release.
+# Test-only dependencies — installed in test and dev containers, NOT in runtime.
 
 # For MinIO/S3 operations in LoRA tests (replaces AWS CLI dependency)
 boto3==1.42.4
@@ -15,17 +10,14 @@ boto3-stubs[s3]==1.42.9  # Type stubs for boto3 S3 client
 datasets==4.4.1
 # For Kubernetes operations in deploy tests
 kr8s==0.20.13
-kubernetes==32.0.1
 kubernetes_asyncio==32.0.0
 matplotlib==3.10.7
 mistral-common==1.9.1
 # For NATS object store verification in router tests
 nats-py==2.12.0
-pmdarima==2.1.1
-prometheus-api-client==0.6.0
 psutil<=7.0.0  # System package, may vary by platform (was >=5.0.0)
-pydantic==2.11.7
-pyright==1.1.407
+# Required by pyproject.toml warning filters (pydantic.warnings.PydanticDeprecatedSince20)
+pydantic>=2.11.4,<2.13
 pytest==8.4.2
 pytest-asyncio==1.3.0
 pytest-benchmark==5.2.3
@@ -38,13 +30,12 @@ pytest-mypy==1.0.1
 pytest-order==1.3.0
 pytest-timeout==2.4.0
 pytest-xdist==3.8.0
-pyyaml==6.0.3
 requests==2.32.5
+# Required by kr8s
+# https://github.com/kr8s-org/kr8s/blob/750022c3ebbb7988cddb5a979aca2ee8074a1069/examples/kubectl-ng/uv.lock#L988
 sniffio==1.3.1
 tabulate==0.9.0
-# Triton client to Dynamo gRPC server
-tritonclient[grpc]<=2.62.0  # May have platform-specific builds
-# add types library stub for PyYAML
+types-aiofiles>=24.1.0
 types-PyYAML==6.0.12.20250915
 types-requests==2.32.4.20250913
 websocket-client==1.9.0

container/deps/requirements.txt

-# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-# Version Pinning Strategy:
-# - Use == for packages that are pure Python and well-tested
-# - Use <= or < for packages that may have platform-specific versions (CUDA, system packages)
-# - Never use >= as it allows untested future versions that may introduce breaking changes,
-#   create non-reproducible builds, and cause dependency conflicts. Every installed version
-#   should be explicitly tested, not an unknown future release.
-# - Platform differences: Some packages (especially CUDA-related) may have different
-#   maximum versions available on different platforms (x86_64 vs aarch64, different CUDA versions)
-
-# For Multimodal EPD (required for device_map="auto" in vision model loading)
-accelerate
-aiconfigurator[webapp] @ git+https://github.com/ai-dynamo/aiconfigurator.git@c59086c9e4fd88159db1f4c4259600ce2ca5435a
-aiofiles
-aiperf @ git+https://github.com/ai-dynamo/aiperf.git@0746a7cdeb259f365e1124ffa94b2a976207b668
-av==15.0.0
-fastapi==0.120.1
-filterpy==1.4.5
-ftfy==6.3.1
-genai-perf==0.0.15
-grpcio-tools<=1.76.0  # May have platform-specific builds
-httpx==0.28.1
-kr8s==0.20.13
-kubernetes==32.0.1
-kubernetes_asyncio<=32.1.1  # May vary by platform
-matplotlib==3.10.7
-msgpack==1.1.2
-msgspec==0.19.0
-mypy==1.18.2
-nvidia-ml-py<=13.580.65  # NVIDIA/CUDA related, may vary by driver version
-nvtx==0.2.14
-opentelemetry-api<=1.38.0  # May need to stay in sync with other components
-opentelemetry-exporter-otlp<=1.38.0  # May need to stay in sync with other components
-opentelemetry-sdk<=1.38.0  # May need to stay in sync with other components
-pip<=25.0.1  # System pip, varies by platform
-pmdarima==2.1.1
-pre-commit==4.5.0
-prometheus-api-client==0.6.0
-prometheus_client==0.23.1
-prophet==1.2.1
-protobuf>=5.29.5,<7.0.0
-pydantic>=2.11.4,<2.13  # vllm==0.12.0 depends on pydantic>=2.12.0
-pydantic-settings<2.13.0  # tensorrt_llm DynamicYamlWithDeepMergeSettingsSource incompatible with 2.13.0+ deep_merge param
-pyright==1.1.407
-PyYAML==6.0.3
-scikit-learn==1.7.2
-scipy<1.14.0  # Upper bound for pmdarima compatibility
-sentencepiece==0.2.1
-# Required by kr8s
-# https://github.com/kr8s-org/kr8s/blob/750022c3ebbb7988cddb5a979aca2ee8074a1069/examples/kubectl-ng/uv.lock#L988
-sniffio==1.3.1
-tensorboard>=2.19.0,<2.21.0
-tensorboardX==2.6.2.2
-# Transformers version constraint for container builds
-# - vLLM 0.11.0: >=4.55.2, vLLM 0.11.2: >=4.56.0,<5
-# - TensorRT-LLM 1.3.0rc5: ==4.57.1
-# - SGLang 0.5.8: ==4.57.1
-# Using >=4.56.0 to satisfy all frameworks
-transformers>=4.56.0
-types-aiofiles==25.1.0.20251011
-types-PyYAML==6.0.12.20250915
-uvicorn==0.38.0

container/deps/requirements.vllm.txt

+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# vLLM-specific runtime dependencies (multimodal features).
+
+av==15.0.0
+ftfy==6.3.1
+nvtx==0.2.14
+sentencepiece==0.2.1

container/templates/dev.Dockerfile

@@ -313,12 +313,12 @@ RUN if [ ! -d /opt/dynamo/venv ]; then \
 # Initialize Git LFS for the dynamo user (required for requirements with lfs=true)
 RUN git lfs install
 
-# Install common and test dependencies (matches main Dockerfile dev stage)
-# This installs pytest-benchmark and other test dependencies required for CI
-# TRT-LLM specific: Also installs cupy-cuda13x with special index strategy (Dockerfile.trtllm lines 768-776)
+# Install only the ADDITIONAL dev/test dependencies.
+# Runtime deps (common, framework, planner, benchmark) are already installed
+# in the parent runtime image — re-resolving them here would risk version drift.
 # SGLang specific: Reinstall pytest to ensure venv has pytest executable with correct shebang
 ARG FRAMEWORK
-RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requirements.txt \
+RUN --mount=type=bind,source=./container/deps/requirements.dev.txt,target=/tmp/requirements.dev.txt \
     --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.test.txt \
     # Cache uv downloads; uv handles its own locking for this cache.
     --mount=type=cache,target=/root/.cache/uv \
@@ -326,7 +326,7 @@ RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requi
     uv pip install \
         --index-strategy unsafe-best-match \
         --extra-index-url https://download.pytorch.org/whl/cu130 \
-        --requirement /tmp/requirements.txt \
+        --requirement /tmp/requirements.dev.txt \
         --requirement /tmp/requirements.test.txt && \
     if [ "${FRAMEWORK}" = "sglang" ]; then \
         uv pip install --force-reinstall --no-deps pytest; \

container/templates/dynamo_runtime.Dockerfile

@@ -134,16 +134,21 @@ RUN --mount=type=cache,target=/home/dynamo/.cache/uv,uid=1000,gid=0,mode=0775 \
 # Initialize Git LFS (required for git+https dependencies with LFS artifacts)
 RUN git lfs install
 
-# Install test and common dependencies
-RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requirements.txt \
-    --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.test.txt \
+# Install runtime dependencies (common + planner + frontend).
+# Frontend deps (tritonclient + grpcio/protobuf pins) are installed here so the resolver
+# sees all constraints in one pass, avoiding grpcio downgrades in the test layer.
+# Test and dev dependencies are NOT installed here — they go in the test and dev images.
+RUN --mount=type=bind,source=./container/deps/requirements.common.txt,target=/tmp/requirements.common.txt \
+    --mount=type=bind,source=./container/deps/requirements.planner.txt,target=/tmp/requirements.planner.txt \
+    --mount=type=bind,source=./container/deps/requirements.frontend.txt,target=/tmp/requirements.frontend.txt \
     --mount=type=cache,target=/home/dynamo/.cache/uv,uid=1000,gid=0,mode=0775 \
     export UV_CACHE_DIR=/home/dynamo/.cache/uv UV_GIT_LFS=1 UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 && \
     uv pip install \
         --index-strategy unsafe-best-match \
         --extra-index-url https://download.pytorch.org/whl/cu130 \
-        --requirement /tmp/requirements.txt \
-        --requirement /tmp/requirements.test.txt
+        --requirement /tmp/requirements.common.txt \
+        --requirement /tmp/requirements.planner.txt \
+        --requirement /tmp/requirements.frontend.txt
 
 # TODO: skip /workspace COPY for dev/local-dev (bind-mounted from host, gets shadowed)
 # Copy workspace source code

container/templates/frontend.Dockerfile

@@ -72,14 +72,16 @@ RUN --mount=type=cache,target=/home/dynamo/.cache/uv,uid=1000,gid=0,mode=0775 \
     mkdir -p /opt/dynamo/venv && \
     uv venv /opt/dynamo/venv --python $PYTHON_VERSION
 
-# Install common and test dependencies. In an ideal world, we'd use a mirror of PyPI for much more reliable downloads.
-RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requirements.txt \
-    --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.test.txt \
+# Install runtime dependencies (common + frontend).
+# Frontend needs tritonclient and its grpcio/protobuf constraints for gRPC serving.
+# Test and dev dependencies are NOT installed here — they go in the test and dev images.
+RUN --mount=type=bind,source=./container/deps/requirements.common.txt,target=/tmp/requirements.common.txt \
+    --mount=type=bind,source=./container/deps/requirements.frontend.txt,target=/tmp/requirements.frontend.txt \
     --mount=type=cache,target=/home/dynamo/.cache/uv,uid=1000,gid=0,mode=0775 \
     export UV_CACHE_DIR=/home/dynamo/.cache/uv UV_GIT_LFS=1 UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 && \
     uv pip install \
-        --requirement /tmp/requirements.txt \
-        --requirement /tmp/requirements.test.txt
+        --requirement /tmp/requirements.common.txt \
+        --requirement /tmp/requirements.frontend.txt
 
 ARG ENABLE_KVBM
 ARG ENABLE_GPU_MEMORY_SERVICE

container/templates/sglang_runtime.Dockerfile

@@ -121,14 +121,17 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
 COPY --chmod=775 --chown=dynamo:0 benchmarks/ /workspace/benchmarks/
 {% endif %}
 
-# Install common and test dependencies as root
-RUN --mount=type=bind,source=container/deps/requirements.txt,target=/tmp/deps/requirements.txt \
-    --mount=type=bind,source=container/deps/requirements.test.txt,target=/tmp/deps/requirements.test.txt \
+# Install runtime dependencies (common + planner + benchmarks) as root.
+# Test and dev dependencies are NOT installed here — they go in the test and dev images.
+RUN --mount=type=bind,source=container/deps/requirements.common.txt,target=/tmp/deps/requirements.common.txt \
+    --mount=type=bind,source=container/deps/requirements.planner.txt,target=/tmp/deps/requirements.planner.txt \
+    --mount=type=bind,source=container/deps/requirements.benchmark.txt,target=/tmp/deps/requirements.benchmark.txt \
     --mount=type=cache,target=/root/.cache/pip,sharing=locked \
     export PIP_CACHE_DIR=/root/.cache/pip && \
     pip install --break-system-packages \
-        --requirement /tmp/deps/requirements.txt \
-        --requirement /tmp/deps/requirements.test.txt \
+        --requirement /tmp/deps/requirements.common.txt \
+        --requirement /tmp/deps/requirements.planner.txt \
+        --requirement /tmp/deps/requirements.benchmark.txt \
         sglang==${SGLANG_VERSION} && \
     #TODO: Temporary change until upstream sglang runtime image is updated
     pip install --break-system-packages "urllib3>=2.6.3"
@@ -142,7 +145,7 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
     chmod -R g+w /workspace/benchmarks
 {% endif %}
 
-# Force-reinstall NVIDIA packages in a separate layer so requirements.txt changes don't trigger re-download
+# Force-reinstall NVIDIA packages in a separate layer so requirements changes don't trigger re-download
 RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
     export PIP_CACHE_DIR=/root/.cache/pip && \
     CUDA_MAJOR=$(nvcc --version | egrep -o 'cuda_[0-9]+' | cut -d_ -f2) && \

container/templates/trtllm_runtime.Dockerfile

@@ -252,17 +252,20 @@ RUN --mount=type=cache,target=/home/dynamo/.cache/uv,uid=1000,gid=0,mode=0775 \
         if [ -n "$GMS_WHEEL" ]; then uv pip install "$GMS_WHEEL"; fi; \
     fi
 
-# Install common and test dependencies
+# Install runtime dependencies (common + planner + benchmarks).
+# Test and dev dependencies are NOT installed here — they go in the test and dev images.
 # --no-cache is intentional: mixed indexes (PyPI + PyTorch CUDA wheels) risk serving stale/wrong-variant cached wheels
-RUN --mount=type=bind,source=./container/deps/requirements.txt,target=/tmp/requirements.txt \
-    --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/requirements.test.txt \
+RUN --mount=type=bind,source=./container/deps/requirements.common.txt,target=/tmp/requirements.common.txt \
+    --mount=type=bind,source=./container/deps/requirements.planner.txt,target=/tmp/requirements.planner.txt \
+    --mount=type=bind,source=./container/deps/requirements.benchmark.txt,target=/tmp/requirements.benchmark.txt \
     export UV_GIT_LFS=1 UV_HTTP_TIMEOUT=300 UV_HTTP_RETRIES=5 && \
     uv pip install \
         --no-cache \
         --index-strategy unsafe-best-match \
         --extra-index-url https://download.pytorch.org/whl/cu130 \
-        --requirement /tmp/requirements.txt \
-        --requirement /tmp/requirements.test.txt \
+        --requirement /tmp/requirements.common.txt \
+        --requirement /tmp/requirements.planner.txt \
+        --requirement /tmp/requirements.benchmark.txt \
         cupy-cuda13x
 
 # Copy tests, deploy and components for CI with correct ownership