fix: Clean up Velonix PR workflows (#5405)

Signed-off-by: Dillon Cullinan <dcullinan@nvidia.com>

fix: Clean up Velonix PR workflows (#5405)
Signed-off-by: Dillon Cullinan <dcullinan@nvidia.com>
202dfc36 · Dillon Cullinan · GitHub · c5458b99 · 202dfc36
Unverified Commit 202dfc36 authored Jan 14, 2026 by Dillon Cullinan Committed by GitHub Jan 14, 2026
Hide whitespace changes
Inline Side-by-side

Showing with 36 additions and 255 deletions

.github/workflows/pr.yaml .github/workflows/pr.yaml +36 -255

No files found.
--- a/.github/workflows/container-validation-backends.yml
+++ b/.github/workflows/container-validation-backends.yml
 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-name: Docker Build and Test
+name: PR
 on:
  push:
@@ -65,7 +65,7 @@ jobs:
        platform:
          - { arch: amd64, runner: cpu-amd-m5-2xlarge }
          - { arch: arm64, runner: cpu-arm-r8g-4xlarge }
-    name: operator (${{ matrix.platform.arch }})
+    name: operator-build (${{ matrix.platform.arch }})
    runs-on: ${{ matrix.platform.runner }}
    steps:
      - name: Output Node Name
@@ -132,70 +132,6 @@ jobs:
          azure_push: 'true'
          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-  vllm-cuda-13:
-    needs: changed-files
-    if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.vllm == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - { arch: amd64, runner: gpu-l40-amd64 }
-          - { arch: arm64, runner: cpu-arm-r8g-4xlarge }
-    name: vllm-cuda-13 (${{ matrix.platform.arch }})
-    runs-on: ${{ matrix.platform.runner }}
-    steps:
-      - name: Output Node Name
-        shell: bash
-        run: |
-          echo ${K8S_NODE_NAME}
-      - name: Checkout code
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
-        with:
-          lfs: true
-      - name: Docker Login
-        uses: ./.github/actions/docker-login
-        with:
-          ngc_ci_access_token: ${{ secrets.NGC_CI_ACCESS_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
-          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
-      - name: Build Container
-        id: build-image
-        uses: ./.github/actions/docker-build
-        with:
-          framework: vllm
-          target: runtime
-          platform: 'linux/${{ matrix.platform.arch }}'
-          cuda_version: '13.0'
-          ci_token: ${{ secrets.CI_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - name: Docker Tag and Push
-        uses: ./.github/actions/docker-tag-push
-        with:
-          local_image: ${{ steps.build-image.outputs.image_tag }}
-          push_tags: ai-dynamo/dynamo:${{ github.sha }}-vllm-cuda13-${{ matrix.platform.arch }}
-          conditional_tag: ${{ github.ref_name == 'main' && format('ai-dynamo/dynamo:main-vllm-cuda13-{0}', matrix.platform.arch) || '' }}
-          aws_push: 'true'
-          azure_push: 'true'
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-      - name: Run tests
-        uses: ./.github/actions/pytest
-        with:
-          image_tag: ${{ steps.build-image.outputs.image_tag }}
-          # GH ARM runners have no GPUs, run CPU tests only on ARM
-          pytest_marks: ${{ matrix.platform.arch == 'amd64' && 'pre_merge and vllm and (gpu_0 or gpu_1)' || 'pre_merge and vllm and gpu_0' }}
-          framework: "vllm"
-          test_type: "pre_merge"
-          platform_arch: ${{ matrix.platform.arch }}
  vllm:
    needs: changed-files
    if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.vllm == 'true'
@@ -205,9 +141,15 @@ jobs:
        platform:
          - { arch: amd64, runner: gpu-l40-amd64 }
          - { arch: arm64, runner: cpu-arm-r8g-4xlarge }
-    name: vllm (${{ matrix.platform.arch }})
+        cuda_version:
+          - { major_minor: '13.0', major: '13' }
+          - { major_minor: '12.9', major: '12' }
+    name: vllm-build-test (cuda${{ matrix.cuda_version.major_minor}}, ${{ matrix.platform.arch }})
    runs-on: ${{ matrix.platform.runner }}
-    steps:
+    timeout-minutes: 90
+    env:
+      FRAMEWORK: vllm
+    steps: &runtime-container-build-push-test
      - name: Output Node Name
        shell: bash
        run: |
@@ -229,10 +171,10 @@ jobs:
        id: build-image
        uses: ./.github/actions/docker-build
        with:
-          framework: vllm
+          framework: ${{ env.FRAMEWORK }}
          target: runtime
          platform: 'linux/${{ matrix.platform.arch }}'
-          cuda_version: '12.9'
+          cuda_version: ${{ matrix.cuda_version.major_minor }}
          ci_token: ${{ secrets.CI_TOKEN }}
          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
          sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }}
@@ -244,88 +186,30 @@ jobs:
        with:
          local_image: ${{ steps.build-image.outputs.image_tag }}
          push_tags: |
-            ai-dynamo/dynamo:${{ github.sha }}-vllm-${{ matrix.platform.arch }}
+            ai-dynamo/dynamo:${{ github.sha }}-${{ env.FRAMEWORK }}-${{ matrix.platform.arch }}
-            ai-dynamo/dynamo:${{ github.sha }}-vllm-cuda12-${{ matrix.platform.arch }}
+            ai-dynamo/dynamo:${{ github.sha }}-${{ env.FRAMEWORK }}-cuda${{ matrix.cuda_version.major }}-${{ matrix.platform.arch }}
-          conditional_tag: ${{ github.ref_name == 'main' && format('ai-dynamo/dynamo:main-vllm-cuda12-{0}', matrix.platform.arch) || '' }}
+          conditional_tag: ${{ github.ref_name == 'main' && format('ai-dynamo/dynamo:main-{0}-cuda{1}-{2}', env.FRAMEWORK, matrix.cuda_version.major, matrix.platform.arch) || '' }}
          aws_push: 'true'
          azure_push: 'true'
          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
      - name: Run tests
+        # TODO: enable testing on ARM when these tests stop failing on collection on machines without GPUs:
+        #   components/src/dynamo/trtllm/tests/test_trtllm_autodeploy.py
+        #   components/src/dynamo/trtllm/tests/test_trtllm_unit.py
+        if: ${{ env.FRAMEWORK != 'trtllm' || matrix.platform.arch == 'amd64' }} # Excludes trtllm on arm64
        uses: ./.github/actions/pytest
        with:
          image_tag: ${{ steps.build-image.outputs.image_tag }}
          # GH ARM runners have no GPUs, run CPU tests only on ARM
-          pytest_marks: ${{ matrix.platform.arch == 'amd64' && 'pre_merge and vllm and (gpu_0 or gpu_1)' || 'pre_merge and vllm and gpu_0' }}
+          pytest_marks: ${{ matrix.platform.arch == 'amd64' && format('pre_merge and {0} and (gpu_0 or gpu_1)', env.FRAMEWORK) || format('pre_merge and {0} and gpu_0', env.FRAMEWORK) }}
-          framework: "vllm"
+          framework: ${{ env.FRAMEWORK }}
          test_type: "pre_merge"
          platform_arch: ${{ matrix.platform.arch }}
          enable_mypy: 'true'
          hf_token: ${{ secrets.HF_TOKEN }}
-  sglang-cuda-13:
-    needs: changed-files
-    if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.sglang == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - { arch: amd64, runner: gpu-l40-amd64 }
-          - { arch: arm64, runner: cpu-arm-r8g-4xlarge }
-    name: sglang-cuda-13 (${{ matrix.platform.arch }})
-    runs-on: ${{ matrix.platform.runner }}
-    steps:
-      - name: Output Node Name
-        shell: bash
-        run: |
-          echo ${K8S_NODE_NAME}
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
-      - name: Docker Login
-        uses: ./.github/actions/docker-login
-        with:
-          ngc_ci_access_token: ${{ secrets.NGC_CI_ACCESS_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
-          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
-      - name: Build Container
-        id: build-image
-        uses: ./.github/actions/docker-build
-        with:
-          framework: sglang
-          target: runtime
-          platform: 'linux/${{ matrix.platform.arch }}'
-          cuda_version: '13.0'
-          ci_token: ${{ secrets.CI_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - name: Docker Tag and Push
-        uses: ./.github/actions/docker-tag-push
-        with:
-          local_image: ${{ steps.build-image.outputs.image_tag }}
-          push_tags: ai-dynamo/dynamo:${{ github.sha }}-sglang-cuda13-${{ matrix.platform.arch }}
-          conditional_tag: ${{ github.ref_name == 'main' && format('ai-dynamo/dynamo:main-sglang-cuda13-{0}', matrix.platform.arch) || '' }}
-          aws_push: 'true'
-          azure_push: 'true'
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-      - name: Run tests
-        uses: ./.github/actions/pytest
-        with:
-          image_tag: ${{ steps.build-image.outputs.image_tag }}
-          # GH arm runners have no GPUs, testing arm containers on cpu only
-          pytest_marks: ${{ matrix.platform.arch == 'amd64' && 'pre_merge and sglang and (gpu_0 or gpu_1)' || 'pre_merge and sglang and gpu_0' }}
-          framework: "sglang"
-          test_type: "pre_merge"
-          platform_arch: ${{ matrix.platform.arch }}
  sglang:
    needs: changed-files
    if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.sglang == 'true'
@@ -335,64 +219,15 @@ jobs:
        platform:
          - { arch: amd64, runner: gpu-l40-amd64 }
          - { arch: arm64, runner: cpu-arm-r8g-4xlarge }
-    name: sglang (${{ matrix.platform.arch }})
+        cuda_version:
+          - { major_minor: '13.0', major: '13' }
+          - { major_minor: '12.9', major: '12' }
+    name: sglang-build-test (cuda${{ matrix.cuda_version.major_minor}}, ${{ matrix.platform.arch }})
    runs-on: ${{ matrix.platform.runner }}
-    steps:
+    timeout-minutes: 90
-      - name: Output Node Name
+    env:
-        shell: bash
+      FRAMEWORK: sglang
-        run: |
+    steps: *runtime-container-build-push-test
-          echo ${K8S_NODE_NAME}
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
-        with:
-          lfs: true
-      - name: Docker Login
-        uses: ./.github/actions/docker-login
-        with:
-          ngc_ci_access_token: ${{ secrets.NGC_CI_ACCESS_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
-          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
-      - name: Build Container
-        id: build-image
-        uses: ./.github/actions/docker-build
-        with:
-          framework: sglang
-          cuda_version: '12.9'
-          target: runtime
-          platform: 'linux/${{ matrix.platform.arch }}'
-          ci_token: ${{ secrets.CI_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - name: Docker Tag and Push
-        uses: ./.github/actions/docker-tag-push
-        with:
-          local_image: ${{ steps.build-image.outputs.image_tag }}
-          push_tags: |
-            ai-dynamo/dynamo:${{ github.sha }}-sglang-${{ matrix.platform.arch }}
-            ai-dynamo/dynamo:${{ github.sha }}-sglang-cuda12-${{ matrix.platform.arch }}
-          conditional_tag: ${{ github.ref_name == 'main' && format('ai-dynamo/dynamo:main-sglang-cuda12-{0}', matrix.platform.arch) || '' }}
-          aws_push: 'true'
-          azure_push: 'true'
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-      - name: Run tests
-        uses: ./.github/actions/pytest
-        with:
-          image_tag: ${{ steps.build-image.outputs.image_tag }}
-          # GH arm runners have no GPUs, testing arm containers on cpu only
-          pytest_marks: ${{ matrix.platform.arch == 'amd64' && 'pre_merge and sglang and (gpu_0 or gpu_1)' || 'pre_merge and sglang and gpu_0' }}
-          framework: "sglang"
-          test_type: "pre_merge"
-          platform_arch: ${{ matrix.platform.arch }}
-          enable_mypy: 'true'
-          hf_token: ${{ secrets.HF_TOKEN }}
  trtllm:
    needs: changed-files
@@ -403,68 +238,14 @@ jobs:
        platform:
          - { arch: amd64, runner: gpu-l40-amd64 }
          - { arch: arm64, runner: cpu-arm-r8g-4xlarge }
-    name: trtllm (${{ matrix.platform.arch }})
+        cuda_version:
+          - { major_minor: '13.0', major: '13' }
+    name: trtllm-build-test (cuda${{ matrix.cuda_version.major_minor}}, ${{ matrix.platform.arch }})
    runs-on: ${{ matrix.platform.runner }}
-    steps:
+    timeout-minutes: 90
-      - name: Output Node Name
+    env:
-        shell: bash
+      FRAMEWORK: trtllm
-        run: |
+    steps: *runtime-container-build-push-test
-          echo ${K8S_NODE_NAME}
-      - name: Checkout code
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
-        with:
-          lfs: true
-      - name: Docker Login
-        uses: ./.github/actions/docker-login
-        with:
-          ngc_ci_access_token: ${{ secrets.NGC_CI_ACCESS_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
-          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
-      - name: Build Container
-        id: build-image
-        uses: ./.github/actions/docker-build
-        with:
-          framework: trtllm
-          cuda_version: '13.0'
-          target: runtime
-          platform: 'linux/${{ matrix.platform.arch }}'
-          ci_token: ${{ secrets.CI_TOKEN }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }}
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - name: Docker Tag and Push
-        uses: ./.github/actions/docker-tag-push
-        with:
-          local_image: ${{ steps.build-image.outputs.image_tag }}
-          push_tags: |
-            ai-dynamo/dynamo:${{ github.sha }}-trtllm-${{ matrix.platform.arch }}
-            ai-dynamo/dynamo:${{ github.sha }}-trtllm-cuda13-${{ matrix.platform.arch }}
-          conditional_tag: ${{ github.ref_name == 'main' && format('ai-dynamo/dynamo:main-trtllm-cuda13-{0}', matrix.platform.arch) || '' }}
-          aws_push: 'true'
-          azure_push: 'true'
-          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
-          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-      - name: Run tests
-        # TODO: enable testing on ARM when these tests stop failing on collection on machines without GPUs:
-        #  components/src/dynamo/trtllm/tests/test_trtllm_autodeploy.py
-        #  components/src/dynamo/trtllm/tests/test_trtllm_unit.py
-        if: ${{ matrix.platform.arch != 'arm64' }}
-        uses: ./.github/actions/pytest
-        with:
-          image_tag: ${{ steps.build-image.outputs.image_tag }}
-          # GH arm runners have no GPUs, testing arm containers on cpu only
-          pytest_marks: ${{ matrix.platform.arch == 'amd64' && 'pre_merge and trtllm and (gpu_0 or gpu_1)' || 'pre_merge and trtllm and gpu_0' }}
-          framework: "trtllm"
-          test_type: "pre_merge"
-          platform_arch: ${{ matrix.platform.arch }}
-          enable_mypy: 'true'
-          hf_token: ${{ secrets.HF_TOKEN }}
  deploy-operator:
    runs-on: cpu-amd-m5-2xlarge