ci: switch from dorny/paths-filter to tj-actions/changed-files (#5160)

Signed-off-by: alec-flowers <aflowers@nvidia.com>

.github/filters.yaml

+all:
+  - '**'
+
 docs: &docs
   - 'docs/**'
   - '**/*.md'
@@ -9,9 +12,11 @@ ci: &ci
   - '.github/filters.yaml'
   - '.github/actions/**'
 
+# Will run tests for all 3 containers (vLLM, SGLang, TRTLLM)
 core:
   - *ci
   - 'container/build.sh'
+  - 'container/Dockerfile'
   - 'container/deps/*'
   - 'lib/**'
   - 'tests/**'

.github/scripts/package.json

+{
+  "name": "ci-filter-tests",
+  "version": "1.0.0",
+  "description": "Tests for .github/filters.yaml pattern matching",
+  "scripts": {
+    "test": "node test-filters.js"
+  },
+  "dependencies": {
+    "micromatch": "^4.0.8",
+    "yaml": "^2.7.1"
+  }
+}
+

.github/scripts/test-filters.js

+#!/usr/bin/env node
+/**
+ * Test script for .github/filters.yaml pattern matching.
+ * Reads patterns directly from filters.yaml and validates behavior.
+ *
+ * Usage:
+ *   cd .github/scripts
+ *   npm install
+ *   npm test
+ *
+ * This validates that tj-actions/changed-files will correctly:
+ * - Match backend-specific files to their respective filters (vllm, sglang, trtllm)
+ * - Exclude doc files (*.md, *.rst, *.txt) from core via negation patterns
+ * - Match CI/infrastructure changes to core
+ */
+
+const fs = require('fs');
+const path = require('path');
+const micromatch = require('micromatch');
+const YAML = require('yaml');
+
+// Find filters.yaml relative to this script
+const scriptDir = path.dirname(__filename);
+const filtersPath = path.resolve(scriptDir, '../filters.yaml');
+
+console.log(`Reading filters from: ${filtersPath}\n`);
+
+// Parse YAML (handles anchors/aliases automatically)
+const filtersYaml = fs.readFileSync(filtersPath, 'utf8');
+const filters = YAML.parse(filtersYaml);
+
+// Flatten nested arrays (YAML anchors create nested arrays)
+function flattenPatterns(patterns) {
+  if (!patterns || !Array.isArray(patterns)) return [];
+  return patterns.flat(Infinity).filter(p => typeof p === 'string');
+}
+
+// Simulate tj-actions/changed-files behavior with negation
+function checkFilter(file, patterns) {
+  const flat = flattenPatterns(patterns);
+  if (flat.length === 0) return false;
+
+  const positive = flat.filter(p => !p.startsWith('!'));
+  const negative = flat.filter(p => p.startsWith('!')).map(p => p.slice(1));
+
+  const matchesPositive = micromatch.isMatch(file, positive);
+  const matchesNegative = negative.length > 0 && micromatch.isMatch(file, negative);
+
+  return matchesPositive && !matchesNegative;
+}
+
+// Test cases: [file, expectations, description]
+// expectations: { filterName: expectedValue, ... }
+const testCases = [
+  // Backend-specific files should only trigger their backend
+  {
+    file: 'examples/backends/vllm/launch/dsr1_dep.sh',
+    expect: { core: false, vllm: true, sglang: false, trtllm: false },
+    desc: 'vllm script triggers only vllm'
+  },
+  {
+    file: 'examples/backends/sglang/example.py',
+    expect: { core: false, vllm: false, sglang: true, trtllm: false },
+    desc: 'sglang script triggers only sglang'
+  },
+  {
+    file: 'examples/backends/trtllm/example.py',
+    expect: { core: false, vllm: false, sglang: false, trtllm: true },
+    desc: 'trtllm script triggers only trtllm'
+  },
+  {
+    file: 'components/src/dynamo/vllm/worker.py',
+    expect: { core: false, vllm: true },
+    desc: 'vllm component triggers only vllm'
+  },
+
+  // Doc files should be excluded from core (negation patterns)
+  {
+    file: 'lib/README.md',
+    expect: { core: false, vllm: false, docs: true },
+    desc: 'lib README excluded from core, matches docs'
+  },
+  {
+    file: 'tests/README.md',
+    expect: { core: false, docs: true },
+    desc: 'tests README excluded from core'
+  },
+  {
+    file: 'lib/docs/guide.txt',
+    expect: { core: false, docs: true },
+    desc: 'txt file excluded from core'
+  },
+  {
+    file: 'docs/guide.md',
+    expect: { core: false, docs: true },
+    desc: 'docs folder matches docs filter'
+  },
+
+  // Code files should trigger core
+  {
+    file: 'lib/runtime/src/main.rs',
+    expect: { core: true, vllm: false },
+    desc: 'rust file triggers core'
+  },
+  {
+    file: 'lib/runtime/Cargo.toml',
+    expect: { core: true },
+    desc: 'Cargo.toml triggers core'
+  },
+  {
+    file: 'tests/test_something.py',
+    expect: { core: true },
+    desc: 'python test triggers core'
+  },
+  {
+    file: 'components/src/dynamo/router/router.py',
+    expect: { core: true },
+    desc: 'router triggers core'
+  },
+  {
+    file: 'components/src/dynamo/frontend/server.py',
+    expect: { core: true },
+    desc: 'frontend triggers core'
+  },
+
+  // CI files should trigger core
+  {
+    file: '.github/workflows/ci.yml',
+    expect: { core: true },
+    desc: 'workflow triggers core'
+  },
+  {
+    file: '.github/filters.yaml',
+    expect: { core: true },
+    desc: 'filters.yaml triggers core'
+  },
+  {
+    file: '.github/actions/docker-build/action.yml',
+    expect: { core: true },
+    desc: 'action triggers core'
+  },
+
+  // Root level files
+  {
+    file: 'pyproject.toml',
+    expect: { core: true },
+    desc: 'root toml triggers core'
+  },
+  {
+    file: 'setup.py',
+    expect: { core: true },
+    desc: 'root py triggers core'
+  },
+
+  // Operator and deploy
+  {
+    file: 'deploy/cloud/operator/main.go',
+    expect: { core: false, operator: true },
+    desc: 'operator file triggers operator'
+  },
+  {
+    file: 'deploy/cloud/helm/values.yaml',
+    expect: { core: false, deploy: true },
+    desc: 'helm file triggers deploy'
+  },
+];
+
+// Print available filters
+console.log('Loaded filters:', Object.keys(filters).join(', '));
+console.log('');
+
+console.log('Testing filter patterns\n');
+console.log('File                                           | Result');
+console.log('-----------------------------------------------|--------');
+
+let passed = 0;
+let failed = 0;
+
+testCases.forEach(({ file, expect, desc }) => {
+  const results = {};
+  let allMatch = true;
+
+  // Check each expected filter
+  for (const [filterName, expectedValue] of Object.entries(expect)) {
+    const actual = checkFilter(file, filters[filterName]);
+    results[filterName] = actual;
+    if (actual !== expectedValue) {
+      allMatch = false;
+    }
+  }
+
+  if (allMatch) {
+    passed++;
+    const matchedFilters = Object.entries(results)
+      .filter(([_, v]) => v)
+      .map(([k, _]) => k)
+      .join(', ') || 'none';
+    console.log(`✓ ${file.padEnd(45)} | ${matchedFilters}`);
+  } else {
+    failed++;
+    console.log(`✗ ${file.padEnd(45)} | FAIL`);
+    console.log(`  ${desc}`);
+    for (const [filterName, expectedValue] of Object.entries(expect)) {
+      const actual = results[filterName];
+      if (actual !== expectedValue) {
+        console.log(`  ${filterName}: expected=${expectedValue}, got=${actual}`);
+      }
+    }
+  }
+});
+
+console.log(`\n${passed}/${testCases.length} tests passed`);
+
+if (failed > 0) {
+  console.error(`\n${failed} test(s) failed!`);
+  process.exit(1);
+}
+
+console.log('\nAll filter tests passed! ✓');

.github/workflows/container-validation-backends.yml

@@ -30,20 +30,73 @@ jobs:
     runs-on: ubuntu-latest
     environment: ${{ github.event_name == 'workflow_dispatch' && 'protected-deploy' || '' }}
     outputs:
-      core: ${{ steps.filter.outputs.core }}
-      operator: ${{ steps.filter.outputs.operator }}
-      deploy: ${{ steps.filter.outputs.deploy }}
-      vllm: ${{ steps.filter.outputs.vllm }}
-      sglang: ${{ steps.filter.outputs.sglang }}
-      trtllm: ${{ steps.filter.outputs.trtllm }}
+      core: ${{ steps.filter.outputs.core_any_changed }}
+      operator: ${{ steps.filter.outputs.operator_any_changed }}
+      deploy: ${{ steps.filter.outputs.deploy_any_changed }}
+      vllm: ${{ steps.filter.outputs.vllm_any_changed }}
+      sglang: ${{ steps.filter.outputs.sglang_any_changed }}
+      trtllm: ${{ steps.filter.outputs.trtllm_any_changed }}
     steps:
       - name: Checkout code
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
+        with:
+          fetch-depth: 0
+      - name: Get merge base
+        id: merge-base
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # For PR branches, compare against merge-base with PR's target branch
+          # For main/release branches, use default behavior (previous commit)
+          if [[ "${{ github.ref_name }}" =~ ^pull-request/([0-9]+)$ ]]; then
+            PR_NUMBER="${BASH_REMATCH[1]}"
+            BASE_BRANCH=$(gh pr view "$PR_NUMBER" --json baseRefName -q '.baseRefName')
+            echo "PR #$PR_NUMBER targets branch: $BASE_BRANCH"
+            BASE_SHA=$(git merge-base "origin/$BASE_BRANCH" HEAD)
+            echo "base_sha=$BASE_SHA" >> $GITHUB_OUTPUT
+            echo "Using merge-base with $BASE_BRANCH: $BASE_SHA"
+          else
+            echo "base_sha=" >> $GITHUB_OUTPUT
+            echo "Using default comparison (previous commit)"
+          fi
       - name: Check for changes
-        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36  # v3.0.2
+        uses: tj-actions/changed-files@aa08304bd477b800d468db44fe10f6c61f7f7b11  # v42.1.0
         id: filter
         with:
-          filters: .github/filters.yaml
+          files_yaml_from_source_file: .github/filters.yaml
+          base_sha: ${{ steps.merge-base.outputs.base_sha }}
+      - name: Debug changed files
+        run: |
+          echo "=== Base SHA Used ==="
+          echo "base_sha: ${{ steps.merge-base.outputs.base_sha || 'default (previous commit)' }}"
+          echo ""
+          echo "=== All Changed Files (${{ steps.filter.outputs.all_all_changed_files_count }} total) ==="
+          echo "${{ steps.filter.outputs.all_all_changed_files }}"
+          echo ""
+          echo "=== Filters That Matched ==="
+          echo "changed_keys: ${{ steps.filter.outputs.changed_keys }}"
+          echo ""
+          echo "=== Filter Results (any_changed) ==="
+          echo "docs: ${{ steps.filter.outputs.docs_any_changed }}"
+          echo "ci: ${{ steps.filter.outputs.ci_any_changed }}"
+          echo "core: ${{ steps.filter.outputs.core_any_changed }}"
+          echo "operator: ${{ steps.filter.outputs.operator_any_changed }}"
+          echo "deploy: ${{ steps.filter.outputs.deploy_any_changed }}"
+          echo "planner: ${{ steps.filter.outputs.planner_any_changed }}"
+          echo "vllm: ${{ steps.filter.outputs.vllm_any_changed }}"
+          echo "sglang: ${{ steps.filter.outputs.sglang_any_changed }}"
+          echo "trtllm: ${{ steps.filter.outputs.trtllm_any_changed }}"
+          echo ""
+          echo "=== Files Matching Each Filter ==="
+          echo "docs: ${{ steps.filter.outputs.docs_all_changed_files }}"
+          echo "ci: ${{ steps.filter.outputs.ci_all_changed_files }}"
+          echo "core: ${{ steps.filter.outputs.core_all_changed_files }}"
+          echo "operator: ${{ steps.filter.outputs.operator_all_changed_files }}"
+          echo "deploy: ${{ steps.filter.outputs.deploy_all_changed_files }}"
+          echo "planner: ${{ steps.filter.outputs.planner_all_changed_files }}"
+          echo "vllm: ${{ steps.filter.outputs.vllm_all_changed_files }}"
+          echo "sglang: ${{ steps.filter.outputs.sglang_all_changed_files }}"
+          echo "trtllm: ${{ steps.filter.outputs.trtllm_all_changed_files }}"
 
   backend-status-check:
     runs-on: ubuntu-latest

.github/workflows/container-validation-dynamo.yml

@@ -19,15 +19,25 @@ jobs:
   changed-files:
     runs-on: ubuntu-latest
     outputs:
-      core: ${{ steps.filter.outputs.core }}
+      core: ${{ steps.filter.outputs.core_any_changed }}
     steps:
       - name: Checkout code
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
+        with:
+          fetch-depth: 0
       - name: Check for changes
-        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36  # v3.0.2
+        uses: tj-actions/changed-files@aa08304bd477b800d468db44fe10f6c61f7f7b11  # v42.1.0
         id: filter
         with:
-          filters: .github/filters.yaml
+          files_yaml_from_source_file: .github/filters.yaml
+      - name: Debug changed files
+        run: |
+          echo "=== All Changed Files (${{ steps.filter.outputs.all_all_changed_files_count }} total) ==="
+          echo "${{ steps.filter.outputs.all_all_changed_files }}"
+          echo ""
+          echo "=== Filter Results ==="
+          echo "core_any_changed: ${{ steps.filter.outputs.core_any_changed }}"
+          echo "core files: ${{ steps.filter.outputs.core_all_changed_files }}"
 
   dynamo-status-check:
     runs-on: ubuntu-latest

.gitignore

@@ -112,3 +112,7 @@ profiling_results*
 
 # Direnv
 .envrc
+
+# Node.js
+node_modules/
+package-lock.json