ci: add uncovered files check and improve filter coverage (#5214)

Signed-off-by: alec-flowers <aflowers@nvidia.com>

.github/FILTERS.md

+# CI Filters
+
+The `filters.yaml` file controls which CI jobs run based on changed files.
+
+## How It Works
+
+When you open a PR, CI checks which files changed and runs only relevant jobs:
+
+| Filter | Triggers |
+|--------|----------|
+| `core` | Main test suite (vLLM, SGLang, TRT-LLM containers) |
+| `operator` | Kubernetes operator tests |
+| `deploy` | Helm chart validation |
+| `vllm` / `sglang` / `trtllm` | Backend-specific tests |
+| `docs` | Nothing (classification only) |
+| `examples` | Nothing (classification only) |
+| `ignore` | Nothing (classification only) |
+
+> **Note:** `docs`, `examples`, and `ignore` don't trigger any CI jobs. They exist to satisfy coverage requirements - every file must match at least one filter.
+
+## Fixing "Uncovered Files" Errors
+
+If CI fails with:
+```
+ERROR: The following files are not covered by any CI filter
+```
+
+Add patterns to `filters.yaml`:
+
+1. **New source files** → Add to `core` or relevant backend filter
+2. **New examples/recipes** → Add to `examples`
+3. **Documentation** → Add to `docs`
+4. **Config files that don't need CI** → Add to `ignore`
+
+## Testing Locally
+
+```bash
+cd .github/scripts
+npm install
+npm run coverage  # Check if all repo files are covered
+```
+
+## Pattern Syntax
+
+- `**` matches any path depth (but not dotfiles by default)
+- `*` matches within a directory
+- `!pattern` excludes files (used in `core` to skip docs)
+- For dotfiles, add explicit pattern like `dir/.*`
+
+Example: `lib/**/*.rs` matches all Rust files under `lib/`.
+
+## Adding a New Filter Group
+
+If you create a new filter in `filters.yaml`, you must also update the workflows:
+
+1. Add the filter to `filters.yaml`
+2. Update **both** workflow files to include the new filter in the uncovered files check:
+   - `.github/workflows/container-validation-backends.yml`
+   - `.github/workflows/container-validation-dynamo.yml`
+
+In each workflow, find the `COVERED_FILES` line and add your new filter:
+
+```bash
+COVERED_FILES=$(echo "... ${{ steps.filter.outputs.YOURFILTER_all_modified_files }} ..." | ...)
+```
+
+If you skip this step, CI will fail with "uncovered files" even though your filter exists.
+

.github/filters.yaml

+# CI Filter Patterns - See FILTERS.md for documentation
+#
+# Filters that trigger CI jobs:
+#   core     -> dynamo build-test, all backend builds (vllm/sglang/trtllm), deploy tests
+#   operator -> operator build and test
+#   deploy   -> deploy-test-vllm-disagg-router
+#   vllm     -> vllm build and test
+#   sglang   -> sglang build and test
+#   trtllm   -> trtllm build and test
+#
+# Filters for coverage only (no CI triggered):
+#   docs, examples, ignore, planner
+
 all:
   - '**'
 
-docs: &docs
+docs:
   - 'docs/**'
   - '**/*.md'
   - '**/*.rst'
   - '**/*.txt'
+  - '**/.gitignore'
+  - '**/.helmignore'
+  - '**/.gitattributes'
+  - 'LICENSE'
+  - 'CODEOWNERS'
+
+examples:
+  - 'recipes/**'
+  - 'examples/**'
+  - 'benchmarks/**'
+  - '.devcontainer/**'
+  - 'deploy/discovery/**'
+  - 'deploy/inference-gateway/**'
+  - 'deploy/observability/**'
+  - 'deploy/pre-deployment/**'
+
+ignore:
+  - '.lycheeignore'
+  - '.pre-commit-config.yaml'
+  - '.coderabbit.yaml'
+  - '.clang-format'
+  - 'dynamo.code-workspace'
+  - 'deploy/*'
+  - 'launch/**'
+  - '.github/scripts/**'
+  - '.github/ISSUE_TEMPLATE/**'
+  - '.github/pull_request_template.md'
+  - '.github/release.yml'
+  - '.github/copy-pr-bot.yaml'
+  - '.github/dco.yml'
+  - 'container/Dockerfile.aws'
+  - 'container/Dockerfile.docs'
+  - 'container/Dockerfile.epp'
+  - 'container/Dockerfile.frontend'
+  - 'container/Dockerfile.local_dev'
+  - 'container/run.sh'
+  - 'container/use-sccache.sh'
 
 ci: &ci
   - '.github/workflows/**'
   - '.github/filters.yaml'
   - '.github/actions/**'
+  - '.github/scripts/parse_buildkit_output.py'
 
-# Will run tests for all 3 containers (vLLM, SGLang, TRTLLM)
 core:
   - *ci
   - 'container/build.sh'
   - 'container/Dockerfile'
+  - '.dockerignore'
   - 'container/deps/*'
+  - '.cargo/config.toml'
   - 'lib/**'
   - 'tests/**'
   - 'components/src/dynamo/router/**'
@@ -28,36 +80,39 @@ core:
   - '*.lock'
   - '*.py'
   - '*.rs'
-  - '!**/*.md' # Avoid running on doc changes
-  - '!**/*.rst' # Avoid running on doc changes
-  - '!**/*.txt' # Avoid running on doc changes
+  - 'scripts/report_pytest_markers.py'
+  - '!**/*.md'
+  - '!**/*.rst'
+  - '!**/*.txt'
 
 operator:
   - *ci
-  - 'deploy/cloud/operator/**'
+  - 'deploy/operator/**'
+  - 'deploy/operator/.*'
 
 deploy:
-  - 'deploy/cloud/helm/**'
+  - 'deploy/helm/**'
   - 'deploy/utils/**'
 
 planner:
   - 'components/src/dynamo/planner/**'
   - 'tests/planner/**'
 
-vllm: &vllm
+vllm:
   - 'container/Dockerfile.vllm'
   - 'container/deps/requirements.vllm.txt'
   - 'container/deps/vllm/**'
   - 'examples/backends/vllm/**'
   - 'components/src/dynamo/vllm/**'
 
-sglang: &sglang
+sglang:
   - 'container/Dockerfile.sglang'
   - 'examples/backends/sglang/**'
   - 'components/src/dynamo/sglang/**'
 
-trtllm: &trtllm
+trtllm:
   - 'container/Dockerfile.trtllm'
+  - 'container/deps/trtllm/**'
   - 'examples/backends/trtllm/**'
   - 'components/src/dynamo/trtllm/**'
   - 'container/build_trtllm_wheel.sh'

.github/scripts/package.json

@@ -3,7 +3,8 @@
   "version": "1.0.0",
   "description": "Tests for .github/filters.yaml pattern matching",
   "scripts": {
-    "test": "node test-filters.js"
+    "test": "node test-filters.js",
+    "coverage": "node test-filters.js --coverage"
   },
   "dependencies": {
     "micromatch": "^4.0.8",

.github/scripts/test-filters.js

@@ -6,19 +6,25 @@
  * Usage:
  *   cd .github/scripts
  *   npm install
- *   npm test
+ *   npm test                    # Run pattern tests only
+ *   npm run coverage            # Check full repo coverage
+ *   npm test -- --coverage      # Run both
  *
  * This validates that tj-actions/changed-files will correctly:
  * - Match backend-specific files to their respective filters (vllm, sglang, trtllm)
  * - Exclude doc files (*.md, *.rst, *.txt) from core via negation patterns
  * - Match CI/infrastructure changes to core
+ * - (with --coverage) Ensure all files in repo are covered by at least one filter
  */
 
 const fs = require('fs');
 const path = require('path');
+const { execSync } = require('child_process');
 const micromatch = require('micromatch');
 const YAML = require('yaml');
 
+const runCoverage = process.argv.includes('--coverage');
+
 // Find filters.yaml relative to this script
 const scriptDir = path.dirname(__filename);
 const filtersPath = path.resolve(scriptDir, '../filters.yaml');
@@ -154,12 +160,12 @@ const testCases = [
 
   // Operator and deploy
   {
-    file: 'deploy/cloud/operator/main.go',
+    file: 'deploy/operator/cmd/main.go',
     expect: { core: false, operator: true },
     desc: 'operator file triggers operator'
   },
   {
-    file: 'deploy/cloud/helm/values.yaml',
+    file: 'deploy/helm/charts/platform/values.yaml',
     expect: { core: false, deploy: true },
     desc: 'helm file triggers deploy'
   },
@@ -217,3 +223,72 @@ if (failed > 0) {
 }
 
 console.log('\nAll filter tests passed! ✓');
+
+// --- Coverage Check ---
+// Validates that all files in the repo are covered by at least one specific filter
+
+if (runCoverage) {
+  console.log('\n' + '='.repeat(60));
+  console.log('Running full repository coverage check...\n');
+
+  // Get repo root (two levels up from .github/scripts)
+  const repoRoot = path.resolve(scriptDir, '../..');
+
+  // Get all tracked files using git
+  let allFiles;
+  try {
+    const output = execSync('git ls-files', { cwd: repoRoot, encoding: 'utf8' });
+    allFiles = output.trim().split('\n').filter(f => f.length > 0);
+  } catch (err) {
+    console.error('Failed to run git ls-files. Are you in a git repository?');
+    process.exit(1);
+  }
+
+  console.log(`Found ${allFiles.length} tracked files in repository\n`);
+
+  // Specific filters to check (exclude 'all' since it matches everything)
+  const specificFilters = Object.keys(filters).filter(f => f !== 'all');
+
+  // Check each file
+  const uncoveredFiles = [];
+
+  for (const file of allFiles) {
+    let covered = false;
+    for (const filterName of specificFilters) {
+      if (checkFilter(file, filters[filterName])) {
+        covered = true;
+        break;
+      }
+    }
+    if (!covered) {
+      uncoveredFiles.push(file);
+    }
+  }
+
+  if (uncoveredFiles.length > 0) {
+    console.error(`ERROR: ${uncoveredFiles.length} file(s) not covered by any CI filter:\n`);
+
+    // Group by directory for readability
+    const byDir = {};
+    for (const file of uncoveredFiles) {
+      const dir = path.dirname(file) || '.';
+      if (!byDir[dir]) byDir[dir] = [];
+      byDir[dir].push(path.basename(file));
+    }
+
+    for (const [dir, files] of Object.entries(byDir).sort()) {
+      console.log(`  ${dir}/`);
+      for (const file of files.slice(0, 10)) {
+        console.log(`    - ${file}`);
+      }
+      if (files.length > 10) {
+        console.log(`    ... and ${files.length - 10} more`);
+      }
+    }
+
+    console.log('\nPlease add patterns for these files to .github/filters.yaml');
+    process.exit(1);
+  }
+
+  console.log(`All ${allFiles.length} files are covered by CI filters! ✓`);
+}

.github/workflows/container-validation-backends.yml

@@ -30,12 +30,12 @@ jobs:
     runs-on: ubuntu-latest
     environment: ${{ github.event_name == 'workflow_dispatch' && 'protected-deploy' || '' }}
     outputs:
-      core: ${{ steps.filter.outputs.core_any_changed }}
-      operator: ${{ steps.filter.outputs.operator_any_changed }}
-      deploy: ${{ steps.filter.outputs.deploy_any_changed }}
-      vllm: ${{ steps.filter.outputs.vllm_any_changed }}
-      sglang: ${{ steps.filter.outputs.sglang_any_changed }}
-      trtllm: ${{ steps.filter.outputs.trtllm_any_changed }}
+      core: ${{ steps.filter.outputs.core_any_modified }}
+      operator: ${{ steps.filter.outputs.operator_any_modified }}
+      deploy: ${{ steps.filter.outputs.deploy_any_modified }}
+      vllm: ${{ steps.filter.outputs.vllm_any_modified }}
+      sglang: ${{ steps.filter.outputs.sglang_any_modified }}
+      trtllm: ${{ steps.filter.outputs.trtllm_any_modified }}
     steps:
       - name: Checkout code
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
@@ -70,37 +70,57 @@ jobs:
           echo "=== Base SHA Used ==="
           echo "base_sha: ${{ steps.merge-base.outputs.base_sha || 'default (previous commit)' }}"
           echo ""
-          echo "=== All Changed Files (${{ steps.filter.outputs.all_all_changed_files_count }} total) ==="
-          echo "${{ steps.filter.outputs.all_all_changed_files }}"
+          echo "=== All Modified Files (${{ steps.filter.outputs.all_all_modified_files_count }} total) ==="
+          echo "${{ steps.filter.outputs.all_all_modified_files }}"
           echo ""
           echo "=== Filters That Matched ==="
           echo "changed_keys: ${{ steps.filter.outputs.changed_keys }}"
           echo ""
-          echo "=== Filter Results (any_changed) ==="
-          echo "docs: ${{ steps.filter.outputs.docs_any_changed }}"
-          echo "ci: ${{ steps.filter.outputs.ci_any_changed }}"
-          echo "core: ${{ steps.filter.outputs.core_any_changed }}"
-          echo "operator: ${{ steps.filter.outputs.operator_any_changed }}"
-          echo "deploy: ${{ steps.filter.outputs.deploy_any_changed }}"
-          echo "planner: ${{ steps.filter.outputs.planner_any_changed }}"
-          echo "vllm: ${{ steps.filter.outputs.vllm_any_changed }}"
-          echo "sglang: ${{ steps.filter.outputs.sglang_any_changed }}"
-          echo "trtllm: ${{ steps.filter.outputs.trtllm_any_changed }}"
+          echo "=== Filter Results ==="
+          echo "docs: ${{ steps.filter.outputs.docs_any_modified }}"
+          echo "ci: ${{ steps.filter.outputs.ci_any_modified }}"
+          echo "core: ${{ steps.filter.outputs.core_any_modified }}"
+          echo "operator: ${{ steps.filter.outputs.operator_any_modified }}"
+          echo "deploy: ${{ steps.filter.outputs.deploy_any_modified }}"
+          echo "planner: ${{ steps.filter.outputs.planner_any_modified }}"
+          echo "vllm: ${{ steps.filter.outputs.vllm_any_modified }}"
+          echo "sglang: ${{ steps.filter.outputs.sglang_any_modified }}"
+          echo "trtllm: ${{ steps.filter.outputs.trtllm_any_modified }}"
           echo ""
           echo "=== Files Matching Each Filter ==="
-          echo "docs: ${{ steps.filter.outputs.docs_all_changed_files }}"
-          echo "ci: ${{ steps.filter.outputs.ci_all_changed_files }}"
-          echo "core: ${{ steps.filter.outputs.core_all_changed_files }}"
-          echo "operator: ${{ steps.filter.outputs.operator_all_changed_files }}"
-          echo "deploy: ${{ steps.filter.outputs.deploy_all_changed_files }}"
-          echo "planner: ${{ steps.filter.outputs.planner_all_changed_files }}"
-          echo "vllm: ${{ steps.filter.outputs.vllm_all_changed_files }}"
-          echo "sglang: ${{ steps.filter.outputs.sglang_all_changed_files }}"
-          echo "trtllm: ${{ steps.filter.outputs.trtllm_all_changed_files }}"
+          echo "docs: ${{ steps.filter.outputs.docs_all_modified_files }}"
+          echo "ci: ${{ steps.filter.outputs.ci_all_modified_files }}"
+          echo "core: ${{ steps.filter.outputs.core_all_modified_files }}"
+          echo "operator: ${{ steps.filter.outputs.operator_all_modified_files }}"
+          echo "deploy: ${{ steps.filter.outputs.deploy_all_modified_files }}"
+          echo "planner: ${{ steps.filter.outputs.planner_all_modified_files }}"
+          echo "vllm: ${{ steps.filter.outputs.vllm_all_modified_files }}"
+          echo "sglang: ${{ steps.filter.outputs.sglang_all_modified_files }}"
+          echo "trtllm: ${{ steps.filter.outputs.trtllm_all_modified_files }}"
+      - name: Check for uncovered files
+        run: |
+          # Combine all filter-specific files into one list
+          COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u)
+
+          # Get all modified files
+          ALL_FILES=$(echo "${{ steps.filter.outputs.all_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u)
+
+          # Find files in ALL but not in COVERED
+          UNCOVERED=$(comm -23 <(echo "$ALL_FILES") <(echo "$COVERED_FILES"))
+
+          if [ -n "$UNCOVERED" ]; then
+            echo "::error::The following files are not covered by any CI filter:"
+            echo "$UNCOVERED"
+            echo ""
+            echo "Please add these paths to .github/filters.yaml"
+            echo "See .github/FILTERS.md for documentation"
+            exit 1
+          fi
+          echo "All modified files are covered by CI filters."
 
   backend-status-check:
     runs-on: ubuntu-latest
-    needs: [vllm, sglang, trtllm, operator]
+    needs: [changed-files, vllm, sglang, trtllm, operator]
     if: always()
     steps:
       - name: "Check all dependent jobs"

.github/workflows/container-validation-dynamo.yml

@@ -19,7 +19,7 @@ jobs:
   changed-files:
     runs-on: ubuntu-latest
     outputs:
-      core: ${{ steps.filter.outputs.core_any_changed }}
+      core: ${{ steps.filter.outputs.core_any_modified }}
     steps:
       - name: Checkout code
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
@@ -32,19 +32,39 @@ jobs:
           files_yaml_from_source_file: .github/filters.yaml
       - name: Debug changed files
         run: |
-          echo "=== All Changed Files (${{ steps.filter.outputs.all_all_changed_files_count }} total) ==="
-          echo "${{ steps.filter.outputs.all_all_changed_files }}"
+          echo "=== All Modified Files (${{ steps.filter.outputs.all_all_modified_files_count }} total) ==="
+          echo "${{ steps.filter.outputs.all_all_modified_files }}"
           echo ""
           echo "=== Filter Results ==="
-          echo "core_any_changed: ${{ steps.filter.outputs.core_any_changed }}"
-          echo "core files: ${{ steps.filter.outputs.core_all_changed_files }}"
+          echo "core_any_modified: ${{ steps.filter.outputs.core_any_modified }}"
+          echo "core files: ${{ steps.filter.outputs.core_all_modified_files }}"
+      - name: Check for uncovered files
+        run: |
+          # Combine all filter-specific files into one list
+          COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u)
+
+          # Get all modified files
+          ALL_FILES=$(echo "${{ steps.filter.outputs.all_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u)
+
+          # Find files in ALL but not in COVERED
+          UNCOVERED=$(comm -23 <(echo "$ALL_FILES") <(echo "$COVERED_FILES"))
+
+          if [ -n "$UNCOVERED" ]; then
+            echo "::error::The following files are not covered by any CI filter:"
+            echo "$UNCOVERED"
+            echo ""
+            echo "Please add these paths to .github/filters.yaml"
+            echo "See .github/FILTERS.md for documentation"
+            exit 1
+          fi
+          echo "All modified files are covered by CI filters."
 
   dynamo-status-check:
     runs-on: ubuntu-latest
-    needs: [build-test]
+    needs: [changed-files, build-test]
     if: always()
     steps:
-      - name: "Check build-test result"
+      - name: "Check all dependent jobs"
         run: |
           echo '${{ toJson(needs) }}' | jq -e 'to_entries | map(.value.result) | all(. as $result | ["success", "skipped"] | any($result == .))'
 

.gitignore

@@ -103,6 +103,8 @@ TensorRT-LLM
 # Ruler Generated Files
 /.cursor/instructions.md
 /.cursor/instructions.md.bak
+/.cursor
+/.claude
 /CLAUDE.md
 /CLAUDE.md.bak