fix: test uploading to not overwrite itself for pytest trtllm, sglang, and vllm jobs (#3859)

.github/actions/pytest/action.yml

@@ -73,6 +73,11 @@ runs:
       shell: bash
       run: |
 
+        # Sanitize test_type for filenames (always set this for artifact upload)
+        # Remove commas and spaces from test_type for use in filenames
+        STR_TEST_TYPE=$(echo "${{ inputs.test_type }}" | tr ', ' '_')
+        echo "STR_TEST_TYPE=${STR_TEST_TYPE}" >> $GITHUB_ENV
+
         # Check for JUnit XML file and determine test status
         JUNIT_FILE="test-results/pytest_test_report.xml"
 
@@ -84,17 +89,24 @@ runs:
           ERROR_TESTS=$(grep -o 'errors="[0-9]*"' "$JUNIT_FILE" | grep -o '[0-9]*' | head -1 || echo "0")
           echo "📊 ${TOTAL_TESTS} tests completed (${FAILED_TESTS} failed, ${ERROR_TESTS} errors)"
 
-          # Create metadata file with step context information
-          METADATA_FILE="test-results/test_metadata.json"
+          # Create uniquely named metadata file with step context information
+          # Use framework-testtype-arch to make it unique per test run
+          METADATA_FILE="test-results/test_metadata_${{ inputs.framework }}_${STR_TEST_TYPE}_${{ inputs.platform_arch }}.json"
+          JUNIT_NAME="pytest_test_report_${{ inputs.framework }}_${STR_TEST_TYPE}_${{ inputs.platform_arch }}.xml"
+
+          # Rename XML file to unique name
+          mv "$JUNIT_FILE" "test-results/$JUNIT_NAME"
+
           echo '{' > "$METADATA_FILE"
           echo '  "job_name": "${{ github.job }}",' >> "$METADATA_FILE"
           echo '  "framework": "${{ inputs.framework }}",' >> "$METADATA_FILE"
           echo '  "test_type": "${{ inputs.test_type }}",' >> "$METADATA_FILE"
           echo '  "platform_arch": "${{ inputs.platform_arch }}",' >> "$METADATA_FILE"
-          echo '  "junit_xml_file": "pytest_test_report.xml",' >> "$METADATA_FILE"
+          echo '  "junit_xml_file": "'"$JUNIT_NAME"'",' >> "$METADATA_FILE"
           echo '  "step_name": "Run ${{ inputs.test_type }} tests"' >> "$METADATA_FILE"
           echo '}' >> "$METADATA_FILE"
-          echo "📝 Created test metadata file"
+          echo "📝 Created test metadata file: $METADATA_FILE"
+          echo "📝 Renamed XML file to: $JUNIT_NAME"
         else
           echo "⚠️  JUnit XML file not found - test results may not be available for upload"
           TOTAL_TESTS=0
@@ -109,6 +121,8 @@ runs:
       uses: actions/upload-artifact@v4
       if: always()  # Always upload test results, even if tests failed
       with:
-        name: test-results-${{ inputs.framework }}-${{ inputs.test_type }}-${{ env.PLATFORM_ARCH }}
-        path: test-results/${{ env.PYTEST_XML_FILE }}
+        name: test-results-${{ inputs.framework }}-${{ env.STR_TEST_TYPE }}-${{ env.PLATFORM_ARCH }}
+        path: |
+          test-results/pytest_test_report_${{ inputs.framework }}_${{ env.STR_TEST_TYPE }}_${{ inputs.platform_arch }}.xml
+          test-results/test_metadata_${{ inputs.framework }}_${{ env.STR_TEST_TYPE }}_${{ inputs.platform_arch }}.json
         retention-days: 7
\ No newline at end of file

.github/workflows/upload_complete_workflow_metrics.py

@@ -825,8 +825,45 @@ class WorkflowMetricsUploader:
         job_name = job_data.get("name", "")
         job_id = str(job_data["id"])
 
+        # Skip deployment test jobs (No pytest metadata files are created)
+        if job_name.lower().startswith("deploy"):
+            print(f"⏭️  Skipping test metrics for deployment job '{job_name}'")
+            return
+
         print(f"🧪 Looking for test results for job '{job_name}'")
 
+        # Determine framework from job name to filter metadata files
+        framework = None
+        job_name_lower = job_name.lower()
+        if "vllm" in job_name_lower:
+            framework = "vllm"
+        elif "sglang" in job_name_lower:
+            framework = "sglang"
+        elif "trtllm" in job_name_lower:
+            framework = "trtllm"
+
+        if not framework:
+            print(f"⚠️  Could not determine framework from job name: {job_name}")
+            return
+
+        # Determine platform architecture from job name
+        # Job names typically look like: "vllm (amd64)" or "sglang (arm64)"
+        platform_arch = None
+        if "(amd64)" in job_name_lower or "amd64" in job_name_lower:
+            platform_arch = "amd64"
+        elif "(arm64)" in job_name_lower or "arm64" in job_name_lower:
+            platform_arch = "arm64"
+
+        if not platform_arch:
+            print(
+                f"⚠️  Could not determine platform architecture from job name: {job_name}"
+            )
+            # Default to amd64 if not specified
+            platform_arch = "amd64"
+            print(f"   Defaulting to platform_arch: {platform_arch}")
+
+        print(f"📦 Job framework: {framework}, platform_arch: {platform_arch}")
+
         # Look for test results directory
         test_results_dir = "test-results"
         if not os.path.exists(test_results_dir):
@@ -834,13 +871,21 @@ class WorkflowMetricsUploader:
             return
 
         # Look for metadata files to get accurate step and framework info
-        metadata_files = glob.glob(f"{test_results_dir}/test_metadata.json")
+        # Updated pattern to match new unique naming: test_metadata_<framework>_<test_type>_<arch>.json
+        # Filter by both framework AND architecture to only process this job's tests
+        metadata_files = glob.glob(
+            f"{test_results_dir}/test_metadata_{framework}_*_{platform_arch}.json"
+        )
 
         if not metadata_files:
-            print(f"⚠️  No test metadata files found in {test_results_dir}")
+            print(
+                f"⚠️  No test metadata files found for framework '{framework}' with arch '{platform_arch}' in {test_results_dir}"
+            )
             return
 
-        print(f"📄 Found {len(metadata_files)} test metadata files")
+        print(
+            f"📄 Found {len(metadata_files)} test metadata files for {framework} ({platform_arch})"
+        )
 
         total_tests_processed = 0