extract warnings in GH workflows (#20487)

Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

.github/workflows/self-scheduled.yml

@@ -398,6 +398,56 @@ jobs:
           name: ${{ matrix.machine_type }}_run_tests_torch_cuda_extensions_gpu_test_reports
           path: /workspace/transformers/reports/${{ matrix.machine_type }}_tests_torch_cuda_extensions_gpu
 
+  run_extract_warnings:
+    name: Extract warnings in CI artifacts
+    runs-on: ubuntu-latest
+    if: always()
+    needs: [
+      check_runner_status,
+      check_runners,
+      setup,
+      run_tests_single_gpu,
+      run_tests_multi_gpu,
+      run_examples_gpu,
+      run_pipelines_tf_gpu,
+      run_pipelines_torch_gpu,
+      run_all_tests_torch_cuda_extensions_gpu
+    ]
+    steps:
+      - name: Checkout transformers
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - name: Install transformers
+        run: pip install transformers
+
+      - name: Show installed libraries and their versions
+        run: pip freeze
+
+      - name: Create output directory
+        run: mkdir warnings_in_ci
+
+      - uses: actions/download-artifact@v2
+        with:
+          path: warnings_in_ci
+
+      - name: Show artifacts
+        run: echo "$(python3 -c 'import os; d = os.listdir(); print(d)')"
+        working-directory: warnings_in_ci
+
+      - name: Extract warnings in CI artifacts
+        run: |
+          python3 utils/extract_warnings.py --workflow_run_id ${{ github.run_id }} --output_dir warnings_in_ci --token ${{ secrets.ACCESS_REPO_INFO_TOKEN }} --from_gh
+          echo "$(python3 -c 'import os; import json; fp = open("warnings_in_ci/selected_warnings.json"); d = json.load(fp); d = "\n".join(d) ;print(d)')"
+
+      - name: Upload artifact
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: warnings_in_ci
+          path: warnings_in_ci/selected_warnings.json
+
   send_results:
     name: Send results to webhook
     runs-on: ubuntu-latest
@@ -411,7 +461,8 @@ jobs:
       run_examples_gpu,
       run_pipelines_tf_gpu,
       run_pipelines_torch_gpu,
-      run_all_tests_torch_cuda_extensions_gpu
+      run_all_tests_torch_cuda_extensions_gpu,
+      run_extract_warnings
     ]
     steps:
       - name: Preliminary job status

utils/extract_warnings.py

@@ -11,20 +11,14 @@ from transformers import logging
 logger = logging.get_logger(__name__)
 
 
-def extract_warnings_from_single_artifact(artifact_zip_path, targets):
+def extract_warnings_from_single_artifact(artifact_path, targets):
     """Extract warnings from a downloaded artifact (in .zip format)"""
     selected_warnings = set()
     buffer = []
 
-    try:
-        with zipfile.ZipFile(artifact_zip_path) as z:
-            for filename in z.namelist():
-                if not os.path.isdir(filename):
-                    # read the file
-                    if filename != "warnings.txt":
-                        continue
-                    with z.open(filename) as f:
-                        for line in f:
+    def parse_line(fp):
+        for line in fp:
+            if isinstance(line, bytes):
                 line = line.decode("UTF-8")
             if "warnings summary (final)" in line:
                 continue
@@ -36,14 +30,34 @@ def extract_warnings_from_single_artifact(artifact_zip_path, targets):
                     # Only keep the warnings specified in `targets`
                     if any(f": {x}: " in warning for x in targets):
                         selected_warnings.add(warning)
-                                    buffer = []
+                    buffer.clear()
                 continue
             else:
                 line = line.strip()
                 buffer.append(line)
+
+    if from_gh:
+        for filename in os.listdir(artifact_path):
+            file_path = os.path.join(artifact_path, filename)
+            if not os.path.isdir(file_path):
+                # read the file
+                if filename != "warnings.txt":
+                    continue
+                with open(file_path) as fp:
+                    parse_line(fp)
+    else:
+        try:
+            with zipfile.ZipFile(artifact_path) as z:
+                for filename in z.namelist():
+                    if not os.path.isdir(filename):
+                        # read the file
+                        if filename != "warnings.txt":
+                            continue
+                        with z.open(filename) as fp:
+                            parse_line(fp)
         except Exception:
             logger.warning(
-            f"{artifact_zip_path} is either an invalid zip file or something else wrong. This file is skipped."
+                f"{artifact_path} is either an invalid zip file or something else wrong. This file is skipped."
             )
 
     return selected_warnings
@@ -54,7 +68,7 @@ def extract_warnings(artifact_dir, targets):
 
     selected_warnings = set()
 
-    paths = [os.path.join(artifact_dir, p) for p in os.listdir(artifact_dir) if p.endswith(".zip")]
+    paths = [os.path.join(artifact_dir, p) for p in os.listdir(artifact_dir) if (p.endswith(".zip") or from_gh)]
     for p in paths:
         selected_warnings.update(extract_warnings_from_single_artifact(p, targets))
 
@@ -81,15 +95,26 @@ if __name__ == "__main__":
     parser.add_argument(
         "--token", default=None, type=str, required=True, help="A token that has actions:read permission."
     )
+    # optional parameters
     parser.add_argument(
         "--targets",
         default="DeprecationWarning,UserWarning,FutureWarning",
         type=list_str,
         help="Comma-separated list of target warning(s) which we want to extract.",
     )
+    parser.add_argument(
+        "--from_gh",
+        action="store_true",
+        help="If running from a GitHub action workflow and collecting warnings from its artifacts.",
+    )
 
     args = parser.parse_args()
 
+    from_gh = args.from_gh
+    if from_gh:
+        # The artifacts have to be downloaded using `actions/download-artifact@v2`
+        pass
+    else:
         os.makedirs(args.output_dir, exist_ok=True)
 
         # get download links

utils/notification_service.py

@@ -98,7 +98,9 @@ def dicts_to_sum(objects: Union[Dict[str, Dict], List[dict]]):
 
 
 class Message:
-    def __init__(self, title: str, ci_title: str, model_results: Dict, additional_results: Dict):
+    def __init__(
+        self, title: str, ci_title: str, model_results: Dict, additional_results: Dict, selected_warnings: List = None
+    ):
         self.title = title
         self.ci_title = ci_title
 
@@ -136,6 +138,10 @@ class Message:
 
         self.thread_ts = None
 
+        if selected_warnings is None:
+            selected_warnings = []
+        self.selected_warnings = selected_warnings
+
     @property
     def time(self) -> str:
         all_results = [*self.model_results.values(), *self.additional_results.values()]
@@ -198,6 +204,22 @@ class Message:
             },
         }
 
+    @property
+    def warnings(self) -> Dict:
+        return {
+            "type": "section",
+            "text": {
+                "type": "plain_text",
+                "text": f"There were {len(self.selected_warnings)} warnings being selected.",
+                "emoji": True,
+            },
+            "accessory": {
+                "type": "button",
+                "text": {"type": "plain_text", "text": "Check warnings", "emoji": True},
+                "url": f"{github_actions_job_links['Extract warnings in CI artifacts']}",
+            },
+        }
+
     @staticmethod
     def get_device_report(report, rjust=6):
         if "single" in report and "multi" in report:
@@ -384,6 +406,9 @@ class Message:
         if self.n_model_failures == 0 and self.n_additional_failures == 0:
             blocks.append(self.no_failures)
 
+        if len(self.selected_warnings) > 0:
+            blocks.append(self.warnings)
+
         return json.dumps(blocks)
 
     @staticmethod
@@ -910,7 +935,13 @@ if __name__ == "__main__":
                             {"line": line, "trace": stacktraces.pop(0)}
                         )
 
-    message = Message(title, ci_title, model_results, additional_results)
+    selected_warnings = []
+    if "warnings_in_ci" in available_artifacts:
+        directory = available_artifacts["warnings_in_ci"].paths[0]["path"]
+        with open(os.path.join(directory, "selected_warnings.json")) as fp:
+            selected_warnings = json.load(fp)
+
+    message = Message(title, ci_title, model_results, additional_results, selected_warnings=selected_warnings)
 
     # send report only if there is any failure (for push CI)
     if message.n_failures or ci_event != "push":