notification_service.py 49.2 KB
Newer Older
Lysandre Debut's avatar
Lysandre Debut committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
import ast
import collections
17
import datetime
18
19
20
import functools
import json
import operator
Lysandre Debut's avatar
Lysandre Debut committed
21
22
23
import os
import re
import sys
24
25
import time
from typing import Dict, List, Optional, Union
Lysandre Debut's avatar
Lysandre Debut committed
26

27
import requests
28
from get_ci_error_statistics import get_jobs
29
from get_previous_daily_ci import get_last_daily_ci_reports
30
from huggingface_hub import HfApi
Lysandre Debut's avatar
Lysandre Debut committed
31
32
33
from slack_sdk import WebClient


34
api = HfApi()
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
client = WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"])

NON_MODEL_TEST_MODULES = [
    "benchmark",
    "deepspeed",
    "extended",
    "fixtures",
    "generation",
    "onnx",
    "optimization",
    "pipelines",
    "sagemaker",
    "trainer",
    "utils",
]


Lysandre Debut's avatar
Lysandre Debut committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def handle_test_results(test_results):
    expressions = test_results.split(" ")

    failed = 0
    success = 0

    # When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
    # When it is too long, those signs are not present.
    time_spent = expressions[-2] if "=" in expressions[-1] else expressions[-1]

    for i, expression in enumerate(expressions):
        if "failed" in expression:
            failed += int(expressions[i - 1])
        if "passed" in expression:
            success += int(expressions[i - 1])

    return failed, success, time_spent


71
72
73
74
75
76
def handle_stacktraces(test_results):
    # These files should follow the following architecture:
    # === FAILURES ===
    # <path>:<line>: Error ...
    # <path>:<line>: Error ...
    # <empty line>
Lysandre Debut's avatar
Lysandre Debut committed
77

78
79
80
81
82
83
    total_stacktraces = test_results.split("\n")[1:-1]
    stacktraces = []
    for stacktrace in total_stacktraces:
        try:
            line = stacktrace[: stacktrace.index(" ")].split(":")[-2]
            error_message = stacktrace[stacktrace.index(" ") :]
Lysandre Debut's avatar
Lysandre Debut committed
84

85
86
87
            stacktraces.append(f"(line {line}) {error_message}")
        except Exception:
            stacktraces.append("Cannot retrieve error message.")
Lysandre Debut's avatar
Lysandre Debut committed
88

89
    return stacktraces
Lysandre Debut's avatar
Lysandre Debut committed
90

Lysandre Debut's avatar
Lysandre Debut committed
91

92
93
94
def dicts_to_sum(objects: Union[Dict[str, Dict], List[dict]]):
    if isinstance(objects, dict):
        lists = objects.values()
Lysandre Debut's avatar
Lysandre Debut committed
95
    else:
96
97
98
99
100
101
102
103
104
        lists = objects

    # Convert each dictionary to counter
    counters = map(collections.Counter, lists)
    # Sum all the counters
    return functools.reduce(operator.add, counters)


class Message:
105
    def __init__(
106
107
108
109
110
111
112
        self,
        title: str,
        ci_title: str,
        model_results: Dict,
        additional_results: Dict,
        selected_warnings: List = None,
        prev_ci_artifacts=None,
113
    ):
114
        self.title = title
115
        self.ci_title = ci_title
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

        # Failures and success of the modeling tests
        self.n_model_success = sum(r["success"] for r in model_results.values())
        self.n_model_single_gpu_failures = sum(dicts_to_sum(r["failed"])["single"] for r in model_results.values())
        self.n_model_multi_gpu_failures = sum(dicts_to_sum(r["failed"])["multi"] for r in model_results.values())

        # Some suites do not have a distinction between single and multi GPU.
        self.n_model_unknown_failures = sum(dicts_to_sum(r["failed"])["unclassified"] for r in model_results.values())
        self.n_model_failures = (
            self.n_model_single_gpu_failures + self.n_model_multi_gpu_failures + self.n_model_unknown_failures
        )

        # Failures and success of the additional tests
        self.n_additional_success = sum(r["success"] for r in additional_results.values())

131
132
133
134
135
136
137
138
139
140
141
        if len(additional_results) > 0:
            # `dicts_to_sum` uses `dicts_to_sum` which requires a non empty dictionary. Let's just add an empty entry.
            all_additional_failures = dicts_to_sum([r["failed"] for r in additional_results.values()])
            self.n_additional_single_gpu_failures = all_additional_failures["single"]
            self.n_additional_multi_gpu_failures = all_additional_failures["multi"]
            self.n_additional_unknown_gpu_failures = all_additional_failures["unclassified"]
        else:
            self.n_additional_single_gpu_failures = 0
            self.n_additional_multi_gpu_failures = 0
            self.n_additional_unknown_gpu_failures = 0

142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        self.n_additional_failures = (
            self.n_additional_single_gpu_failures
            + self.n_additional_multi_gpu_failures
            + self.n_additional_unknown_gpu_failures
        )

        # Results
        self.n_failures = self.n_model_failures + self.n_additional_failures
        self.n_success = self.n_model_success + self.n_additional_success
        self.n_tests = self.n_failures + self.n_success

        self.model_results = model_results
        self.additional_results = additional_results

        self.thread_ts = None

158
159
160
161
        if selected_warnings is None:
            selected_warnings = []
        self.selected_warnings = selected_warnings

162
163
        self.prev_ci_artifacts = prev_ci_artifacts

164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
    @property
    def time(self) -> str:
        all_results = [*self.model_results.values(), *self.additional_results.values()]
        time_spent = [r["time_spent"].split(", ")[0] for r in all_results if len(r["time_spent"])]
        total_secs = 0

        for time in time_spent:
            time_parts = time.split(":")

            # Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
            if len(time_parts) == 1:
                time_parts = [0, 0, time_parts[0]]

            hours, minutes, seconds = int(time_parts[0]), int(time_parts[1]), float(time_parts[2])
            total_secs += hours * 3600 + minutes * 60 + seconds

        hours, minutes, seconds = total_secs // 3600, (total_secs % 3600) // 60, total_secs % 60
        return f"{int(hours)}h{int(minutes)}m{int(seconds)}s"

    @property
    def header(self) -> Dict:
        return {"type": "header", "text": {"type": "plain_text", "text": self.title}}

187
188
189
190
    @property
    def ci_title_section(self) -> Dict:
        return {"type": "section", "text": {"type": "mrkdwn", "text": self.ci_title}}

191
192
193
194
195
196
197
198
    @property
    def no_failures(self) -> Dict:
        return {
            "type": "section",
            "text": {
                "type": "plain_text",
                "text": f"🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.",
                "emoji": True,
Lysandre Debut's avatar
Lysandre Debut committed
199
            },
200
201
202
203
            "accessory": {
                "type": "button",
                "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}",
Lysandre Debut's avatar
Lysandre Debut committed
204
            },
205
206
207
208
209
210
211
212
        }

    @property
    def failures(self) -> Dict:
        return {
            "type": "section",
            "text": {
                "type": "plain_text",
Sylvain Gugger's avatar
Sylvain Gugger committed
213
                "text": (
214
215
216
                    f"There were {self.n_failures} failures, out of {self.n_tests} tests.\n"
                    f"Number of model failures: {self.n_model_failures}.\n"
                    f"The suite ran in {self.time}."
Sylvain Gugger's avatar
Sylvain Gugger committed
217
                ),
218
                "emoji": True,
Lysandre Debut's avatar
Lysandre Debut committed
219
            },
220
221
222
223
            "accessory": {
                "type": "button",
                "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}",
Lysandre Debut's avatar
Lysandre Debut committed
224
            },
Lysandre Debut's avatar
Lysandre Debut committed
225
        }
226

227
228
    @property
    def warnings(self) -> Dict:
229
230
231
232
        # If something goes wrong, let's avoid the CI report failing to be sent.
        button_text = "Check warnings (Link not found)"
        # Use the workflow run link
        job_link = f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}"
Yih-Dar's avatar
Yih-Dar committed
233
234
235
236
237
238
239

        for job in github_actions_jobs:
            if "Extract warnings in CI artifacts" in job["name"] and job["conclusion"] == "success":
                button_text = "Check warnings"
                # Use the actual job link
                job_link = job["html_url"]
                break
240

241
242
243
244
        huggingface_hub_warnings = [x for x in self.selected_warnings if "huggingface_hub" in x]
        text = f"There are {len(self.selected_warnings)} warnings being selected."
        text += f"\n{len(huggingface_hub_warnings)} of them are from `huggingface_hub`."

245
246
247
248
        return {
            "type": "section",
            "text": {
                "type": "plain_text",
249
                "text": text,
250
251
252
253
                "emoji": True,
            },
            "accessory": {
                "type": "button",
254
255
                "text": {"type": "plain_text", "text": button_text, "emoji": True},
                "url": job_link,
256
257
258
            },
        }

259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
    @staticmethod
    def get_device_report(report, rjust=6):
        if "single" in report and "multi" in report:
            return f"{str(report['single']).rjust(rjust)} | {str(report['multi']).rjust(rjust)} | "
        elif "single" in report:
            return f"{str(report['single']).rjust(rjust)} | {'0'.rjust(rjust)} | "
        elif "multi" in report:
            return f"{'0'.rjust(rjust)} | {str(report['multi']).rjust(rjust)} | "

    @property
    def category_failures(self) -> Dict:
        model_failures = [v["failed"] for v in self.model_results.values()]

        category_failures = {}

        for model_failure in model_failures:
            for key, value in model_failure.items():
                if key not in category_failures:
                    category_failures[key] = dict(value)
                else:
                    category_failures[key]["unclassified"] += value["unclassified"]
                    category_failures[key]["single"] += value["single"]
                    category_failures[key]["multi"] += value["multi"]

        individual_reports = []
        for key, value in category_failures.items():
            device_report = self.get_device_report(value)

            if sum(value.values()):
                if device_report:
                    individual_reports.append(f"{device_report}{key}")
                else:
                    individual_reports.append(key)

        header = "Single |  Multi | Category\n"
Yih-Dar's avatar
Yih-Dar committed
294
295
296
        category_failures_report = prepare_reports(
            title="The following modeling categories had failures", header=header, reports=individual_reports
        )
297

Yih-Dar's avatar
Yih-Dar committed
298
        return {"type": "section", "text": {"type": "mrkdwn", "text": category_failures_report}}
299

300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    def compute_diff_for_failure_reports(self, curr_failure_report, prev_failure_report):  # noqa
        # Remove the leading and training parts that don't contain failure count information.
        model_failures = curr_failure_report.split("\n")[3:-2]
        prev_model_failures = prev_failure_report.split("\n")[3:-2]
        entries_changed = set(model_failures).difference(prev_model_failures)

        prev_map = {}
        for f in prev_model_failures:
            items = [x.strip() for x in f.split("| ")]
            prev_map[items[-1]] = [int(x) for x in items[:-1]]

        curr_map = {}
        for f in entries_changed:
            items = [x.strip() for x in f.split("| ")]
            curr_map[items[-1]] = [int(x) for x in items[:-1]]

        diff_map = {}
        for k, v in curr_map.items():
            if k not in prev_map:
                diff_map[k] = v
            else:
                diff = [x - y for x, y in zip(v, prev_map[k])]
                if max(diff) > 0:
                    diff_map[k] = diff

        entries_changed = []
        for model_name, diff_values in diff_map.items():
            diff = [str(x) for x in diff_values]
            diff = [f"+{x}" if (x != "0" and not x.startswith("-")) else x for x in diff]
            diff = [x.rjust(9) for x in diff]
            device_report = " | ".join(diff) + " | "
            report = f"{device_report}{model_name}"
            entries_changed.append(report)
        entries_changed = sorted(entries_changed, key=lambda s: s.split("| ")[-1])

        return entries_changed

337
    @property
338
    def model_failures(self) -> List[Dict]:
339
340
341
342
        # Obtain per-model failures
        def per_model_sum(model_category_dict):
            return dicts_to_sum(model_category_dict["failed"].values())

343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
        failures = {}
        non_model_failures = {
            k: per_model_sum(v) for k, v in self.model_results.items() if sum(per_model_sum(v).values())
        }

        for k, v in self.model_results.items():
            if k in NON_MODEL_TEST_MODULES:
                pass

            if sum(per_model_sum(v).values()):
                dict_failed = dict(v["failed"])
                pytorch_specific_failures = dict_failed.pop("PyTorch")
                tensorflow_specific_failures = dict_failed.pop("TensorFlow")
                other_failures = dicts_to_sum(dict_failed.values())

                failures[k] = {
                    "PyTorch": pytorch_specific_failures,
                    "TensorFlow": tensorflow_specific_failures,
                    "other": other_failures,
                }
363
364
365
366

        model_reports = []
        other_module_reports = []

367
368
369
370
371
372
373
374
375
        for key, value in non_model_failures.items():
            if key in NON_MODEL_TEST_MODULES:
                device_report = self.get_device_report(value)

                if sum(value.values()):
                    if device_report:
                        report = f"{device_report}{key}"
                    else:
                        report = key
376
377
378

                    other_module_reports.append(report)

379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
        for key, value in failures.items():
            device_report_values = [
                value["PyTorch"]["single"],
                value["PyTorch"]["multi"],
                value["TensorFlow"]["single"],
                value["TensorFlow"]["multi"],
                sum(value["other"].values()),
            ]

            if sum(device_report_values):
                device_report = " | ".join([str(x).rjust(9) for x in device_report_values]) + " | "
                report = f"{device_report}{key}"

                model_reports.append(report)

394
        # (Possibly truncated) reports for the current workflow run - to be sent to Slack channels
395
        model_header = "Single PT |  Multi PT | Single TF |  Multi TF |     Other | Category\n"
396
        sorted_model_reports = sorted(model_reports, key=lambda s: s.split("| ")[-1])
Yih-Dar's avatar
Yih-Dar committed
397
398
399
        model_failures_report = prepare_reports(
            title="These following model modules had failures", header=model_header, reports=sorted_model_reports
        )
400
401

        module_header = "Single |  Multi | Category\n"
402
        sorted_module_reports = sorted(other_module_reports, key=lambda s: s.split("| ")[-1])
Yih-Dar's avatar
Yih-Dar committed
403
404
405
        module_failures_report = prepare_reports(
            title="The following non-model modules had failures", header=module_header, reports=sorted_module_reports
        )
406

407
        # To be sent to Slack channels
Yih-Dar's avatar
Yih-Dar committed
408
409
410
411
        model_failure_sections = [
            {"type": "section", "text": {"type": "mrkdwn", "text": model_failures_report}},
            {"type": "section", "text": {"type": "mrkdwn", "text": module_failures_report}},
        ]
412

413
414
415
416
417
418
419
420
421
        # Save the complete (i.e. no truncation) failure tables (of the current workflow run)
        # (to be uploaded as artifacts)

        model_failures_report = prepare_reports(
            title="These following model modules had failures",
            header=model_header,
            reports=sorted_model_reports,
            to_truncate=False,
        )
422
        file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/model_failures_report.txt")
423
424
425
426
427
428
429
430
431
        with open(file_path, "w", encoding="UTF-8") as fp:
            fp.write(model_failures_report)

        module_failures_report = prepare_reports(
            title="The following non-model modules had failures",
            header=module_header,
            reports=sorted_module_reports,
            to_truncate=False,
        )
432
        file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/module_failures_report.txt")
433
434
435
        with open(file_path, "w", encoding="UTF-8") as fp:
            fp.write(module_failures_report)

436
        if self.prev_ci_artifacts is not None:
437
            # if the last run produces artifact named `ci_results_{job_name}`
438
            if (
439
440
                f"ci_results_{job_name}" in self.prev_ci_artifacts
                and "model_failures_report.txt" in self.prev_ci_artifacts[f"ci_results_{job_name}"]
441
            ):
442
                # Compute the difference of the previous/current (model failure) table
443
                prev_model_failures = self.prev_ci_artifacts[f"ci_results_{job_name}"]["model_failures_report.txt"]
444
445
446
447
448
449
450
451
452
                entries_changed = self.compute_diff_for_failure_reports(model_failures_report, prev_model_failures)
                if len(entries_changed) > 0:
                    # Save the complete difference
                    diff_report = prepare_reports(
                        title="Changed model modules failures",
                        header=model_header,
                        reports=entries_changed,
                        to_truncate=False,
                    )
453
                    file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/changed_model_failures_report.txt")
454
455
456
457
458
459
460
461
462
463
464
465
                    with open(file_path, "w", encoding="UTF-8") as fp:
                        fp.write(diff_report)

                    # To be sent to Slack channels
                    diff_report = prepare_reports(
                        title="*Changed model modules failures*",
                        header=model_header,
                        reports=entries_changed,
                    )
                    model_failure_sections.append(
                        {"type": "section", "text": {"type": "mrkdwn", "text": diff_report}},
                    )
466

Yih-Dar's avatar
Yih-Dar committed
467
        return model_failure_sections
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487

    @property
    def additional_failures(self) -> Dict:
        failures = {k: v["failed"] for k, v in self.additional_results.items()}
        errors = {k: v["error"] for k, v in self.additional_results.items()}

        individual_reports = []
        for key, value in failures.items():
            device_report = self.get_device_report(value)

            if sum(value.values()) or errors[key]:
                report = f"{key}"
                if errors[key]:
                    report = f"[Errored out] {report}"
                if device_report:
                    report = f"{device_report}{report}"

                individual_reports.append(report)

        header = "Single |  Multi | Category\n"
Yih-Dar's avatar
Yih-Dar committed
488
489
490
        failures_report = prepare_reports(
            title="The following non-modeling tests had failures", header=header, reports=individual_reports
        )
491

Yih-Dar's avatar
Yih-Dar committed
492
        return {"type": "section", "text": {"type": "mrkdwn", "text": failures_report}}
Lysandre Debut's avatar
Lysandre Debut committed
493

494
495
496
497
    @property
    def payload(self) -> str:
        blocks = [self.header]

498
499
500
        if self.ci_title:
            blocks.append(self.ci_title_section)

501
502
503
504
        if self.n_model_failures > 0 or self.n_additional_failures > 0:
            blocks.append(self.failures)

        if self.n_model_failures > 0:
Yih-Dar's avatar
Yih-Dar committed
505
506
507
508
            blocks.append(self.category_failures)
            for block in self.model_failures:
                if block["text"]["text"]:
                    blocks.append(block)
509
510
511
512
513
514
515

        if self.n_additional_failures > 0:
            blocks.append(self.additional_failures)

        if self.n_model_failures == 0 and self.n_additional_failures == 0:
            blocks.append(self.no_failures)

516
517
518
        if len(self.selected_warnings) > 0:
            blocks.append(self.warnings)

519
520
521
522
        new_failure_blocks = self.get_new_model_failure_blocks(with_header=False)
        if len(new_failure_blocks) > 0:
            blocks.extend(new_failure_blocks)

523
524
525
        return json.dumps(blocks)

    @staticmethod
526
    def error_out(title, ci_title="", runner_not_available=False, runner_failed=False, setup_failed=False):
527
528
529
530
531
532
533
534
        blocks = []
        title_block = {"type": "header", "text": {"type": "plain_text", "text": title}}
        blocks.append(title_block)

        if ci_title:
            ci_title_block = {"type": "section", "text": {"type": "mrkdwn", "text": ci_title}}
            blocks.append(ci_title_block)

535
        offline_runners = []
536
537
        if runner_not_available:
            text = "💔 CI runners are not available! Tests are not run. 😭"
538
539
540
            result = os.environ.get("OFFLINE_RUNNERS")
            if result is not None:
                offline_runners = json.loads(result)
541
542
        elif runner_failed:
            text = "💔 CI runners have problems! Tests are not run. 😭"
543
544
        elif setup_failed:
            text = "💔 Setup job failed. Tests are not run. 😭"
545
546
547
548
549
550
551
552
553
554
        else:
            text = "💔 There was an issue running the tests. 😭"

        error_block_1 = {
            "type": "header",
            "text": {
                "type": "plain_text",
                "text": text,
            },
        }
555
556
557
558
559
560
561

        text = ""
        if len(offline_runners) > 0:
            text = "\n  • " + "\n  • ".join(offline_runners)
            text = f"The following runners are offline:\n{text}\n\n"
        text += "🙏 Let's fix it ASAP! 🙏"

562
563
564
565
        error_block_2 = {
            "type": "section",
            "text": {
                "type": "plain_text",
566
                "text": text,
567
568
569
570
571
572
573
574
575
576
            },
            "accessory": {
                "type": "button",
                "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}",
            },
        }
        blocks.extend([error_block_1, error_block_2])

        payload = json.dumps(blocks)
577
578

        print("Sending the following payload")
579
        print(json.dumps({"blocks": blocks}))
580
581

        client.chat_postMessage(
Yih-Dar's avatar
Yih-Dar committed
582
            channel=SLACK_REPORT_CHANNEL_ID,
583
            text=text,
584
585
586
587
            blocks=payload,
        )

    def post(self):
588
        payload = self.payload
589
        print("Sending the following payload")
590
        print(json.dumps({"blocks": json.loads(payload)}))
591
592
593
594

        text = f"{self.n_failures} failures out of {self.n_tests} tests," if self.n_failures else "All tests passed."

        self.thread_ts = client.chat_postMessage(
Yih-Dar's avatar
Yih-Dar committed
595
            channel=SLACK_REPORT_CHANNEL_ID,
596
            blocks=payload,
597
598
599
600
            text=text,
        )

    def get_reply_blocks(self, job_name, job_result, failures, device, text):
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
        """
        failures: A list with elements of the form {"line": full test name, "trace": error trace}
        """
        # `text` must be less than 3001 characters in Slack SDK
        # keep some room for adding "[Truncated]" when necessary
        MAX_ERROR_TEXT = 3000 - len("[Truncated]")

        failure_text = ""
        for idx, error in enumerate(failures):
            new_text = failure_text + f'*{error["line"]}*\n_{error["trace"]}_\n\n'
            if len(new_text) > MAX_ERROR_TEXT:
                # `failure_text` here has length <= 3000
                failure_text = failure_text + "[Truncated]"
                break
            # `failure_text` here has length <= MAX_ERROR_TEXT
            failure_text = new_text
617
618
619
620
621
622
623

        title = job_name
        if device is not None:
            title += f" ({device}-gpu)"

        content = {"type": "section", "text": {"type": "mrkdwn", "text": text}}

624
625
626
627
628
629
        # TODO: Make sure we always have a valid job link (or at least a way not to break the report sending)
        # Currently we get the device from a job's artifact name.
        # If a device is found, the job name should contain the device type, for example, `XXX (single-gpu)`.
        # This could be done by adding `machine_type` in a job's `strategy`.
        # (If `job_result["job_link"][device]` is `None`, we get an error: `... [ERROR] must provide a string ...`)
        if job_result["job_link"] is not None and job_result["job_link"][device] is not None:
630
631
632
            content["accessory"] = {
                "type": "button",
                "text": {"type": "plain_text", "text": "GitHub Action job", "emoji": True},
633
                "url": job_result["job_link"][device],
634
635
636
637
638
            }

        return [
            {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
            content,
639
            {"type": "section", "text": {"type": "mrkdwn", "text": failure_text}},
640
641
        ]

642
    def get_new_model_failure_blocks(self, with_header=True, to_truncate=True):
643
        if self.prev_ci_artifacts is None:
Yih-Dar's avatar
Yih-Dar committed
644
            return []
645

646
647
648
        sorted_dict = sorted(self.model_results.items(), key=lambda t: t[0])

        prev_model_results = {}
649
650
651
652
653
        if (
            f"ci_results_{job_name}" in self.prev_ci_artifacts
            and "model_results.json" in self.prev_ci_artifacts[f"ci_results_{job_name}"]
        ):
            prev_model_results = json.loads(self.prev_ci_artifacts[f"ci_results_{job_name}"]["model_results.json"])
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680

        all_failure_lines = {}
        for job, job_result in sorted_dict:
            if len(job_result["failures"]):
                devices = sorted(job_result["failures"].keys(), reverse=True)
                for device in devices:
                    failures = job_result["failures"][device]
                    prev_error_lines = {}
                    if job in prev_model_results and device in prev_model_results[job]["failures"]:
                        prev_error_lines = {error["line"] for error in prev_model_results[job]["failures"][device]}

                    url = None
                    if job_result["job_link"] is not None and job_result["job_link"][device] is not None:
                        url = job_result["job_link"][device]

                    for idx, error in enumerate(failures):
                        if error["line"] in prev_error_lines:
                            continue

                        new_text = f'{error["line"]}\n\n'

                        if new_text not in all_failure_lines:
                            all_failure_lines[new_text] = []

                        all_failure_lines[new_text].append(f"<{url}|{device}>" if url is not None else device)

        MAX_ERROR_TEXT = 3000 - len("[Truncated]") - len("```New model failures```\n\n")
681
682
        if not to_truncate:
            MAX_ERROR_TEXT = float("inf")
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
        failure_text = ""
        for line, devices in all_failure_lines.items():
            new_text = failure_text + f"{'|'.join(devices)} gpu\n{line}"
            if len(new_text) > MAX_ERROR_TEXT:
                # `failure_text` here has length <= 3000
                failure_text = failure_text + "[Truncated]"
                break
            # `failure_text` here has length <= MAX_ERROR_TEXT
            failure_text = new_text

        blocks = []
        if failure_text:
            if with_header:
                blocks.append(
                    {"type": "header", "text": {"type": "plain_text", "text": "New model failures", "emoji": True}}
                )
            else:
                failure_text = f"*New model failures*\n\n{failure_text}"
            blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": failure_text}})

        return blocks

705
706
707
708
709
710
711
712
713
714
715
    def post_reply(self):
        if self.thread_ts is None:
            raise ValueError("Can only post reply if a post has been made.")

        sorted_dict = sorted(self.model_results.items(), key=lambda t: t[0])
        for job, job_result in sorted_dict:
            if len(job_result["failures"]):
                for device, failures in job_result["failures"].items():
                    text = "\n".join(
                        sorted([f"*{k}*: {v[device]}" for k, v in job_result["failed"].items() if v[device]])
                    )
Lysandre Debut's avatar
Lysandre Debut committed
716

717
                    blocks = self.get_reply_blocks(job, job_result, failures, device, text=text)
Lysandre Debut's avatar
Lysandre Debut committed
718

719
720
                    print("Sending the following reply")
                    print(json.dumps({"blocks": blocks}))
Lysandre Debut's avatar
Lysandre Debut committed
721

722
                    client.chat_postMessage(
Yih-Dar's avatar
Yih-Dar committed
723
                        channel=SLACK_REPORT_CHANNEL_ID,
724
725
726
727
                        text=f"Results for {job}",
                        blocks=blocks,
                        thread_ts=self.thread_ts["ts"],
                    )
Lysandre Debut's avatar
Lysandre Debut committed
728

729
730
731
                    time.sleep(1)

        for job, job_result in self.additional_results.items():
Lysandre Debut's avatar
Lysandre Debut committed
732
            if len(job_result["failures"]):
733
734
735
736
737
738
                for device, failures in job_result["failures"].items():
                    blocks = self.get_reply_blocks(
                        job,
                        job_result,
                        failures,
                        device,
739
                        text=f'Number of failures: {job_result["failed"][device]}',
740
741
742
743
744
745
                    )

                    print("Sending the following reply")
                    print(json.dumps({"blocks": blocks}))

                    client.chat_postMessage(
Yih-Dar's avatar
Yih-Dar committed
746
                        channel=SLACK_REPORT_CHANNEL_ID,
747
748
749
750
751
752
753
                        text=f"Results for {job}",
                        blocks=blocks,
                        thread_ts=self.thread_ts["ts"],
                    )

                    time.sleep(1)

754
755
756
757
758
759
        blocks = self.get_new_model_failure_blocks()
        if blocks:
            print("Sending the following reply")
            print(json.dumps({"blocks": blocks}))

            client.chat_postMessage(
Yih-Dar's avatar
Yih-Dar committed
760
                channel=SLACK_REPORT_CHANNEL_ID,
761
762
763
764
765
766
767
                text="Results for new failures",
                blocks=blocks,
                thread_ts=self.thread_ts["ts"],
            )

            time.sleep(1)

768
769
        # To save the list of new model failures
        blocks = self.get_new_model_failure_blocks(to_truncate=False)
Yih-Dar's avatar
Yih-Dar committed
770
771
772
773
774
        if blocks:
            failure_text = blocks[-1]["text"]["text"]
            file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/new_model_failures.txt")
            with open(file_path, "w", encoding="UTF-8") as fp:
                fp.write(failure_text)
775

776

777
def retrieve_artifact(artifact_path: str, gpu: Optional[str]):
778
779
780
781
782
    if gpu not in [None, "single", "multi"]:
        raise ValueError(f"Invalid GPU for artifact. Passed GPU: `{gpu}`.")

    _artifact = {}

783
784
    if os.path.exists(artifact_path):
        files = os.listdir(artifact_path)
785
786
        for file in files:
            try:
787
                with open(os.path.join(artifact_path, file)) as f:
788
789
                    _artifact[file.split(".")[0]] = f.read()
            except UnicodeDecodeError as e:
790
                raise ValueError(f"Could not open {os.path.join(artifact_path, file)}.") from e
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812

    return _artifact


def retrieve_available_artifacts():
    class Artifact:
        def __init__(self, name: str, single_gpu: bool = False, multi_gpu: bool = False):
            self.name = name
            self.single_gpu = single_gpu
            self.multi_gpu = multi_gpu
            self.paths = []

        def __str__(self):
            return self.name

        def add_path(self, path: str, gpu: str = None):
            self.paths.append({"name": self.name, "path": path, "gpu": gpu})

    _available_artifacts: Dict[str, Artifact] = {}

    directories = filter(os.path.isdir, os.listdir())
    for directory in directories:
813
814
815
816
817
818
819
820
        artifact_name = directory

        name_parts = artifact_name.split("_postfix_")
        if len(name_parts) > 1:
            artifact_name = name_parts[0]

        if artifact_name.startswith("single-gpu"):
            artifact_name = artifact_name[len("single-gpu") + 1 :]
821
822
823
824
825
826
827
828

            if artifact_name in _available_artifacts:
                _available_artifacts[artifact_name].single_gpu = True
            else:
                _available_artifacts[artifact_name] = Artifact(artifact_name, single_gpu=True)

            _available_artifacts[artifact_name].add_path(directory, gpu="single")

829
        elif artifact_name.startswith("multi-gpu"):
830
            artifact_name = artifact_name[len("multi-gpu") + 1 :]
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846

            if artifact_name in _available_artifacts:
                _available_artifacts[artifact_name].multi_gpu = True
            else:
                _available_artifacts[artifact_name] = Artifact(artifact_name, multi_gpu=True)

            _available_artifacts[artifact_name].add_path(directory, gpu="multi")
        else:
            if artifact_name not in _available_artifacts:
                _available_artifacts[artifact_name] = Artifact(artifact_name)

            _available_artifacts[artifact_name].add_path(directory)

    return _available_artifacts


Yih-Dar's avatar
Yih-Dar committed
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
def prepare_reports(title, header, reports, to_truncate=True):
    report = ""

    MAX_ERROR_TEXT = 3000 - len("[Truncated]")
    if not to_truncate:
        MAX_ERROR_TEXT = float("inf")

    if len(reports) > 0:
        # `text` must be less than 3001 characters in Slack SDK
        # keep some room for adding "[Truncated]" when necessary

        for idx in range(len(reports)):
            _report = header + "\n".join(reports[: idx + 1])
            new_report = f"{title}:\n```\n{_report}\n```\n"
            if len(new_report) > MAX_ERROR_TEXT:
                # `report` here has length <= 3000
                report = report + "[Truncated]"
                break
            report = new_report

    return report


870
if __name__ == "__main__":
Yih-Dar's avatar
Yih-Dar committed
871
872
    SLACK_REPORT_CHANNEL_ID = os.environ["SLACK_REPORT_CHANNEL"]

Yih-Dar's avatar
Yih-Dar committed
873
874
    # runner_status = os.environ.get("RUNNER_STATUS")
    # runner_env_status = os.environ.get("RUNNER_ENV_STATUS")
875
876
    setup_status = os.environ.get("SETUP_STATUS")

Yih-Dar's avatar
Yih-Dar committed
877
878
879
880
881
    # runner_not_available = True if runner_status is not None and runner_status != "success" else False
    # runner_failed = True if runner_env_status is not None and runner_env_status != "success" else False
    # Let's keep the lines regardig runners' status (we might be able to use them again in the future)
    runner_not_available = False
    runner_failed = False
Yih-Dar's avatar
Yih-Dar committed
882
883
    # Some jobs don't depend (`needs`) on the job `setup`: in this case, the status of the job `setup` is `skipped`.
    setup_failed = False if setup_status in ["skipped", "success"] else True
884

885
886
887
888
    org = "huggingface"
    repo = "transformers"
    repository_full_name = f"{org}/{repo}"

Yih-Dar's avatar
Yih-Dar committed
889
890
891
    # This env. variable is set in workflow file (under the job `send_results`).
    ci_event = os.environ["CI_EVENT"]

892
893
894
    # To find the PR number in a commit title, for example, `Add AwesomeFormer model (#99999)`
    pr_number_re = re.compile(r"\(#(\d+)\)$")

Yih-Dar's avatar
Yih-Dar committed
895
    title = f"🤗 Results of {ci_event} - {os.getenv('CI_TEST_JOB')}."
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
    # Add Commit/PR title with a link for push CI
    # (check the title in 2 env. variables - depending on the CI is triggered via `push` or `workflow_run` event)
    ci_title_push = os.environ.get("CI_TITLE_PUSH")
    ci_title_workflow_run = os.environ.get("CI_TITLE_WORKFLOW_RUN")
    ci_title = ci_title_push if ci_title_push else ci_title_workflow_run

    ci_sha = os.environ.get("CI_SHA")

    ci_url = None
    if ci_sha:
        ci_url = f"https://github.com/{repository_full_name}/commit/{ci_sha}"

    if ci_title is not None:
        if ci_url is None:
            raise ValueError(
                "When a title is found (`ci_title`), it means a `push` event or a `workflow_run` even (triggered by "
                "another `push` event), and the commit SHA has to be provided in order to create the URL to the "
                "commit page."
            )
        ci_title = ci_title.strip().split("\n")[0].strip()

        # Retrieve the PR title and author login to complete the report
        commit_number = ci_url.split("/")[-1]
        ci_detail_url = f"https://api.github.com/repos/{repository_full_name}/commits/{commit_number}"
        ci_details = requests.get(ci_detail_url).json()
        ci_author = ci_details["author"]["login"]

        merged_by = None
        # Find the PR number (if any) and change the url to the actual PR page.
        numbers = pr_number_re.findall(ci_title)
        if len(numbers) > 0:
            pr_number = numbers[0]
            ci_detail_url = f"https://api.github.com/repos/{repository_full_name}/pulls/{pr_number}"
            ci_details = requests.get(ci_detail_url).json()

            ci_author = ci_details["user"]["login"]
            ci_url = f"https://github.com/{repository_full_name}/pull/{pr_number}"

            merged_by = ci_details["merged_by"]["login"]

        if merged_by is None:
            ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author}"
        else:
            ci_title = f"<{ci_url}|{ci_title}>\nAuthor: {ci_author} | Merged by: {merged_by}"

941
942
943
    elif ci_sha:
        ci_title = f"<{ci_url}|commit: {ci_sha}>"

944
945
946
    else:
        ci_title = ""

947
948
    if runner_not_available or runner_failed or setup_failed:
        Message.error_out(title, ci_title, runner_not_available, runner_failed, setup_failed)
949
950
        exit(0)

Yih-Dar's avatar
Yih-Dar committed
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
    # sys.argv[0] is always `utils/notification_service.py`.
    arguments = sys.argv[1:]
    # In our usage in `.github/workflows/slack-report.yml`, we always pass an argument when calling this script.
    # The argument could be an empty string `""` if a job doesn't depend on the job `setup`.
    if arguments[0] == "":
        models = []
    else:
        model_list_as_str = arguments[0]
        try:
            folder_slices = ast.literal_eval(model_list_as_str)
            # Need to change from elements like `models/bert` to `models_bert` (the ones used as artifact names).
            models = [x.replace("models/", "models_") for folders in folder_slices for x in folders]
        except Exception:
            Message.error_out(title, ci_title)
            raise ValueError("Errored out.")
966

967
    github_actions_jobs = get_jobs(
968
969
        workflow_run_id=os.environ["GITHUB_RUN_ID"], token=os.environ["ACCESS_REPO_INFO_TOKEN"]
    )
970
971
972
973
974
975
976
977
978
979
    github_actions_job_links = {job["name"]: job["html_url"] for job in github_actions_jobs}

    artifact_name_to_job_map = {}
    for job in github_actions_jobs:
        for step in job["steps"]:
            if step["name"].startswith("Test suite reports artifacts: "):
                artifact_name = step["name"][len("Test suite reports artifacts: ") :]
                artifact_name_to_job_map[artifact_name] = job
                break

980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
    available_artifacts = retrieve_available_artifacts()

    modeling_categories = [
        "PyTorch",
        "TensorFlow",
        "Flax",
        "Tokenizers",
        "Pipelines",
        "Trainer",
        "ONNX",
        "Auto",
        "Unclassified",
    ]

    # This dict will contain all the information relative to each model:
    # - Failures: the total, as well as the number of failures per-category defined above
    # - Success: total
    # - Time spent: as a comma-separated list of elapsed time
    # - Failures: as a line-break separated list of errors
    model_results = {
        model: {
            "failed": {m: {"unclassified": 0, "single": 0, "multi": 0} for m in modeling_categories},
            "success": 0,
            "time_spent": "",
            "failures": {},
1005
            "job_link": {},
1006
1007
        }
        for model in models
1008
        if f"run_models_gpu_{model}_test_reports" in available_artifacts
1009
1010
1011
1012
1013
    }

    unclassified_model_failures = []

    for model in model_results.keys():
1014
        for artifact_path in available_artifacts[f"run_models_gpu_{model}_test_reports"].paths:
1015
            artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"])
1016
1017
            if "stats" in artifact:
                # Link to the GitHub Action job
1018
1019
                job = artifact_name_to_job_map[artifact_path["path"]]
                model_results[model]["job_link"][artifact_path["gpu"]] = job["html_url"]
1020
1021
1022
1023
1024
1025
1026
                failed, success, time_spent = handle_test_results(artifact["stats"])
                model_results[model]["success"] += success
                model_results[model]["time_spent"] += time_spent[1:-1] + ", "

                stacktraces = handle_stacktraces(artifact["failures_line"])

                for line in artifact["summary_short"].split("\n"):
1027
1028
                    if line.startswith("FAILED "):
                        line = line[len("FAILED ") :]
1029
1030
1031
                        line = line.split()[0].replace("\n", "")

                        if artifact_path["gpu"] not in model_results[model]["failures"]:
1032
                            model_results[model]["failures"][artifact_path["gpu"]] = []
1033

1034
1035
1036
                        model_results[model]["failures"][artifact_path["gpu"]].append(
                            {"line": line, "trace": stacktraces.pop(0)}
                        )
1037

1038
                        if re.search("test_modeling_tf_", line):
1039
1040
                            model_results[model]["failed"]["TensorFlow"][artifact_path["gpu"]] += 1

1041
                        elif re.search("test_modeling_flax_", line):
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
                            model_results[model]["failed"]["Flax"][artifact_path["gpu"]] += 1

                        elif re.search("test_modeling", line):
                            model_results[model]["failed"]["PyTorch"][artifact_path["gpu"]] += 1

                        elif re.search("test_tokenization", line):
                            model_results[model]["failed"]["Tokenizers"][artifact_path["gpu"]] += 1

                        elif re.search("test_pipelines", line):
                            model_results[model]["failed"]["Pipelines"][artifact_path["gpu"]] += 1

                        elif re.search("test_trainer", line):
                            model_results[model]["failed"]["Trainer"][artifact_path["gpu"]] += 1

                        elif re.search("onnx", line):
                            model_results[model]["failed"]["ONNX"][artifact_path["gpu"]] += 1

                        elif re.search("auto", line):
                            model_results[model]["failed"]["Auto"][artifact_path["gpu"]] += 1

                        else:
                            model_results[model]["failed"]["Unclassified"][artifact_path["gpu"]] += 1
                            unclassified_model_failures.append(line)

    # Additional runs
    additional_files = {
1068
1069
1070
1071
        "PyTorch pipelines": "run_pipelines_torch_gpu_test_reports",
        "TensorFlow pipelines": "run_pipelines_tf_gpu_test_reports",
        "Examples directory": "run_examples_gpu_test_reports",
        "Torch CUDA extension tests": "run_torch_cuda_extensions_gpu_test_reports",
1072
1073
    }

1074
    if ci_event in ["push", "Nightly CI"] or ci_event.startswith("Past CI"):
Yih-Dar's avatar
Yih-Dar committed
1075
1076
1077
        del additional_files["Examples directory"]
        del additional_files["PyTorch pipelines"]
        del additional_files["TensorFlow pipelines"]
1078
1079
1080
1081
1082
    elif ci_event.startswith("Scheduled CI (AMD)"):
        del additional_files["TensorFlow pipelines"]
        del additional_files["Torch CUDA extension tests"]
    elif ci_event.startswith("Push CI (AMD)"):
        additional_files = {}
Yih-Dar's avatar
Yih-Dar committed
1083

Yih-Dar's avatar
Yih-Dar committed
1084
1085
1086
1087
1088
1089
1090
    # A map associating the job names (specified by `inputs.job` in a workflow file) with the keys of
    # `additional_files`. This is used to remove some entries in `additional_files` that are not concerned by a
    # specific job. See below.
    job_to_test_map = {
        "run_pipelines_torch_gpu": "PyTorch pipelines",
        "run_pipelines_tf_gpu": "TensorFlow pipelines",
        "run_examples_gpu": "Examples directory",
1091
        "run_torch_cuda_extensions_gpu": "Torch CUDA extension tests",
Yih-Dar's avatar
Yih-Dar committed
1092
1093
1094
1095
1096
1097
1098
1099
1100
    }

    # Remove some entries in `additional_files` if they are not concerned.
    test_name = None
    job_name = os.getenv("CI_TEST_JOB")
    if job_name in job_to_test_map:
        test_name = job_to_test_map[job_name]
    additional_files = {k: v for k, v in additional_files.items() if k == test_name}

1101
1102
1103
1104
1105
1106
1107
    additional_results = {
        key: {
            "failed": {"unclassified": 0, "single": 0, "multi": 0},
            "success": 0,
            "time_spent": "",
            "error": False,
            "failures": {},
1108
            "job_link": {},
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
        }
        for key in additional_files.keys()
    }

    for key in additional_results.keys():
        # If a whole suite of test fails, the artifact isn't available.
        if additional_files[key] not in available_artifacts:
            additional_results[key]["error"] = True
            continue

        for artifact_path in available_artifacts[additional_files[key]].paths:
1120
            # Link to the GitHub Action job
1121
1122
            job = artifact_name_to_job_map[artifact_path["path"]]
            additional_results[key]["job_link"][artifact_path["gpu"]] = job["html_url"]
1123

1124
            artifact = retrieve_artifact(artifact_path["path"], artifact_path["gpu"])
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
            stacktraces = handle_stacktraces(artifact["failures_line"])

            failed, success, time_spent = handle_test_results(artifact["stats"])
            additional_results[key]["failed"][artifact_path["gpu"] or "unclassified"] += failed
            additional_results[key]["success"] += success
            additional_results[key]["time_spent"] += time_spent[1:-1] + ", "

            if len(artifact["errors"]):
                additional_results[key]["error"] = True

            if failed:
                for line in artifact["summary_short"].split("\n"):
1137
1138
                    if line.startswith("FAILED "):
                        line = line[len("FAILED ") :]
1139
1140
1141
                        line = line.split()[0].replace("\n", "")

                        if artifact_path["gpu"] not in additional_results[key]["failures"]:
1142
                            additional_results[key]["failures"][artifact_path["gpu"]] = []
1143

1144
1145
1146
                        additional_results[key]["failures"][artifact_path["gpu"]].append(
                            {"line": line, "trace": stacktraces.pop(0)}
                        )
1147

Yih-Dar's avatar
Yih-Dar committed
1148
    # Let's only check the warning for the model testing job. Currently, the job `run_extract_warnings` is only run
1149
    # when `inputs.job` (in the workflow file) is `run_models_gpu`. The reason is: otherwise we need to save several
Yih-Dar's avatar
Yih-Dar committed
1150
    # artifacts with different names which complicates the logic for an insignificant part of the CI workflow reporting.
1151
    selected_warnings = []
1152
    if job_name == "run_models_gpu":
Yih-Dar's avatar
Yih-Dar committed
1153
1154
1155
1156
        if "warnings_in_ci" in available_artifacts:
            directory = available_artifacts["warnings_in_ci"].paths[0]["path"]
            with open(os.path.join(directory, "selected_warnings.json")) as fp:
                selected_warnings = json.load(fp)
1157

1158
1159
    if not os.path.isdir(os.path.join(os.getcwd(), f"ci_results_{job_name}")):
        os.makedirs(os.path.join(os.getcwd(), f"ci_results_{job_name}"))
1160

1161
1162
1163
    target_workflow = "huggingface/transformers/.github/workflows/self-scheduled-caller.yml@refs/heads/main"
    is_scheduled_ci_run = os.environ.get("CI_WORKFLOW_REF") == target_workflow

Yih-Dar's avatar
Yih-Dar committed
1164
1165
    # Only the model testing job is concerned: this condition is to avoid other jobs to upload the empty list as
    # results.
1166
    if job_name == "run_models_gpu":
1167
        with open(f"ci_results_{job_name}/model_results.json", "w", encoding="UTF-8") as fp:
Yih-Dar's avatar
Yih-Dar committed
1168
            json.dump(model_results, fp, indent=4, ensure_ascii=False)
1169

1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
        # upload results to Hub dataset (only for the scheduled daily CI run on `main`)
        if is_scheduled_ci_run:
            api.upload_file(
                path_or_fileobj=f"ci_results_{job_name}/model_results.json",
                path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/model_results.json",
                repo_id="hf-internal-testing/transformers_daily_ci",
                repo_type="dataset",
                token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
            )

1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
    # Must have the same keys as in `additional_results`.
    # The values are used as the file names where to save the corresponding CI job results.
    test_to_result_name = {
        "PyTorch pipelines": "torch_pipeline",
        "TensorFlow pipelines": "tf_pipeline",
        "Examples directory": "example",
        "Torch CUDA extension tests": "deepspeed",
    }
    for job, job_result in additional_results.items():
        with open(f"ci_results_{job_name}/{test_to_result_name[job]}_results.json", "w", encoding="UTF-8") as fp:
            json.dump(job_result, fp, indent=4, ensure_ascii=False)

1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
        # upload results to Hub dataset (only for the scheduled daily CI run on `main`)
        if is_scheduled_ci_run:
            api.upload_file(
                path_or_fileobj=f"ci_results_{job_name}/{test_to_result_name[job]}_results.json",
                path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/{test_to_result_name[job]}_results.json",
                repo_id="hf-internal-testing/transformers_daily_ci",
                repo_type="dataset",
                token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
            )

1202
    prev_ci_artifacts = None
1203
1204
    if is_scheduled_ci_run:
        if job_name == "run_models_gpu":
1205
1206
1207
1208
1209
1210
1211
            # Get the last previously completed CI's failure tables
            artifact_names = [f"ci_results_{job_name}"]
            output_dir = os.path.join(os.getcwd(), "previous_reports")
            os.makedirs(output_dir, exist_ok=True)
            prev_ci_artifacts = get_last_daily_ci_reports(
                artifact_names=artifact_names, output_dir=output_dir, token=os.environ["ACCESS_REPO_INFO_TOKEN"]
            )
1212
1213
1214
1215
1216
1217
1218
1219
1220

    message = Message(
        title,
        ci_title,
        model_results,
        additional_results,
        selected_warnings=selected_warnings,
        prev_ci_artifacts=prev_ci_artifacts,
    )
1221

Yih-Dar's avatar
Yih-Dar committed
1222
    # send report only if there is any failure (for push CI)
1223
    if message.n_failures or (ci_event != "push" and not ci_event.startswith("Push CI (AMD)")):
Yih-Dar's avatar
Yih-Dar committed
1224
1225
        message.post()
        message.post_reply()