create_circleci_config.py 28 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# coding=utf-8
# Copyright 2022 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import copy
import os
19
import random
20
21
22
from dataclasses import dataclass
from typing import Any, Dict, List, Optional

23
import yaml
24
25


26
27
28
29
30
31
32
33
COMMON_ENV_VARIABLES = {
    "OMP_NUM_THREADS": 1,
    "TRANSFORMERS_IS_CI": True,
    "PYTEST_TIMEOUT": 120,
    "RUN_PIPELINE_TESTS": False,
    "RUN_PT_TF_CROSS_TESTS": False,
    "RUN_PT_FLAX_CROSS_TESTS": False,
}
Yih-Dar's avatar
Yih-Dar committed
34
# Disable the use of {"s": None} as the output is way too long, causing the navigation on CircleCI impractical
35
COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile", "v": None}
36
DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.8.12"}]
37
38


39
40
41
42
43
44
45
46
47
48
49
class EmptyJob:
    job_name = "empty"

    def to_dict(self):
        return {
            "working_directory": "~/transformers",
            "docker": copy.deepcopy(DEFAULT_DOCKER_IMAGE),
            "steps":["checkout"],
        }


50
51
52
53
54
@dataclass
class CircleCIJob:
    name: str
    additional_env: Dict[str, Any] = None
    cache_name: str = None
Arthur's avatar
Arthur committed
55
    cache_version: str = "0.8.2"
56
57
58
59
    docker_image: List[Dict[str, str]] = None
    install_steps: List[str] = None
    marker: Optional[str] = None
    parallelism: Optional[int] = 1
Yih-Dar's avatar
Yih-Dar committed
60
    pytest_num_workers: int = 12
61
    pytest_options: Dict[str, Any] = None
Yih-Dar's avatar
Yih-Dar committed
62
    resource_class: Optional[str] = "2xlarge"
63
64
    tests_to_run: Optional[List[str]] = None
    working_directory: str = "~/transformers"
65
66
    # This should be only used for doctest job!
    command_timeout: Optional[int] = None
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

    def __post_init__(self):
        # Deal with defaults for mutable attributes.
        if self.additional_env is None:
            self.additional_env = {}
        if self.cache_name is None:
            self.cache_name = self.name
        if self.docker_image is None:
            # Let's avoid changing the default list and make a copy.
            self.docker_image = copy.deepcopy(DEFAULT_DOCKER_IMAGE)
        if self.install_steps is None:
            self.install_steps = []
        if self.pytest_options is None:
            self.pytest_options = {}
        if isinstance(self.tests_to_run, str):
            self.tests_to_run = [self.tests_to_run]
83
84
        if self.parallelism is None:
            self.parallelism = 1
85
86

    def to_dict(self):
87
88
        env = COMMON_ENV_VARIABLES.copy()
        env.update(self.additional_env)
89
90
91
92
93

        cache_branch_prefix = os.environ.get("CIRCLE_BRANCH", "pull")
        if cache_branch_prefix != "main":
            cache_branch_prefix = "pull"

94
95
96
        job = {
            "working_directory": self.working_directory,
            "docker": self.docker_image,
97
            "environment": env,
98
99
100
101
102
103
104
105
106
107
108
        }
        if self.resource_class is not None:
            job["resource_class"] = self.resource_class
        if self.parallelism is not None:
            job["parallelism"] = self.parallelism
        steps = [
            "checkout",
            {"attach_workspace": {"at": "~/transformers/test_preparation"}},
            {
                "restore_cache": {
                    "keys": [
109
                        # check the fully-matched cache first
110
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
111
112
113
114
                        # try the partially-matched cache from `main`
                        f"v{self.cache_version}-{self.cache_name}-main-pip-",
                        # try the general partially-matched cache
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-",
115
116
117
                    ]
                }
            },
118
119
120
            {
                "restore_cache": {
                    "keys": [
121
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
122
123
                        f"v{self.cache_version}-{self.cache_name}-main-site-packages-",
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-",
124
125
126
                    ]
                }
            },
127
128
        ]
        steps.extend([{"run": l} for l in self.install_steps])
129
        steps.extend([{"run": 'pip install "fsspec>=2023.5.0,<2023.10.0"'}])
130
        steps.extend([{"run": "pip install pytest-subtests"}])
131
132
        steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
        steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
133
134

        all_options = {**COMMON_PYTEST_OPTIONS, **self.pytest_options}
135
        pytest_flags = [f"--{key}={value}" if (value is not None or key in ["doctest-modules"]) else f"-{key}" for key, value in all_options.items()]
136
137
138
        pytest_flags.append(
            f"--make-reports={self.name}" if "examples" in self.name else f"--make-reports=tests_{self.name}"
        )
139
140
141

        steps.append({"run": {"name": "Create `test-results` directory", "command": "mkdir test-results"}})

142
143
144
        test_command = ""
        if self.command_timeout:
            test_command = f"timeout {self.command_timeout} "
145
        test_command += f"python -m pytest --junitxml=test-results/junit.xml -n {self.pytest_num_workers} " + " ".join(pytest_flags)
amyeroberts's avatar
amyeroberts committed
146

147
148
149
150
151
        if self.parallelism == 1:
            if self.tests_to_run is None:
                test_command += " << pipeline.parameters.tests_to_run >>"
            else:
                test_command += " " + " ".join(self.tests_to_run)
152
        else:
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
            # We need explicit list instead of `pipeline.parameters.tests_to_run` (only available at job runtime)
            tests = self.tests_to_run
            if tests is None:
                folder = os.environ["test_preparation_dir"]
                test_file = os.path.join(folder, "filtered_test_list.txt")
                if os.path.exists(test_file):
                    with open(test_file) as f:
                        tests = f.read().split(" ")

            # expand the test list
            if tests == ["tests"]:
                tests = [os.path.join("tests", x) for x in os.listdir("tests")]
            expanded_tests = []
            for test in tests:
                if test.endswith(".py"):
                    expanded_tests.append(test)
                elif test == "tests/models":
                    expanded_tests.extend([os.path.join(test, x) for x in os.listdir(test)])
                elif test == "tests/pipelines":
                    expanded_tests.extend([os.path.join(test, x) for x in os.listdir(test)])
                else:
                    expanded_tests.append(test)
            # Avoid long tests always being collected together
            random.shuffle(expanded_tests)
            tests = " ".join(expanded_tests)

            # Each executor to run ~10 tests
            n_executors = max(len(tests) // 10, 1)
            # Avoid empty test list on some executor(s) or launching too many executors
            if n_executors > self.parallelism:
                n_executors = self.parallelism
            job["parallelism"] = n_executors

            # Need to be newline separated for the command `circleci tests split` below
            command = f'echo {tests} | tr " " "\\n" >> tests.txt'
            steps.append({"run": {"name": "Get tests", "command": command}})

            command = 'TESTS=$(circleci tests split tests.txt) && echo $TESTS > splitted_tests.txt'
            steps.append({"run": {"name": "Split tests", "command": command}})

            steps.append({"store_artifacts": {"path": "~/transformers/tests.txt"}})
            steps.append({"store_artifacts": {"path": "~/transformers/splitted_tests.txt"}})

196
197
198
199
            test_command = ""
            if self.timeout:
                test_command = f"timeout {self.timeout} "
            test_command += f"python -m pytest -n {self.pytest_num_workers} " + " ".join(pytest_flags)
200
            test_command += " $(cat splitted_tests.txt)"
201
202
        if self.marker is not None:
            test_command += f" -m {self.marker}"
203
204
205
206
207
208
209
210
211
212
213

        if self.name == "pr_documentation_tests":
            # can't use ` | tee tee tests_output.txt` as usual
            test_command += " > tests_output.txt"
            # Save the return code, so we can check if it is timeout in the next step.
            test_command += '; touch "$?".txt'
            # Never fail the test step for the doctest job. We will check the results in the next step, and fail that
            # step instead if the actual test failures are found. This is to avoid the timeout being reported as test
            # failure.
            test_command = f"({test_command}) || true"
        else:
214
            test_command = f"({test_command} | tee tests_output.txt) || true"
215
        steps.append({"run": {"name": "Run tests", "command": test_command}})
216

217
218
219
220
        # Deal with errors
        check_test_command = f'if [ -s reports/{self.job_name}/errors.txt ]; '
        check_test_command += 'then echo "Some tests errored out!"; echo ""; '
        check_test_command += f'cat reports/{self.job_name}/errors.txt; '
221
222
        check_test_command += 'echo ""; echo ""; '

223
        py_command = f'import os; fp = open("reports/{self.job_name}/summary_short.txt"); failed = os.linesep.join([x for x in fp.read().split(os.linesep) if x.startswith("ERROR ")]); fp.close(); fp = open("summary_short.txt", "w"); fp.write(failed); fp.close()'
224
        check_test_command += f"$(python3 -c '{py_command}'); "
225
        check_test_command += 'cat summary_short.txt; echo ""; exit -1; '
226

227
228
229
230
231
232
233
234
        # Deeal with failed tests
        check_test_command += f'elif [ -s reports/{self.job_name}/failures_short.txt ]; '
        check_test_command += 'then echo "Some tests failed!"; echo ""; '
        check_test_command += f'cat reports/{self.job_name}/failures_short.txt; '
        check_test_command += 'echo ""; echo ""; '

        py_command = f'import os; fp = open("reports/{self.job_name}/summary_short.txt"); failed = os.linesep.join([x for x in fp.read().split(os.linesep) if x.startswith("FAILED ")]); fp.close(); fp = open("summary_short.txt", "w"); fp.write(failed); fp.close()'
        check_test_command += f"$(python3 -c '{py_command}'); "
235
        check_test_command += 'cat summary_short.txt; echo ""; exit -1; '
236

237
        check_test_command += f'elif [ -s reports/{self.job_name}/stats.txt ]; then echo "All tests pass!"; '
238

239
240
        # return code `124` means the previous (pytest run) step is timeout
        if self.name == "pr_documentation_tests":
241
242
243
244
245
            check_test_command += 'elif [ -f 124.txt ]; then echo "doctest timeout!"; '

        check_test_command += 'else echo "other fatal error"; echo ""; exit -1; fi;'

        steps.append({"run": {"name": "Check test results", "command": check_test_command}})
246

247
248
        steps.append({"store_test_results": {"path": "test-results"}})

249
250
        steps.append({"store_artifacts": {"path": "~/transformers/tests_output.txt"}})
        steps.append({"store_artifacts": {"path": "~/transformers/reports"}})
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269

        # save cache at the end: so pytest step runs before cache saving and we can see results earlier
        steps.append(
            {
                "save_cache": {
                    "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
                    "paths": ["~/.cache/pip"],
                }
            }
        )
        steps.append(
            {
                "save_cache": {
                    "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
                    "paths": ["~/.pyenv/versions/"],
                }
            }
        )

270
271
272
273
274
275
276
277
278
279
280
281
282
        job["steps"] = steps
        return job

    @property
    def job_name(self):
        return self.name if "examples" in self.name else f"tests_{self.name}"


# JOBS
torch_and_tf_job = CircleCIJob(
    "torch_and_tf",
    additional_env={"RUN_PT_TF_CROSS_TESTS": True},
    install_steps=[
283
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
284
        "git lfs install",
Yih-Dar's avatar
Yih-Dar committed
285
286
287
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
        "pip install -U --upgrade-strategy eager tensorflow_probability",
288
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
Yih-Dar's avatar
Yih-Dar committed
289
        # TODO: remove this one after fixing the dependency issue(s) above
Yih-Dar's avatar
Yih-Dar committed
290
        "pip install -U --upgrade-strategy eager torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu",
291
292
293
294
295
296
297
298
299
300
301
    ],
    marker="is_pt_tf_cross_test",
    pytest_options={"rA": None, "durations": 0},
)


torch_and_flax_job = CircleCIJob(
    "torch_and_flax",
    additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
302
303
        "pip install -U --upgrade-strategy eager --upgrade pip",
        "pip install -U --upgrade-strategy eager .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
304
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
305
306
307
308
309
310
311
312
313
314
    ],
    marker="is_pt_flax_cross_test",
    pytest_options={"rA": None, "durations": 0},
)


torch_job = CircleCIJob(
    "torch",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
Yih-Dar's avatar
Yih-Dar committed
315
316
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
317
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
318
    ],
319
    parallelism=1,
Yih-Dar's avatar
Yih-Dar committed
320
    pytest_num_workers=12,
321
322
323
324
325
326
)


tf_job = CircleCIJob(
    "tf",
    install_steps=[
327
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
Yih-Dar's avatar
Yih-Dar committed
328
329
330
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
        "pip install -U --upgrade-strategy eager tensorflow_probability",
331
    ],
332
    parallelism=1,
333
334
335
336
337
338
339
)


flax_job = CircleCIJob(
    "flax",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
340
341
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece,flax-speech,vision]",
342
    ],
343
    parallelism=1,
344
345
346
347
348
)


pipelines_torch_job = CircleCIJob(
    "pipelines_torch",
349
    additional_env={"RUN_PIPELINE_TESTS": True},
350
351
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
352
353
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
354
    ],
355
    marker="is_pipeline_test",
Yih-Dar's avatar
Yih-Dar committed
356
    pytest_num_workers=12,
357
358
359
360
361
)


pipelines_tf_job = CircleCIJob(
    "pipelines_tf",
362
    additional_env={"RUN_PIPELINE_TESTS": True},
363
    install_steps=[
Sylvain Gugger's avatar
Sylvain Gugger committed
364
        "sudo apt-get -y update && sudo apt-get install -y cmake",
Yih-Dar's avatar
Yih-Dar committed
365
366
367
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,vision]",
        "pip install -U --upgrade-strategy eager tensorflow_probability",
368
    ],
369
    marker="is_pipeline_test",
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
)


custom_tokenizers_job = CircleCIJob(
    "custom_tokenizers",
    additional_env={"RUN_CUSTOM_TOKENIZERS": True},
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y cmake",
        {
            "name": "install jumanpp",
            "command":
                "wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc3/jumanpp-2.0.0-rc3.tar.xz\n"
                "tar xvf jumanpp-2.0.0-rc3.tar.xz\n"
                "mkdir jumanpp-2.0.0-rc3/bld\n"
                "cd jumanpp-2.0.0-rc3/bld\n"
                "sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n"
                "sudo make install\n",
        },
Yih-Dar's avatar
Yih-Dar committed
388
389
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
390
391
392
393
394
395
396
397
398
399
400
401
402
403
        "python -m unidic download",
    ],
    parallelism=None,
    resource_class=None,
    tests_to_run=[
        "./tests/models/bert_japanese/test_tokenization_bert_japanese.py",
        "./tests/models/openai/test_tokenization_openai.py",
        "./tests/models/clip/test_tokenization_clip.py",
    ],
)


examples_torch_job = CircleCIJob(
    "examples_torch",
404
    additional_env={"OMP_NUM_THREADS": 8},
405
406
407
    cache_name="torch_examples",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
408
409
410
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,torch,sentencepiece,testing,torch-speech]",
        "pip install -U --upgrade-strategy eager -r examples/pytorch/_tests_requirements.txt",
411
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
412
    ],
413
    pytest_num_workers=1,
414
415
416
417
418
419
420
)


examples_tensorflow_job = CircleCIJob(
    "examples_tensorflow",
    cache_name="tensorflow_examples",
    install_steps=[
421
        "sudo apt-get -y update && sudo apt-get install -y cmake",
Yih-Dar's avatar
Yih-Dar committed
422
423
424
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tensorflow,sentencepiece,testing]",
        "pip install -U --upgrade-strategy eager -r examples/tensorflow/_tests_requirements.txt",
425
426
427
428
429
430
431
432
    ],
)


examples_flax_job = CircleCIJob(
    "examples_flax",
    cache_name="flax_examples",
    install_steps=[
Yih-Dar's avatar
Yih-Dar committed
433
434
435
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece]",
        "pip install -U --upgrade-strategy eager -r examples/flax/_tests_requirements.txt",
436
437
438
439
440
441
    ],
)


hub_job = CircleCIJob(
    "hub",
Sylvain Gugger's avatar
Sylvain Gugger committed
442
    additional_env={"HUGGINGFACE_CO_STAGING": True},
443
444
445
446
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install git-lfs",
        'git config --global user.email "ci@dummy.com"',
        'git config --global user.name "ci"',
Yih-Dar's avatar
Yih-Dar committed
447
        "pip install --upgrade --upgrade-strategy eager pip",
Sylvain Gugger's avatar
Sylvain Gugger committed
448
        "pip install -U --upgrade-strategy eager .[torch,sentencepiece,testing,vision]",
449
450
451
452
453
454
455
456
457
    ],
    marker="is_staging_test",
    pytest_num_workers=1,
)


onnx_job = CircleCIJob(
    "onnx",
    install_steps=[
458
        "sudo apt-get -y update && sudo apt-get install -y cmake",
Yih-Dar's avatar
Yih-Dar committed
459
460
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
461
462
463
464
465
466
    ],
    pytest_options={"k onnx": None},
    pytest_num_workers=1,
)


467
468
exotic_models_job = CircleCIJob(
    "exotic_models",
469
470
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
Yih-Dar's avatar
Yih-Dar committed
471
472
473
474
475
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[torch,testing,vision]",
        "pip install -U --upgrade-strategy eager torchvision",
        "pip install -U --upgrade-strategy eager scipy",
        "pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'",
476
        "sudo apt install tesseract-ocr",
Yih-Dar's avatar
Yih-Dar committed
477
        "pip install -U --upgrade-strategy eager pytesseract",
NielsRogge's avatar
NielsRogge committed
478
        "pip install --upgrade-strategy eager sentencepiece",
Yih-Dar's avatar
Yih-Dar committed
479
        "pip install -U --upgrade-strategy eager natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels",
NielsRogge's avatar
NielsRogge committed
480
481
482
        "pip install -U --upgrade-strategy eager python-Levenshtein",
        "pip install -U --upgrade-strategy eager opencv-python",
        "pip install -U --upgrade-strategy eager nltk",
Matt's avatar
Matt committed
483
        "pip uninstall -y torch torchvision torchaudio && pip install -U --upgrade-strategy eager 'torch<2.2.0' 'torchvision<0.17' 'torchaudio<2.2.0'"
484
485
486
487
    ],
    tests_to_run=[
        "tests/models/*layoutlmv*",
        "tests/models/*nat",
NielsRogge's avatar
NielsRogge committed
488
        "tests/models/deta",
NielsRogge's avatar
NielsRogge committed
489
        "tests/models/udop",
NielsRogge's avatar
NielsRogge committed
490
        "tests/models/nougat",
491
492
493
494
495
496
    ],
    pytest_num_workers=1,
    pytest_options={"durations": 100},
)


Sylvain Gugger's avatar
Sylvain Gugger committed
497
498
499
repo_utils_job = CircleCIJob(
    "repo_utils",
    install_steps=[
Yih-Dar's avatar
Yih-Dar committed
500
501
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[quality,testing,torch]",
Sylvain Gugger's avatar
Sylvain Gugger committed
502
503
504
    ],
    parallelism=None,
    pytest_num_workers=1,
505
    resource_class="large",
Sylvain Gugger's avatar
Sylvain Gugger committed
506
507
508
    tests_to_run="tests/repo_utils",
)

509
510
511
512
513

# We also include a `dummy.py` file in the files to be doc-tested to prevent edge case failure. Otherwise, the pytest
# hangs forever during test collection while showing `collecting 0 items / 21 errors`. (To see this, we have to remove
# the bash output redirection.)
py_command = 'from utils.tests_fetcher import get_doctest_files; to_test = get_doctest_files() + ["dummy.py"]; to_test = " ".join(to_test); print(to_test)'
514
py_command = f"$(python3 -c '{py_command}')"
515
command = f'echo "{py_command}" > pr_documentation_tests_temp.txt'
516
517
518
519
doc_test_job = CircleCIJob(
    "pr_documentation_tests",
    additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
    install_steps=[
520
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
Yih-Dar's avatar
Yih-Dar committed
521
522
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager -e .[dev]",
523
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
524
        "pip install --upgrade --upgrade-strategy eager 'pytest<8.0.0' pytest-sugar",
Yih-Dar's avatar
Yih-Dar committed
525
        "pip install -U --upgrade-strategy eager natten==0.15.1+torch210cpu -f https://shi-labs.com/natten/wheels",
526
        "pip install -U --upgrade-strategy eager g2p-en",
Yih-Dar's avatar
Yih-Dar committed
527
528
        # TODO: remove this one after fixing the dependency issue(s) above
        "pip install -U --upgrade-strategy eager torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu",
529
530
531
532
533
534
        "find -name __pycache__ -delete",
        "find . -name \*.pyc -delete",
        # Add an empty file to keep the test step running correctly even no file is selected to be tested.
        "touch dummy.py",
        {
            "name": "Get files to test",
535
            "command": command,
536
537
        },
        {
538
            "name": "Show information in `Get files to test`",
539
            "command":
540
                "cat pr_documentation_tests_temp.txt"
541
542
        },
        {
543
            "name": "Get the last line in `pr_documentation_tests.txt`",
544
            "command":
545
                "tail -n1 pr_documentation_tests_temp.txt | tee pr_documentation_tests.txt"
546
547
        },
    ],
548
    tests_to_run="$(cat pr_documentation_tests.txt)",  # noqa
549
    pytest_options={"-doctest-modules": None, "doctest-glob": "*.md", "dist": "loadfile", "rvsA": None},
550
551
552
553
    command_timeout=1200,  # test cannot run longer than 1200 seconds
    pytest_num_workers=1,
)

554
555
556
557
558
559
560
561
562
REGULAR_TESTS = [
    torch_and_tf_job,
    torch_and_flax_job,
    torch_job,
    tf_job,
    flax_job,
    custom_tokenizers_job,
    hub_job,
    onnx_job,
563
    exotic_models_job,
564
565
566
567
568
569
570
571
572
573
]
EXAMPLES_TESTS = [
    examples_torch_job,
    examples_tensorflow_job,
    examples_flax_job,
]
PIPELINE_TESTS = [
    pipelines_torch_job,
    pipelines_tf_job,
]
Sylvain Gugger's avatar
Sylvain Gugger committed
574
REPO_UTIL_TESTS = [repo_utils_job]
575
576
DOC_TESTS = [doc_test_job]

577
578
579
580

def create_circleci_config(folder=None):
    if folder is None:
        folder = os.getcwd()
581
582
    # Used in CircleCIJob.to_dict() to expand the test list (for using parallelism)
    os.environ["test_preparation_dir"] = folder
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
    jobs = []
    all_test_file = os.path.join(folder, "test_list.txt")
    if os.path.exists(all_test_file):
        with open(all_test_file) as f:
            all_test_list = f.read()
    else:
        all_test_list = []
    if len(all_test_list) > 0:
        jobs.extend(PIPELINE_TESTS)

    test_file = os.path.join(folder, "filtered_test_list.txt")
    if os.path.exists(test_file):
        with open(test_file) as f:
            test_list = f.read()
    else:
        test_list = []
    if len(test_list) > 0:
        jobs.extend(REGULAR_TESTS)

Yih-Dar's avatar
Yih-Dar committed
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
        extended_tests_to_run = set(test_list.split())
        # Extend the test files for cross test jobs
        for job in jobs:
            if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
                for test_path in copy.copy(extended_tests_to_run):
                    dir_path, fn = os.path.split(test_path)
                    if fn.startswith("test_modeling_tf_"):
                        fn = fn.replace("test_modeling_tf_", "test_modeling_")
                    elif fn.startswith("test_modeling_flax_"):
                        fn = fn.replace("test_modeling_flax_", "test_modeling_")
                    else:
                        if job.job_name == "test_torch_and_tf":
                            fn = fn.replace("test_modeling_", "test_modeling_tf_")
                        elif job.job_name == "test_torch_and_flax":
                            fn = fn.replace("test_modeling_", "test_modeling_flax_")
                    new_test_file = str(os.path.join(dir_path, fn))
                    if os.path.isfile(new_test_file):
                        if new_test_file not in extended_tests_to_run:
                            extended_tests_to_run.add(new_test_file)
        extended_tests_to_run = sorted(extended_tests_to_run)
        for job in jobs:
            if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
                job.tests_to_run = extended_tests_to_run
        fn = "filtered_test_list_cross_tests.txt"
        f_path = os.path.join(folder, fn)
        with open(f_path, "w") as fp:
            fp.write(" ".join(extended_tests_to_run))

630
631
    example_file = os.path.join(folder, "examples_test_list.txt")
    if os.path.exists(example_file) and os.path.getsize(example_file) > 0:
632
        with open(example_file, "r", encoding="utf-8") as f:
633
            example_tests = f.read()
634
635
636
637
638
        for job in EXAMPLES_TESTS:
            framework = job.name.replace("examples_", "").replace("torch", "pytorch")
            if example_tests == "all":
                job.tests_to_run = [f"examples/{framework}"]
            else:
639
                job.tests_to_run = [f for f in example_tests.split(" ") if f.startswith(f"examples/{framework}")]
640

641
642
            if len(job.tests_to_run) > 0:
                jobs.append(job)
643

644
645
646
647
648
649
650
651
652
    doctest_file = os.path.join(folder, "doctest_list.txt")
    if os.path.exists(doctest_file):
        with open(doctest_file) as f:
            doctest_list = f.read()
    else:
        doctest_list = []
    if len(doctest_list) > 0:
        jobs.extend(DOC_TESTS)

Sylvain Gugger's avatar
Sylvain Gugger committed
653
654
655
    repo_util_file = os.path.join(folder, "test_repo_utils.txt")
    if os.path.exists(repo_util_file) and os.path.getsize(repo_util_file) > 0:
        jobs.extend(REPO_UTIL_TESTS)
656

657
658
659
660
661
662
663
664
665
666
667
668
    if len(jobs) == 0:
        jobs = [EmptyJob()]
    config = {"version": "2.1"}
    config["parameters"] = {
        # Only used to accept the parameters from the trigger
        "nightly": {"type": "boolean", "default": False},
        "tests_to_run": {"type": "string", "default": test_list},
    }
    config["jobs"] = {j.job_name: j.to_dict() for j in jobs}
    config["workflows"] = {"version": 2, "run_tests": {"jobs": [j.job_name for j in jobs]}}
    with open(os.path.join(folder, "generated_config.yml"), "w") as f:
        f.write(yaml.dump(config, indent=2, width=1000000, sort_keys=False))
669
670
671
672
673
674
675
676
677
678


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--fetcher_folder", type=str, default=None, help="Only test that all tests and modules are accounted for."
    )
    args = parser.parse_args()

    create_circleci_config(args.fetcher_folder)