create_circleci_config.py 27 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# coding=utf-8
# Copyright 2022 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import copy
18
import glob
19
import os
20
import random
21
22
23
from dataclasses import dataclass
from typing import Any, Dict, List, Optional

24
import yaml
25
26


27
28
29
30
31
32
33
34
COMMON_ENV_VARIABLES = {
    "OMP_NUM_THREADS": 1,
    "TRANSFORMERS_IS_CI": True,
    "PYTEST_TIMEOUT": 120,
    "RUN_PIPELINE_TESTS": False,
    "RUN_PT_TF_CROSS_TESTS": False,
    "RUN_PT_FLAX_CROSS_TESTS": False,
}
Yih-Dar's avatar
Yih-Dar committed
35
36
# Disable the use of {"s": None} as the output is way too long, causing the navigation on CircleCI impractical
COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile"}
37
DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.8.12"}]
38
39


40
41
42
43
44
45
46
47
48
49
50
class EmptyJob:
    job_name = "empty"

    def to_dict(self):
        return {
            "working_directory": "~/transformers",
            "docker": copy.deepcopy(DEFAULT_DOCKER_IMAGE),
            "steps":["checkout"],
        }


51
52
53
54
55
@dataclass
class CircleCIJob:
    name: str
    additional_env: Dict[str, Any] = None
    cache_name: str = None
Yih-Dar's avatar
Yih-Dar committed
56
    cache_version: str = "0.7"
57
58
59
60
61
62
63
64
65
    docker_image: List[Dict[str, str]] = None
    install_steps: List[str] = None
    marker: Optional[str] = None
    parallelism: Optional[int] = 1
    pytest_num_workers: int = 8
    pytest_options: Dict[str, Any] = None
    resource_class: Optional[str] = "xlarge"
    tests_to_run: Optional[List[str]] = None
    working_directory: str = "~/transformers"
66
67
    # This should be only used for doctest job!
    command_timeout: Optional[int] = None
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

    def __post_init__(self):
        # Deal with defaults for mutable attributes.
        if self.additional_env is None:
            self.additional_env = {}
        if self.cache_name is None:
            self.cache_name = self.name
        if self.docker_image is None:
            # Let's avoid changing the default list and make a copy.
            self.docker_image = copy.deepcopy(DEFAULT_DOCKER_IMAGE)
        if self.install_steps is None:
            self.install_steps = []
        if self.pytest_options is None:
            self.pytest_options = {}
        if isinstance(self.tests_to_run, str):
            self.tests_to_run = [self.tests_to_run]
84
85
        if self.parallelism is None:
            self.parallelism = 1
86
87

    def to_dict(self):
88
89
        env = COMMON_ENV_VARIABLES.copy()
        env.update(self.additional_env)
90
91
92
93
94

        cache_branch_prefix = os.environ.get("CIRCLE_BRANCH", "pull")
        if cache_branch_prefix != "main":
            cache_branch_prefix = "pull"

95
96
97
        job = {
            "working_directory": self.working_directory,
            "docker": self.docker_image,
98
            "environment": env,
99
100
101
102
103
104
105
106
107
108
109
        }
        if self.resource_class is not None:
            job["resource_class"] = self.resource_class
        if self.parallelism is not None:
            job["parallelism"] = self.parallelism
        steps = [
            "checkout",
            {"attach_workspace": {"at": "~/transformers/test_preparation"}},
            {
                "restore_cache": {
                    "keys": [
110
                        # check the fully-matched cache first
111
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
112
113
114
115
                        # try the partially-matched cache from `main`
                        f"v{self.cache_version}-{self.cache_name}-main-pip-",
                        # try the general partially-matched cache
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-",
116
117
118
                    ]
                }
            },
119
120
121
            {
                "restore_cache": {
                    "keys": [
122
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
123
124
                        f"v{self.cache_version}-{self.cache_name}-main-site-packages-",
                        f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-",
125
126
127
                    ]
                }
            },
128
129
        ]
        steps.extend([{"run": l} for l in self.install_steps])
130
        steps.extend([{"run": 'pip install "fsspec>=2023.5.0,<2023.10.0"'}])
131
        steps.extend([{"run": "pip install pytest-subtests"}])
132
133
134
135
136
        steps.append(
            {
                "save_cache": {
                    "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-pip-" + '{{ checksum "setup.py" }}',
                    "paths": ["~/.cache/pip"],
137
                }
138
139
140
141
142
143
144
            }
        )
        steps.append(
            {
                "save_cache": {
                    "key": f"v{self.cache_version}-{self.cache_name}-{cache_branch_prefix}-site-packages-" + '{{ checksum "setup.py" }}',
                    "paths": ["~/.pyenv/versions/"],
145
                }
146
147
            }
        )
148
149
        steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
        steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
150
151

        all_options = {**COMMON_PYTEST_OPTIONS, **self.pytest_options}
152
        pytest_flags = [f"--{key}={value}" if (value is not None or key in ["doctest-modules"]) else f"-{key}" for key, value in all_options.items()]
153
154
155
        pytest_flags.append(
            f"--make-reports={self.name}" if "examples" in self.name else f"--make-reports=tests_{self.name}"
        )
156
157
158

        steps.append({"run": {"name": "Create `test-results` directory", "command": "mkdir test-results"}})

159
160
161
        test_command = ""
        if self.command_timeout:
            test_command = f"timeout {self.command_timeout} "
162
        test_command += f"python -m pytest --junitxml=test-results/junit.xml -n {self.pytest_num_workers} " + " ".join(pytest_flags)
amyeroberts's avatar
amyeroberts committed
163

164
165
166
167
168
        if self.parallelism == 1:
            if self.tests_to_run is None:
                test_command += " << pipeline.parameters.tests_to_run >>"
            else:
                test_command += " " + " ".join(self.tests_to_run)
169
        else:
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
            # We need explicit list instead of `pipeline.parameters.tests_to_run` (only available at job runtime)
            tests = self.tests_to_run
            if tests is None:
                folder = os.environ["test_preparation_dir"]
                test_file = os.path.join(folder, "filtered_test_list.txt")
                if os.path.exists(test_file):
                    with open(test_file) as f:
                        tests = f.read().split(" ")

            # expand the test list
            if tests == ["tests"]:
                tests = [os.path.join("tests", x) for x in os.listdir("tests")]
            expanded_tests = []
            for test in tests:
                if test.endswith(".py"):
                    expanded_tests.append(test)
                elif test == "tests/models":
                    expanded_tests.extend([os.path.join(test, x) for x in os.listdir(test)])
                elif test == "tests/pipelines":
                    expanded_tests.extend([os.path.join(test, x) for x in os.listdir(test)])
                else:
                    expanded_tests.append(test)
            # Avoid long tests always being collected together
            random.shuffle(expanded_tests)
            tests = " ".join(expanded_tests)

            # Each executor to run ~10 tests
            n_executors = max(len(tests) // 10, 1)
            # Avoid empty test list on some executor(s) or launching too many executors
            if n_executors > self.parallelism:
                n_executors = self.parallelism
            job["parallelism"] = n_executors

            # Need to be newline separated for the command `circleci tests split` below
            command = f'echo {tests} | tr " " "\\n" >> tests.txt'
            steps.append({"run": {"name": "Get tests", "command": command}})

            command = 'TESTS=$(circleci tests split tests.txt) && echo $TESTS > splitted_tests.txt'
            steps.append({"run": {"name": "Split tests", "command": command}})

            steps.append({"store_artifacts": {"path": "~/transformers/tests.txt"}})
            steps.append({"store_artifacts": {"path": "~/transformers/splitted_tests.txt"}})

213
214
215
216
            test_command = ""
            if self.timeout:
                test_command = f"timeout {self.timeout} "
            test_command += f"python -m pytest -n {self.pytest_num_workers} " + " ".join(pytest_flags)
217
            test_command += " $(cat splitted_tests.txt)"
218
219
        if self.marker is not None:
            test_command += f" -m {self.marker}"
220
221
222
223
224
225
226
227
228
229
230

        if self.name == "pr_documentation_tests":
            # can't use ` | tee tee tests_output.txt` as usual
            test_command += " > tests_output.txt"
            # Save the return code, so we can check if it is timeout in the next step.
            test_command += '; touch "$?".txt'
            # Never fail the test step for the doctest job. We will check the results in the next step, and fail that
            # step instead if the actual test failures are found. This is to avoid the timeout being reported as test
            # failure.
            test_command = f"({test_command}) || true"
        else:
231
            test_command += " || true"
232
        steps.append({"run": {"name": "Run tests", "command": test_command}})
233

234
235
236
237
        # Deal with errors
        check_test_command = f'if [ -s reports/{self.job_name}/errors.txt ]; '
        check_test_command += 'then echo "Some tests errored out!"; echo ""; '
        check_test_command += f'cat reports/{self.job_name}/errors.txt; '
238
239
        check_test_command += 'echo ""; echo ""; '

240
        py_command = f'import os; fp = open("reports/{self.job_name}/summary_short.txt"); failed = os.linesep.join([x for x in fp.read().split(os.linesep) if x.startswith("ERROR ")]); fp.close(); fp = open("summary_short.txt", "w"); fp.write(failed); fp.close()'
241
        check_test_command += f"$(python3 -c '{py_command}'); "
242
        check_test_command += f'cat summary_short.txt; echo ""; exit -1; '
243

244
245
246
247
248
249
250
251
        # Deeal with failed tests
        check_test_command += f'elif [ -s reports/{self.job_name}/failures_short.txt ]; '
        check_test_command += 'then echo "Some tests failed!"; echo ""; '
        check_test_command += f'cat reports/{self.job_name}/failures_short.txt; '
        check_test_command += 'echo ""; echo ""; '

        py_command = f'import os; fp = open("reports/{self.job_name}/summary_short.txt"); failed = os.linesep.join([x for x in fp.read().split(os.linesep) if x.startswith("FAILED ")]); fp.close(); fp = open("summary_short.txt", "w"); fp.write(failed); fp.close()'
        check_test_command += f"$(python3 -c '{py_command}'); "
252
        check_test_command += f'cat summary_short.txt; echo ""; exit -1; '
253

254
        check_test_command += f'elif [ -s reports/{self.job_name}/stats.txt ]; then echo "All tests pass!"; '
255

256
257
        # return code `124` means the previous (pytest run) step is timeout
        if self.name == "pr_documentation_tests":
258
259
260
261
262
            check_test_command += 'elif [ -f 124.txt ]; then echo "doctest timeout!"; '

        check_test_command += 'else echo "other fatal error"; echo ""; exit -1; fi;'

        steps.append({"run": {"name": "Check test results", "command": check_test_command}})
263

264
265
        steps.append({"store_test_results": {"path": "test-results"}})

266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
        steps.append({"store_artifacts": {"path": "~/transformers/tests_output.txt"}})
        steps.append({"store_artifacts": {"path": "~/transformers/reports"}})
        job["steps"] = steps
        return job

    @property
    def job_name(self):
        return self.name if "examples" in self.name else f"tests_{self.name}"


# JOBS
torch_and_tf_job = CircleCIJob(
    "torch_and_tf",
    additional_env={"RUN_PT_TF_CROSS_TESTS": True},
    install_steps=[
281
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
282
        "git lfs install",
Yih-Dar's avatar
Yih-Dar committed
283
284
285
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
        "pip install -U --upgrade-strategy eager tensorflow_probability",
286
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
287
288
289
290
291
292
293
294
295
296
297
    ],
    marker="is_pt_tf_cross_test",
    pytest_options={"rA": None, "durations": 0},
)


torch_and_flax_job = CircleCIJob(
    "torch_and_flax",
    additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
298
299
        "pip install -U --upgrade-strategy eager --upgrade pip",
        "pip install -U --upgrade-strategy eager .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
300
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
301
302
303
304
305
306
307
308
309
310
    ],
    marker="is_pt_flax_cross_test",
    pytest_options={"rA": None, "durations": 0},
)


torch_job = CircleCIJob(
    "torch",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
Yih-Dar's avatar
Yih-Dar committed
311
312
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
313
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
314
    ],
315
    parallelism=1,
Yih-Dar's avatar
Yih-Dar committed
316
    pytest_num_workers=6,
317
318
319
320
321
322
)


tf_job = CircleCIJob(
    "tf",
    install_steps=[
323
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
Yih-Dar's avatar
Yih-Dar committed
324
325
326
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
        "pip install -U --upgrade-strategy eager tensorflow_probability",
327
    ],
328
    parallelism=1,
329
330
331
332
333
334
335
)


flax_job = CircleCIJob(
    "flax",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
336
337
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece,flax-speech,vision]",
338
    ],
339
    parallelism=1,
340
341
342
343
344
)


pipelines_torch_job = CircleCIJob(
    "pipelines_torch",
345
    additional_env={"RUN_PIPELINE_TESTS": True},
346
347
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
348
349
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
350
    ],
351
    marker="is_pipeline_test",
Yih-Dar's avatar
Yih-Dar committed
352
    pytest_num_workers=6,
353
354
355
356
357
)


pipelines_tf_job = CircleCIJob(
    "pipelines_tf",
358
    additional_env={"RUN_PIPELINE_TESTS": True},
359
    install_steps=[
Sylvain Gugger's avatar
Sylvain Gugger committed
360
        "sudo apt-get -y update && sudo apt-get install -y cmake",
Yih-Dar's avatar
Yih-Dar committed
361
362
363
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,vision]",
        "pip install -U --upgrade-strategy eager tensorflow_probability",
364
    ],
365
    marker="is_pipeline_test",
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
)


custom_tokenizers_job = CircleCIJob(
    "custom_tokenizers",
    additional_env={"RUN_CUSTOM_TOKENIZERS": True},
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y cmake",
        {
            "name": "install jumanpp",
            "command":
                "wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc3/jumanpp-2.0.0-rc3.tar.xz\n"
                "tar xvf jumanpp-2.0.0-rc3.tar.xz\n"
                "mkdir jumanpp-2.0.0-rc3/bld\n"
                "cd jumanpp-2.0.0-rc3/bld\n"
                "sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n"
                "sudo make install\n",
        },
Yih-Dar's avatar
Yih-Dar committed
384
385
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
        "python -m unidic download",
    ],
    parallelism=None,
    resource_class=None,
    tests_to_run=[
        "./tests/models/bert_japanese/test_tokenization_bert_japanese.py",
        "./tests/models/openai/test_tokenization_openai.py",
        "./tests/models/clip/test_tokenization_clip.py",
    ],
)


examples_torch_job = CircleCIJob(
    "examples_torch",
    cache_name="torch_examples",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
Yih-Dar's avatar
Yih-Dar committed
403
404
405
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,torch,sentencepiece,testing,torch-speech]",
        "pip install -U --upgrade-strategy eager -r examples/pytorch/_tests_requirements.txt",
406
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
407
408
409
410
411
412
413
414
    ],
)


examples_tensorflow_job = CircleCIJob(
    "examples_tensorflow",
    cache_name="tensorflow_examples",
    install_steps=[
415
        "sudo apt-get -y update && sudo apt-get install -y cmake",
Yih-Dar's avatar
Yih-Dar committed
416
417
418
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[sklearn,tensorflow,sentencepiece,testing]",
        "pip install -U --upgrade-strategy eager -r examples/tensorflow/_tests_requirements.txt",
419
420
421
422
423
424
425
426
    ],
)


examples_flax_job = CircleCIJob(
    "examples_flax",
    cache_name="flax_examples",
    install_steps=[
Yih-Dar's avatar
Yih-Dar committed
427
428
429
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece]",
        "pip install -U --upgrade-strategy eager -r examples/flax/_tests_requirements.txt",
430
431
432
433
434
435
    ],
)


hub_job = CircleCIJob(
    "hub",
Sylvain Gugger's avatar
Sylvain Gugger committed
436
    additional_env={"HUGGINGFACE_CO_STAGING": True},
437
438
439
440
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install git-lfs",
        'git config --global user.email "ci@dummy.com"',
        'git config --global user.name "ci"',
Yih-Dar's avatar
Yih-Dar committed
441
        "pip install --upgrade --upgrade-strategy eager pip",
Sylvain Gugger's avatar
Sylvain Gugger committed
442
        "pip install -U --upgrade-strategy eager .[torch,sentencepiece,testing,vision]",
443
444
445
446
447
448
449
450
451
    ],
    marker="is_staging_test",
    pytest_num_workers=1,
)


onnx_job = CircleCIJob(
    "onnx",
    install_steps=[
452
        "sudo apt-get -y update && sudo apt-get install -y cmake",
Yih-Dar's avatar
Yih-Dar committed
453
454
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
455
456
457
458
459
460
    ],
    pytest_options={"k onnx": None},
    pytest_num_workers=1,
)


461
462
exotic_models_job = CircleCIJob(
    "exotic_models",
463
464
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
Yih-Dar's avatar
Yih-Dar committed
465
466
467
468
469
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[torch,testing,vision]",
        "pip install -U --upgrade-strategy eager torchvision",
        "pip install -U --upgrade-strategy eager scipy",
        "pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'",
470
        "sudo apt install tesseract-ocr",
Yih-Dar's avatar
Yih-Dar committed
471
472
        "pip install -U --upgrade-strategy eager pytesseract",
        "pip install -U --upgrade-strategy eager natten",
NielsRogge's avatar
NielsRogge committed
473
474
475
        "pip install -U --upgrade-strategy eager python-Levenshtein",
        "pip install -U --upgrade-strategy eager opencv-python",
        "pip install -U --upgrade-strategy eager nltk",
476
477
478
479
    ],
    tests_to_run=[
        "tests/models/*layoutlmv*",
        "tests/models/*nat",
NielsRogge's avatar
NielsRogge committed
480
        "tests/models/deta",
NielsRogge's avatar
NielsRogge committed
481
        "tests/models/nougat",
482
483
484
485
486
487
    ],
    pytest_num_workers=1,
    pytest_options={"durations": 100},
)


Sylvain Gugger's avatar
Sylvain Gugger committed
488
489
490
repo_utils_job = CircleCIJob(
    "repo_utils",
    install_steps=[
Yih-Dar's avatar
Yih-Dar committed
491
492
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager .[quality,testing,torch]",
Sylvain Gugger's avatar
Sylvain Gugger committed
493
494
495
    ],
    parallelism=None,
    pytest_num_workers=1,
496
    resource_class="large",
Sylvain Gugger's avatar
Sylvain Gugger committed
497
498
499
    tests_to_run="tests/repo_utils",
)

500
501
502
503
504

# We also include a `dummy.py` file in the files to be doc-tested to prevent edge case failure. Otherwise, the pytest
# hangs forever during test collection while showing `collecting 0 items / 21 errors`. (To see this, we have to remove
# the bash output redirection.)
py_command = 'from utils.tests_fetcher import get_doctest_files; to_test = get_doctest_files() + ["dummy.py"]; to_test = " ".join(to_test); print(to_test)'
505
py_command = f"$(python3 -c '{py_command}')"
506
command = f'echo "{py_command}" > pr_documentation_tests_temp.txt'
507
508
509
510
doc_test_job = CircleCIJob(
    "pr_documentation_tests",
    additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
    install_steps=[
511
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
Yih-Dar's avatar
Yih-Dar committed
512
513
        "pip install --upgrade --upgrade-strategy eager pip",
        "pip install -U --upgrade-strategy eager -e .[dev]",
514
        "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
Yih-Dar's avatar
Yih-Dar committed
515
516
        "pip install --upgrade --upgrade-strategy eager pytest pytest-sugar",
        "pip install -U --upgrade-strategy eager natten",
517
518
519
520
521
522
        "find -name __pycache__ -delete",
        "find . -name \*.pyc -delete",
        # Add an empty file to keep the test step running correctly even no file is selected to be tested.
        "touch dummy.py",
        {
            "name": "Get files to test",
523
            "command": command,
524
525
        },
        {
526
            "name": "Show information in `Get files to test`",
527
            "command":
528
                "cat pr_documentation_tests_temp.txt"
529
530
        },
        {
531
            "name": "Get the last line in `pr_documentation_tests.txt`",
532
            "command":
533
                "tail -n1 pr_documentation_tests_temp.txt | tee pr_documentation_tests.txt"
534
535
        },
    ],
536
    tests_to_run="$(cat pr_documentation_tests.txt)",  # noqa
537
    pytest_options={"-doctest-modules": None, "doctest-glob": "*.md", "dist": "loadfile", "rvsA": None},
538
539
540
541
    command_timeout=1200,  # test cannot run longer than 1200 seconds
    pytest_num_workers=1,
)

542
543
544
545
546
547
548
549
550
REGULAR_TESTS = [
    torch_and_tf_job,
    torch_and_flax_job,
    torch_job,
    tf_job,
    flax_job,
    custom_tokenizers_job,
    hub_job,
    onnx_job,
551
    exotic_models_job,
552
553
554
555
556
557
558
559
560
561
]
EXAMPLES_TESTS = [
    examples_torch_job,
    examples_tensorflow_job,
    examples_flax_job,
]
PIPELINE_TESTS = [
    pipelines_torch_job,
    pipelines_tf_job,
]
Sylvain Gugger's avatar
Sylvain Gugger committed
562
REPO_UTIL_TESTS = [repo_utils_job]
563
564
DOC_TESTS = [doc_test_job]

565
566
567
568

def create_circleci_config(folder=None):
    if folder is None:
        folder = os.getcwd()
569
570
    # Used in CircleCIJob.to_dict() to expand the test list (for using parallelism)
    os.environ["test_preparation_dir"] = folder
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
    jobs = []
    all_test_file = os.path.join(folder, "test_list.txt")
    if os.path.exists(all_test_file):
        with open(all_test_file) as f:
            all_test_list = f.read()
    else:
        all_test_list = []
    if len(all_test_list) > 0:
        jobs.extend(PIPELINE_TESTS)

    test_file = os.path.join(folder, "filtered_test_list.txt")
    if os.path.exists(test_file):
        with open(test_file) as f:
            test_list = f.read()
    else:
        test_list = []
    if len(test_list) > 0:
        jobs.extend(REGULAR_TESTS)

Yih-Dar's avatar
Yih-Dar committed
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
        extended_tests_to_run = set(test_list.split())
        # Extend the test files for cross test jobs
        for job in jobs:
            if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
                for test_path in copy.copy(extended_tests_to_run):
                    dir_path, fn = os.path.split(test_path)
                    if fn.startswith("test_modeling_tf_"):
                        fn = fn.replace("test_modeling_tf_", "test_modeling_")
                    elif fn.startswith("test_modeling_flax_"):
                        fn = fn.replace("test_modeling_flax_", "test_modeling_")
                    else:
                        if job.job_name == "test_torch_and_tf":
                            fn = fn.replace("test_modeling_", "test_modeling_tf_")
                        elif job.job_name == "test_torch_and_flax":
                            fn = fn.replace("test_modeling_", "test_modeling_flax_")
                    new_test_file = str(os.path.join(dir_path, fn))
                    if os.path.isfile(new_test_file):
                        if new_test_file not in extended_tests_to_run:
                            extended_tests_to_run.add(new_test_file)
        extended_tests_to_run = sorted(extended_tests_to_run)
        for job in jobs:
            if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
                job.tests_to_run = extended_tests_to_run
        fn = "filtered_test_list_cross_tests.txt"
        f_path = os.path.join(folder, fn)
        with open(f_path, "w") as fp:
            fp.write(" ".join(extended_tests_to_run))

618
619
    example_file = os.path.join(folder, "examples_test_list.txt")
    if os.path.exists(example_file) and os.path.getsize(example_file) > 0:
620
        with open(example_file, "r", encoding="utf-8") as f:
621
            example_tests = f.read()
622
623
624
625
626
        for job in EXAMPLES_TESTS:
            framework = job.name.replace("examples_", "").replace("torch", "pytorch")
            if example_tests == "all":
                job.tests_to_run = [f"examples/{framework}"]
            else:
627
                job.tests_to_run = [f for f in example_tests.split(" ") if f.startswith(f"examples/{framework}")]
628

629
630
            if len(job.tests_to_run) > 0:
                jobs.append(job)
631

632
633
634
635
636
637
638
639
640
    doctest_file = os.path.join(folder, "doctest_list.txt")
    if os.path.exists(doctest_file):
        with open(doctest_file) as f:
            doctest_list = f.read()
    else:
        doctest_list = []
    if len(doctest_list) > 0:
        jobs.extend(DOC_TESTS)

Sylvain Gugger's avatar
Sylvain Gugger committed
641
642
643
    repo_util_file = os.path.join(folder, "test_repo_utils.txt")
    if os.path.exists(repo_util_file) and os.path.getsize(repo_util_file) > 0:
        jobs.extend(REPO_UTIL_TESTS)
644

645
646
647
648
649
650
651
652
653
654
655
656
    if len(jobs) == 0:
        jobs = [EmptyJob()]
    config = {"version": "2.1"}
    config["parameters"] = {
        # Only used to accept the parameters from the trigger
        "nightly": {"type": "boolean", "default": False},
        "tests_to_run": {"type": "string", "default": test_list},
    }
    config["jobs"] = {j.job_name: j.to_dict() for j in jobs}
    config["workflows"] = {"version": 2, "run_tests": {"jobs": [j.job_name for j in jobs]}}
    with open(os.path.join(folder, "generated_config.yml"), "w") as f:
        f.write(yaml.dump(config, indent=2, width=1000000, sort_keys=False))
657
658
659
660
661
662
663
664
665
666


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--fetcher_folder", type=str, default=None, help="Only test that all tests and modules are accounted for."
    )
    args = parser.parse_args()

    create_circleci_config(args.fetcher_folder)