create_circleci_config.py 23.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# coding=utf-8
# Copyright 2022 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import copy
18
import glob
19
import os
20
import random
21
22
23
24
25
26
from dataclasses import dataclass
from typing import Any, Dict, List, Optional

import yaml


27
28
29
30
31
32
33
34
COMMON_ENV_VARIABLES = {
    "OMP_NUM_THREADS": 1,
    "TRANSFORMERS_IS_CI": True,
    "PYTEST_TIMEOUT": 120,
    "RUN_PIPELINE_TESTS": False,
    "RUN_PT_TF_CROSS_TESTS": False,
    "RUN_PT_FLAX_CROSS_TESTS": False,
}
35
COMMON_PYTEST_OPTIONS = {"max-worker-restart": 0, "dist": "loadfile", "s": None}
36
DEFAULT_DOCKER_IMAGE = [{"image": "cimg/python:3.8.12"}]
37
38


39
40
41
42
43
44
45
46
47
48
49
class EmptyJob:
    job_name = "empty"

    def to_dict(self):
        return {
            "working_directory": "~/transformers",
            "docker": copy.deepcopy(DEFAULT_DOCKER_IMAGE),
            "steps":["checkout"],
        }


50
51
52
53
54
@dataclass
class CircleCIJob:
    name: str
    additional_env: Dict[str, Any] = None
    cache_name: str = None
55
    cache_version: str = "0.6"
56
57
58
59
60
61
62
63
64
    docker_image: List[Dict[str, str]] = None
    install_steps: List[str] = None
    marker: Optional[str] = None
    parallelism: Optional[int] = 1
    pytest_num_workers: int = 8
    pytest_options: Dict[str, Any] = None
    resource_class: Optional[str] = "xlarge"
    tests_to_run: Optional[List[str]] = None
    working_directory: str = "~/transformers"
65
66
    # This should be only used for doctest job!
    command_timeout: Optional[int] = None
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

    def __post_init__(self):
        # Deal with defaults for mutable attributes.
        if self.additional_env is None:
            self.additional_env = {}
        if self.cache_name is None:
            self.cache_name = self.name
        if self.docker_image is None:
            # Let's avoid changing the default list and make a copy.
            self.docker_image = copy.deepcopy(DEFAULT_DOCKER_IMAGE)
        if self.install_steps is None:
            self.install_steps = []
        if self.pytest_options is None:
            self.pytest_options = {}
        if isinstance(self.tests_to_run, str):
            self.tests_to_run = [self.tests_to_run]
83
84
        if self.parallelism is None:
            self.parallelism = 1
85
86

    def to_dict(self):
87
88
        env = COMMON_ENV_VARIABLES.copy()
        env.update(self.additional_env)
89
90
91
        job = {
            "working_directory": self.working_directory,
            "docker": self.docker_image,
92
            "environment": env,
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
        }
        if self.resource_class is not None:
            job["resource_class"] = self.resource_class
        if self.parallelism is not None:
            job["parallelism"] = self.parallelism
        steps = [
            "checkout",
            {"attach_workspace": {"at": "~/transformers/test_preparation"}},
            {
                "restore_cache": {
                    "keys": [
                        f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}',
                        f"v{self.cache_version}-{self.cache_name}-",
                    ]
                }
            },
109
110
111
112
113
114
115
116
            {
                "restore_cache": {
                    "keys": [
                        f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
                        f"v{self.cache_version}-{self.cache_name}-site-packages",
                    ]
                }
            },
117
118
        ]
        steps.extend([{"run": l} for l in self.install_steps])
119
        # TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
120
        steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@e4a419bf6bbaa95d14704cc781d3e81a49cef413"}})
121
122
123
124
125
126
127
128
        steps.append(
            {
                "save_cache": {
                    "key": f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}',
                    "paths": ["~/.cache/pip"],
                }
            }
        )
129
130
131
132
133
134
135
136
        steps.append(
            {
                "save_cache": {
                    "key": f"v{self.cache_version}-{self.cache_name}-" + '{{ checksum "setup.py" }}-site-packages',
                    "paths": ["~/.pyenv/versions/"],
                }
            }
        )
137
138
        steps.append({"run": {"name": "Show installed libraries and their versions", "command": "pip freeze | tee installed.txt"}})
        steps.append({"store_artifacts": {"path": "~/transformers/installed.txt"}})
139
140

        all_options = {**COMMON_PYTEST_OPTIONS, **self.pytest_options}
141
        pytest_flags = [f"--{key}={value}" if (value is not None or key in ["doctest-modules"]) else f"-{key}" for key, value in all_options.items()]
142
143
144
        pytest_flags.append(
            f"--make-reports={self.name}" if "examples" in self.name else f"--make-reports=tests_{self.name}"
        )
145
146
147
148
        test_command = ""
        if self.command_timeout:
            test_command = f"timeout {self.command_timeout} "
        test_command += f"python -m pytest -n {self.pytest_num_workers} " + " ".join(pytest_flags)
amyeroberts's avatar
amyeroberts committed
149

150
151
152
153
154
        if self.parallelism == 1:
            if self.tests_to_run is None:
                test_command += " << pipeline.parameters.tests_to_run >>"
            else:
                test_command += " " + " ".join(self.tests_to_run)
155
        else:
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
            # We need explicit list instead of `pipeline.parameters.tests_to_run` (only available at job runtime)
            tests = self.tests_to_run
            if tests is None:
                folder = os.environ["test_preparation_dir"]
                test_file = os.path.join(folder, "filtered_test_list.txt")
                if os.path.exists(test_file):
                    with open(test_file) as f:
                        tests = f.read().split(" ")

            # expand the test list
            if tests == ["tests"]:
                tests = [os.path.join("tests", x) for x in os.listdir("tests")]
            expanded_tests = []
            for test in tests:
                if test.endswith(".py"):
                    expanded_tests.append(test)
                elif test == "tests/models":
                    expanded_tests.extend([os.path.join(test, x) for x in os.listdir(test)])
                elif test == "tests/pipelines":
                    expanded_tests.extend([os.path.join(test, x) for x in os.listdir(test)])
                else:
                    expanded_tests.append(test)
            # Avoid long tests always being collected together
            random.shuffle(expanded_tests)
            tests = " ".join(expanded_tests)

            # Each executor to run ~10 tests
            n_executors = max(len(tests) // 10, 1)
            # Avoid empty test list on some executor(s) or launching too many executors
            if n_executors > self.parallelism:
                n_executors = self.parallelism
            job["parallelism"] = n_executors

            # Need to be newline separated for the command `circleci tests split` below
            command = f'echo {tests} | tr " " "\\n" >> tests.txt'
            steps.append({"run": {"name": "Get tests", "command": command}})

            command = 'TESTS=$(circleci tests split tests.txt) && echo $TESTS > splitted_tests.txt'
            steps.append({"run": {"name": "Split tests", "command": command}})

            steps.append({"store_artifacts": {"path": "~/transformers/tests.txt"}})
            steps.append({"store_artifacts": {"path": "~/transformers/splitted_tests.txt"}})

199
200
201
202
            test_command = ""
            if self.timeout:
                test_command = f"timeout {self.timeout} "
            test_command += f"python -m pytest -n {self.pytest_num_workers} " + " ".join(pytest_flags)
203
            test_command += " $(cat splitted_tests.txt)"
204
205
        if self.marker is not None:
            test_command += f" -m {self.marker}"
206
207
208
209
210
211
212
213
214
215
216
217

        if self.name == "pr_documentation_tests":
            # can't use ` | tee tee tests_output.txt` as usual
            test_command += " > tests_output.txt"
            # Save the return code, so we can check if it is timeout in the next step.
            test_command += '; touch "$?".txt'
            # Never fail the test step for the doctest job. We will check the results in the next step, and fail that
            # step instead if the actual test failures are found. This is to avoid the timeout being reported as test
            # failure.
            test_command = f"({test_command}) || true"
        else:
            test_command += " | tee tests_output.txt"
218
        steps.append({"run": {"name": "Run tests", "command": test_command}})
219
220
221
222
223
224
225
226
227
228
229

        # return code `124` means the previous (pytest run) step is timeout
        if self.name == "pr_documentation_tests":
            checkout_doctest_command = 'if [ -s reports/tests_pr_documentation_tests/failures_short.txt ]; '
            checkout_doctest_command += 'then echo "some test failed"; '
            checkout_doctest_command += 'cat reports/tests_pr_documentation_tests/failures_short.txt; '
            checkout_doctest_command += 'cat reports/tests_pr_documentation_tests/summary_short.txt; exit -1; '
            checkout_doctest_command += 'elif [ -s reports/tests_pr_documentation_tests/stats.txt ]; then echo "All tests pass!"; '
            checkout_doctest_command += 'elif [ -f 124.txt ]; then echo "doctest timeout!"; else echo "other fatal error)"; exit -1; fi;'
            steps.append({"run": {"name": "Check doctest results", "command": checkout_doctest_command}})

230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
        steps.append({"store_artifacts": {"path": "~/transformers/tests_output.txt"}})
        steps.append({"store_artifacts": {"path": "~/transformers/reports"}})
        job["steps"] = steps
        return job

    @property
    def job_name(self):
        return self.name if "examples" in self.name else f"tests_{self.name}"


# JOBS
torch_and_tf_job = CircleCIJob(
    "torch_and_tf",
    additional_env={"RUN_PT_TF_CROSS_TESTS": True},
    install_steps=[
245
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
246
247
        "git lfs install",
        "pip install --upgrade pip",
248
249
250
        "pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
        "pip install -U tensorflow_probability",
        "pip install -U git+https://github.com/huggingface/accelerate",
251
252
253
254
255
256
257
258
259
260
261
    ],
    marker="is_pt_tf_cross_test",
    pytest_options={"rA": None, "durations": 0},
)


torch_and_flax_job = CircleCIJob(
    "torch_and_flax",
    additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
262
263
264
        "pip install -U --upgrade pip",
        "pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
        "pip install -U git+https://github.com/huggingface/accelerate",
265
266
267
268
269
270
271
272
273
274
275
    ],
    marker="is_pt_flax_cross_test",
    pytest_options={"rA": None, "durations": 0},
)


torch_job = CircleCIJob(
    "torch",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
        "pip install --upgrade pip",
276
277
        "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
        "pip install -U git+https://github.com/huggingface/accelerate",
278
    ],
279
    parallelism=1,
280
281
282
283
284
285
286
    pytest_num_workers=3,
)


tf_job = CircleCIJob(
    "tf",
    install_steps=[
287
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
288
        "pip install --upgrade pip",
289
290
        "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
        "pip install -U tensorflow_probability",
291
    ],
292
    parallelism=1,
Yih-Dar's avatar
Yih-Dar committed
293
    pytest_num_workers=6,
294
295
296
297
298
299
300
301
302
    pytest_options={"rA": None},
)


flax_job = CircleCIJob(
    "flax",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
        "pip install --upgrade pip",
303
        "pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
304
    ],
305
    parallelism=1,
306
307
308
309
310
311
    pytest_options={"rA": None},
)


pipelines_torch_job = CircleCIJob(
    "pipelines_torch",
312
    additional_env={"RUN_PIPELINE_TESTS": True},
313
314
315
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
        "pip install --upgrade pip",
316
        "pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
317
318
    ],
    pytest_options={"rA": None},
319
    marker="is_pipeline_test",
320
321
322
323
324
)


pipelines_tf_job = CircleCIJob(
    "pipelines_tf",
325
    additional_env={"RUN_PIPELINE_TESTS": True},
326
    install_steps=[
Sylvain Gugger's avatar
Sylvain Gugger committed
327
        "sudo apt-get -y update && sudo apt-get install -y cmake",
328
        "pip install --upgrade pip",
329
330
        "pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
        "pip install -U tensorflow_probability",
331
332
    ],
    pytest_options={"rA": None},
333
    marker="is_pipeline_test",
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
)


custom_tokenizers_job = CircleCIJob(
    "custom_tokenizers",
    additional_env={"RUN_CUSTOM_TOKENIZERS": True},
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y cmake",
        {
            "name": "install jumanpp",
            "command":
                "wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc3/jumanpp-2.0.0-rc3.tar.xz\n"
                "tar xvf jumanpp-2.0.0-rc3.tar.xz\n"
                "mkdir jumanpp-2.0.0-rc3/bld\n"
                "cd jumanpp-2.0.0-rc3/bld\n"
                "sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n"
                "sudo make install\n",
        },
        "pip install --upgrade pip",
353
        "pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
        "python -m unidic download",
    ],
    parallelism=None,
    resource_class=None,
    tests_to_run=[
        "./tests/models/bert_japanese/test_tokenization_bert_japanese.py",
        "./tests/models/openai/test_tokenization_openai.py",
        "./tests/models/clip/test_tokenization_clip.py",
    ],
)


examples_torch_job = CircleCIJob(
    "examples_torch",
    cache_name="torch_examples",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
        "pip install --upgrade pip",
372
373
        "pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
        "pip install -U -r examples/pytorch/_tests_requirements.txt",
374
375
376
377
378
379
380
381
    ],
)


examples_tensorflow_job = CircleCIJob(
    "examples_tensorflow",
    cache_name="tensorflow_examples",
    install_steps=[
382
        "sudo apt-get -y update && sudo apt-get install -y cmake",
383
        "pip install --upgrade pip",
384
385
        "pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
        "pip install -U -r examples/tensorflow/_tests_requirements.txt",
386
387
388
389
390
391
392
393
394
    ],
)


examples_flax_job = CircleCIJob(
    "examples_flax",
    cache_name="flax_examples",
    install_steps=[
        "pip install --upgrade pip",
395
396
        "pip install -U .[flax,testing,sentencepiece]",
        "pip install -U -r examples/flax/_tests_requirements.txt",
397
398
399
400
401
402
403
404
405
406
407
    ],
)


hub_job = CircleCIJob(
    "hub",
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install git-lfs",
        'git config --global user.email "ci@dummy.com"',
        'git config --global user.name "ci"',
        "pip install --upgrade pip",
408
        "pip install -U .[torch,sentencepiece,testing]",
409
410
411
412
413
414
415
416
417
    ],
    marker="is_staging_test",
    pytest_num_workers=1,
)


onnx_job = CircleCIJob(
    "onnx",
    install_steps=[
418
        "sudo apt-get -y update && sudo apt-get install -y cmake",
419
        "pip install --upgrade pip",
420
        "pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
421
422
423
424
425
426
    ],
    pytest_options={"k onnx": None},
    pytest_num_workers=1,
)


427
428
exotic_models_job = CircleCIJob(
    "exotic_models",
429
430
431
    install_steps=[
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
        "pip install --upgrade pip",
432
433
434
435
        "pip install -U .[torch,testing,vision]",
        "pip install -U torchvision",
        "pip install -U scipy",
        "pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
436
        "sudo apt install tesseract-ocr",
437
438
        "pip install -U pytesseract",
        "pip install -U natten",
439
440
441
442
    ],
    tests_to_run=[
        "tests/models/*layoutlmv*",
        "tests/models/*nat",
NielsRogge's avatar
NielsRogge committed
443
        "tests/models/deta",
444
445
446
447
448
449
    ],
    pytest_num_workers=1,
    pytest_options={"durations": 100},
)


Sylvain Gugger's avatar
Sylvain Gugger committed
450
451
452
453
repo_utils_job = CircleCIJob(
    "repo_utils",
    install_steps=[
        "pip install --upgrade pip",
454
        "pip install -U .[quality,testing,torch]",
Sylvain Gugger's avatar
Sylvain Gugger committed
455
456
457
    ],
    parallelism=None,
    pytest_num_workers=1,
458
    resource_class="large",
Sylvain Gugger's avatar
Sylvain Gugger committed
459
460
461
    tests_to_run="tests/repo_utils",
)

462
463
464
465
466

# We also include a `dummy.py` file in the files to be doc-tested to prevent edge case failure. Otherwise, the pytest
# hangs forever during test collection while showing `collecting 0 items / 21 errors`. (To see this, we have to remove
# the bash output redirection.)
py_command = 'from utils.tests_fetcher import get_doctest_files; to_test = get_doctest_files() + ["dummy.py"]; to_test = " ".join(to_test); print(to_test)'
467
py_command = f"$(python3 -c '{py_command}')"
468
command = f'echo "{py_command}" > pr_documentation_tests_temp.txt'
469
470
471
472
doc_test_job = CircleCIJob(
    "pr_documentation_tests",
    additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
    install_steps=[
473
        "sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
474
        "pip install --upgrade pip",
475
476
        "pip install -U -e .[dev]",
        "pip install -U git+https://github.com/huggingface/accelerate",
477
        "pip install --upgrade pytest pytest-sugar",
478
        "pip install -U natten",
479
480
481
482
483
484
        "find -name __pycache__ -delete",
        "find . -name \*.pyc -delete",
        # Add an empty file to keep the test step running correctly even no file is selected to be tested.
        "touch dummy.py",
        {
            "name": "Get files to test",
485
            "command": command,
486
487
        },
        {
488
            "name": "Show information in `Get files to test`",
489
            "command":
490
                "cat pr_documentation_tests_temp.txt"
491
492
        },
        {
493
            "name": "Get the last line in `pr_documentation_tests.txt`",
494
            "command":
495
                "tail -n1 pr_documentation_tests_temp.txt | tee pr_documentation_tests.txt"
496
497
        },
    ],
498
    tests_to_run="$(cat pr_documentation_tests.txt)",  # noqa
499
    pytest_options={"-doctest-modules": None, "doctest-glob": "*.md", "dist": "loadfile", "rvsA": None},
500
501
502
503
    command_timeout=1200,  # test cannot run longer than 1200 seconds
    pytest_num_workers=1,
)

504
505
506
507
508
509
510
511
512
REGULAR_TESTS = [
    torch_and_tf_job,
    torch_and_flax_job,
    torch_job,
    tf_job,
    flax_job,
    custom_tokenizers_job,
    hub_job,
    onnx_job,
513
    exotic_models_job,
514
515
516
517
518
519
520
521
522
523
]
EXAMPLES_TESTS = [
    examples_torch_job,
    examples_tensorflow_job,
    examples_flax_job,
]
PIPELINE_TESTS = [
    pipelines_torch_job,
    pipelines_tf_job,
]
Sylvain Gugger's avatar
Sylvain Gugger committed
524
REPO_UTIL_TESTS = [repo_utils_job]
525
526
DOC_TESTS = [doc_test_job]

527
528
529
530

def create_circleci_config(folder=None):
    if folder is None:
        folder = os.getcwd()
531
532
    # Used in CircleCIJob.to_dict() to expand the test list (for using parallelism)
    os.environ["test_preparation_dir"] = folder
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
    jobs = []
    all_test_file = os.path.join(folder, "test_list.txt")
    if os.path.exists(all_test_file):
        with open(all_test_file) as f:
            all_test_list = f.read()
    else:
        all_test_list = []
    if len(all_test_list) > 0:
        jobs.extend(PIPELINE_TESTS)

    test_file = os.path.join(folder, "filtered_test_list.txt")
    if os.path.exists(test_file):
        with open(test_file) as f:
            test_list = f.read()
    else:
        test_list = []
    if len(test_list) > 0:
        jobs.extend(REGULAR_TESTS)

Yih-Dar's avatar
Yih-Dar committed
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
        extended_tests_to_run = set(test_list.split())
        # Extend the test files for cross test jobs
        for job in jobs:
            if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
                for test_path in copy.copy(extended_tests_to_run):
                    dir_path, fn = os.path.split(test_path)
                    if fn.startswith("test_modeling_tf_"):
                        fn = fn.replace("test_modeling_tf_", "test_modeling_")
                    elif fn.startswith("test_modeling_flax_"):
                        fn = fn.replace("test_modeling_flax_", "test_modeling_")
                    else:
                        if job.job_name == "test_torch_and_tf":
                            fn = fn.replace("test_modeling_", "test_modeling_tf_")
                        elif job.job_name == "test_torch_and_flax":
                            fn = fn.replace("test_modeling_", "test_modeling_flax_")
                    new_test_file = str(os.path.join(dir_path, fn))
                    if os.path.isfile(new_test_file):
                        if new_test_file not in extended_tests_to_run:
                            extended_tests_to_run.add(new_test_file)
        extended_tests_to_run = sorted(extended_tests_to_run)
        for job in jobs:
            if job.job_name in ["tests_torch_and_tf", "tests_torch_and_flax"]:
                job.tests_to_run = extended_tests_to_run
        fn = "filtered_test_list_cross_tests.txt"
        f_path = os.path.join(folder, fn)
        with open(f_path, "w") as fp:
            fp.write(" ".join(extended_tests_to_run))

580
581
    example_file = os.path.join(folder, "examples_test_list.txt")
    if os.path.exists(example_file) and os.path.getsize(example_file) > 0:
582
583
584
585
586
587
588
589
590
591
592
        with open(example_file, "r", encoding="utf-8") as f:
            example_tests = f.read().split(" ")
        for job in EXAMPLES_TESTS:
            framework = job.name.replace("examples_", "").replace("torch", "pytorch")
            if example_tests == "all":
                job.tests_to_run = [f"examples/{framework}"]
            else:
                job.tests_to_run = [f for f in example_tests if f.startswith(f"examples/{framework}")]
            
            if len(job.tests_to_run) > 0:
                jobs.append(job)
593

594
595
596
597
598
599
600
601
602
    doctest_file = os.path.join(folder, "doctest_list.txt")
    if os.path.exists(doctest_file):
        with open(doctest_file) as f:
            doctest_list = f.read()
    else:
        doctest_list = []
    if len(doctest_list) > 0:
        jobs.extend(DOC_TESTS)

Sylvain Gugger's avatar
Sylvain Gugger committed
603
604
605
    repo_util_file = os.path.join(folder, "test_repo_utils.txt")
    if os.path.exists(repo_util_file) and os.path.getsize(repo_util_file) > 0:
        jobs.extend(REPO_UTIL_TESTS)
606

607
608
609
610
611
612
613
614
615
616
617
618
    if len(jobs) == 0:
        jobs = [EmptyJob()]
    config = {"version": "2.1"}
    config["parameters"] = {
        # Only used to accept the parameters from the trigger
        "nightly": {"type": "boolean", "default": False},
        "tests_to_run": {"type": "string", "default": test_list},
    }
    config["jobs"] = {j.job_name: j.to_dict() for j in jobs}
    config["workflows"] = {"version": 2, "run_tests": {"jobs": [j.job_name for j in jobs]}}
    with open(os.path.join(folder, "generated_config.yml"), "w") as f:
        f.write(yaml.dump(config, indent=2, width=1000000, sort_keys=False))
619
620
621
622
623
624
625
626
627
628


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--fetcher_folder", type=str, default=None, help="Only test that all tests and modules are accounted for."
    )
    args = parser.parse_args()

    create_circleci_config(args.fetcher_folder)