config.yml 34.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
version: 2.1
orbs:
    gcp-gke: circleci/gcp-gke@1.0.4
    go: circleci/go@1.3.0

# TPU REFERENCES
references:
    checkout_ml_testing: &checkout_ml_testing
        run:
            name: Checkout ml-testing-accelerators
            command: |
                git clone https://github.com/GoogleCloudPlatform/ml-testing-accelerators.git
                cd ml-testing-accelerators
                git fetch origin 5e88ac24f631c27045e62f0e8d5dfcf34e425e25:stable
                git checkout stable
    build_push_docker: &build_push_docker
        run:
            name: Configure Docker
            command: |
                gcloud --quiet auth configure-docker
                cd docker/transformers-pytorch-tpu
22
                if [ -z "$CIRCLE_PR_NUMBER" ]; then docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" . ; else docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" --build-arg "GITHUB_REF=pull/$CIRCLE_PR_NUMBER/head" . ; fi
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
                docker push "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID"
    deploy_cluster: &deploy_cluster
        run:
            name: Deploy the job on the kubernetes cluster
            command: |
                go get github.com/google/go-jsonnet/cmd/jsonnet && \
                export PATH=$PATH:$HOME/go/bin && \
                kubectl create -f docker/transformers-pytorch-tpu/dataset.yaml || true && \
                job_name=$(jsonnet -J ml-testing-accelerators/ docker/transformers-pytorch-tpu/bert-base-cased.jsonnet --ext-str image=$GCR_IMAGE_PATH --ext-str image-tag=$CIRCLE_WORKFLOW_JOB_ID | kubectl create -f -) && \
                job_name=${job_name#job.batch/} && \
                job_name=${job_name% created} && \
                echo "Waiting on kubernetes job: $job_name" && \
                i=0 && \
                # 30 checks spaced 30s apart = 900s total.
                max_checks=30 && \
                status_code=2 && \
                # Check on the job periodically. Set the status code depending on what
                # happened to the job in Kubernetes. If we try max_checks times and
                # still the job hasn't finished, give up and return the starting
                # non-zero status code.
                while [ $i -lt $max_checks ]; do ((i++)); if kubectl get jobs $job_name -o jsonpath='Failed:{.status.failed}' | grep "Failed:1"; then status_code=1 && break; elif kubectl get jobs $job_name -o jsonpath='Succeeded:{.status.succeeded}' | grep "Succeeded:1" ; then status_code=0 && break; else echo "Job not finished yet"; fi; sleep 30; done && \
                echo "Done waiting. Job status code: $status_code" && \
45
46
47
                pod_name=$(kubectl get po -l controller-uid=`kubectl get job $job_name -o "jsonpath={.metadata.labels.controller-uid}"` | awk 'match($0,!/NAME/) {print $1}') && \
                echo "GKE pod name: $pod_name" && \
                kubectl logs -f $pod_name --container=train
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
                echo "Done with log retrieval attempt." && \
                gcloud container images delete "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" --force-delete-tags && \
                exit $status_code
    delete_gke_jobs: &delete_gke_jobs
        run:
            name: Delete GKE Jobs
            command: |
                # Match jobs whose age matches patterns like '1h' or '1d', i.e. any job
                # that has been around longer than 1hr. First print all columns for
                # matches, then execute the delete.
                kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $0}'
                kubectl delete job $(kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $1}')




Julien Chaumond's avatar
Julien Chaumond committed
64
jobs:
Sylvain Gugger's avatar
Sylvain Gugger committed
65
66
    # Fetch the tests to run
    fetch_tests:
67
        working_directory: ~/transformers
68
        docker:
69
            - image: cimg/python:3.7.12
70
71
72
        parallelism: 1
        steps:
            - checkout
73
            - run: pip install --upgrade pip
Sylvain Gugger's avatar
Sylvain Gugger committed
74
75
76
77
            - run: pip install GitPython
            - run: pip install .
            - run: mkdir -p test_preparation
            - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
78
            - store_artifacts:
Sylvain Gugger's avatar
Sylvain Gugger committed
79
                path: ~/transformers/tests_fetched_summary.txt
80
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
81
82
83
84
85
                if [ -f test_list.txt ]; then
                    mv test_list.txt test_preparation/test_list.txt
                else
                    touch test_preparation/test_list.txt
                fi
86
87
88
89
90
91
92
93
94
            - run: python utils/tests_fetcher.py --filters tests examples | tee examples_tests_fetched_summary.txt
            - store_artifacts:
                path: ~/transformers/examples_tests_fetched_summary.txt
            - run: |
                if [ -f test_list.txt ]; then
                    mv test_list.txt test_preparation/examples_test_list.txt
                else
                    touch test_preparation/examples_test_list.txt
                fi
Sylvain Gugger's avatar
Sylvain Gugger committed
95
96
97
98
99

            - persist_to_workspace:
                root: test_preparation/
                paths:
                    test_list.txt
100
                    examples_test_list.txt
101

Sylvain Gugger's avatar
Sylvain Gugger committed
102
103
104
105
106
107
108
109
110
111
    # To run all tests for the nightly build
    fetch_all_tests:
        working_directory: ~/transformers
        docker:
            - image: cimg/python:3.7.12
        parallelism: 1
        steps:
            - run: |
                  mkdir test_preparation
                  echo "tests" > test_preparation/test_list.txt
112
                  echo "tests" > test_preparation/examples_test_list.txt
Sylvain Gugger's avatar
Sylvain Gugger committed
113
114
115
116
117
118
119

            - persist_to_workspace:
                  root: test_preparation/
                  paths:
                      test_list.txt

    run_tests_torch_and_tf:
120
121
        working_directory: ~/transformers
        docker:
122
            - image: cimg/python:3.7.12
123
124
125
126
        environment:
            OMP_NUM_THREADS: 1
            RUN_PT_TF_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
127
            PYTEST_TIMEOUT: 120
128
129
130
131
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
132
133
134
135
136
137
138
            - attach_workspace:
                at: ~/transformers/test_preparation
            - run: |
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
139
140
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
141
                      - v0.5-torch_and_tf-{{ checksum "setup.py" }}
142
                      - v0.5-torch_and_tf-
143
144
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs
            - run: git lfs install
145
            - run: pip install --upgrade pip
146
            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]
147
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
Kamal Raj's avatar
Kamal Raj committed
148
            - run: pip install tensorflow_probability
149
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
150
            - run: pip install git+https://github.com/huggingface/accelerate
151
            - save_cache:
152
                key: v0.5-torch_and_tf-{{ checksum "setup.py" }}
153
154
                paths:
                    - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
155
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf $(cat test_preparation/test_list.txt) -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
156
157
158
159
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
160

161
162
163
    run_tests_torch_and_flax:
        working_directory: ~/transformers
        docker:
164
            - image: cimg/python:3.7.12
165
166
        environment:
            OMP_NUM_THREADS: 1
167
168
            RUN_PT_FLAX_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
169
            PYTEST_TIMEOUT: 120
170
171
172
173
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
174
175
            - attach_workspace:
                at: ~/transformers/test_preparation
176
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
177
178
179
180
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
181
182
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
183
                      - v0.5-torch_and_flax-{{ checksum "setup.py" }}
184
                      - v0.5-torch_and_flax-
185
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
186
            - run: pip install --upgrade pip
187
            - run: pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]
188
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
189
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
190
            - run: pip install git+https://github.com/huggingface/accelerate
191
            - save_cache:
192
                key: v0.5-torch_and_flax-{{ checksum "setup.py" }}
193
194
                paths:
                    - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
195
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax $(cat test_preparation/test_list.txt) -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt
196
197
198
199
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
200

Aymeric Augustin's avatar
Aymeric Augustin committed
201
    run_tests_torch:
202
        working_directory: ~/transformers
Julien Chaumond's avatar
Julien Chaumond committed
203
        docker:
204
            - image: cimg/python:3.7.12
205
206
        environment:
            OMP_NUM_THREADS: 1
207
            TRANSFORMERS_IS_CI: yes
208
            PYTEST_TIMEOUT: 120
209
        resource_class: xlarge
210
        parallelism: 1
Julien Chaumond's avatar
Julien Chaumond committed
211
212
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
213
214
            - attach_workspace:
                at: ~/transformers/test_preparation
215
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
216
217
218
219
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
220
221
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
222
                      - v0.5-torch-{{ checksum "setup.py" }}
223
                      - v0.5-torch-
Sylvain Gugger's avatar
Sylvain Gugger committed
224
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time
225
            - run: pip install --upgrade pip
226
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
227
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
228
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
229
            - run: pip install git+https://github.com/huggingface/accelerate
230
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
231
                  key: v0.5-torch-{{ checksum "setup.py" }}
232
233
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
234
            - run: python -m pytest -n 3 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_torch $(cat test_preparation/test_list.txt) | tee tests_output.txt
235
236
237
238
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
Lysandre Debut's avatar
Lysandre Debut committed
239

Aymeric Augustin's avatar
Aymeric Augustin committed
240
    run_tests_tf:
241
        working_directory: ~/transformers
thomwolf's avatar
thomwolf committed
242
        docker:
243
            - image: cimg/python:3.7.12
244
245
        environment:
            OMP_NUM_THREADS: 1
246
            TRANSFORMERS_IS_CI: yes
247
            PYTEST_TIMEOUT: 120
thomwolf's avatar
thomwolf committed
248
249
250
251
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
252
253
            - attach_workspace:
                at: ~/transformers/test_preparation
254
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
255
256
257
258
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
259
260
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
261
                      - v0.5-tf-{{ checksum "setup.py" }}
262
                      - v0.5-tf-
263
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
264
            - run: pip install --upgrade pip
265
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]
Kamal Raj's avatar
Kamal Raj committed
266
            - run: pip install tensorflow_probability
267
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
268
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
269
                  key: v0.5-tf-{{ checksum "setup.py" }}
270
271
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
272
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_tf $(cat test_preparation/test_list.txt) | tee tests_output.txt
273
274
275
276
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
277

278
279
280
    run_tests_flax:
        working_directory: ~/transformers
        docker:
281
            - image: cimg/python:3.7.12
282
283
        environment:
            OMP_NUM_THREADS: 1
284
            TRANSFORMERS_IS_CI: yes
285
            PYTEST_TIMEOUT: 120
286
287
288
289
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
290
291
            - attach_workspace:
                at: ~/transformers/test_preparation
292
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
293
294
295
296
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
297
298
            - restore_cache:
                keys:
Yih-Dar's avatar
Yih-Dar committed
299
                    - v0.5-flax-{{ checksum "setup.py" }}
300
                    - v0.5-flax-
301
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
302
            - run: pip install --upgrade pip
Sylvain Gugger's avatar
Sylvain Gugger committed
303
            - run: pip install .[flax,testing,sentencepiece,flax-speech,vision]
304
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
305
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
306
                  key: v0.5-flax-{{ checksum "setup.py" }}
307
308
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
309
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_flax $(cat test_preparation/test_list.txt) | tee tests_output.txt
310
311
312
313
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
314

315
316
317
    run_tests_pipelines_torch:
        working_directory: ~/transformers
        docker:
318
            - image: cimg/python:3.7.12
319
320
        environment:
            OMP_NUM_THREADS: 1
321
322
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
323
            PYTEST_TIMEOUT: 120
324
325
326
327
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
328
329
            - attach_workspace:
                at: ~/transformers/test_preparation
330
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
331
332
333
334
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
335
336
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
337
                      - v0.5-torch-{{ checksum "setup.py" }}
338
                      - v0.5-torch-
339
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
340
            - run: pip install --upgrade pip
341
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
342
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
343
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
344
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
345
                  key: v0.5-torch-{{ checksum "setup.py" }}
346
347
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
348
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test $(cat test_preparation/test_list.txt) | tee tests_output.txt
349
350
351
352
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
353

354
355
356
    run_tests_pipelines_tf:
        working_directory: ~/transformers
        docker:
357
            - image: cimg/python:3.7.12
358
359
        environment:
            OMP_NUM_THREADS: 1
360
361
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
362
            PYTEST_TIMEOUT: 120
363
364
365
366
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
367
368
            - attach_workspace:
                at: ~/transformers/test_preparation
369
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
370
371
372
373
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
374
375
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
376
                      - v0.5-tf-{{ checksum "setup.py" }}
377
                      - v0.5-tf-
378
379
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
Kamal Raj's avatar
Kamal Raj committed
380
            - run: pip install tensorflow_probability
381
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
382
                  key: v0.5-tf-{{ checksum "setup.py" }}
383
384
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
385
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf $(cat test_preparation/test_list.txt) -m is_pipeline_test | tee tests_output.txt
386
387
388
389
390
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
391
    run_tests_custom_tokenizers:
392
393
        working_directory: ~/transformers
        docker:
394
            - image: cimg/python:3.7.12
395
396
        environment:
            RUN_CUSTOM_TOKENIZERS: yes
397
            TRANSFORMERS_IS_CI: yes
398
            PYTEST_TIMEOUT: 120
399
400
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
401
402
            - attach_workspace:
                at: ~/transformers/test_preparation
403
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
404
405
406
407
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
408
409
410
            - restore_cache:
                  keys:
                      - v0.5-custom_tokenizers-{{ checksum "setup.py" }}
411
                      - v0.5-custom_tokenizers-
412
413
414
415
416
417
418
419
420
421
422
423
424
            - run: pip install --upgrade pip
            - run: pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]
            - run: python -m unidic download
            - save_cache:
                  key: v0.5-custom_tokenizers-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: python -m pytest --max-worker-restart=0 -s --make-reports=tests_custom_tokenizers ./tests/models/bert_japanese/test_tokenization_bert_japanese.py ./tests/models/openai/test_tokenization_openai.py ./tests/models/clip/test_tokenization_clip.py | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
425
    run_examples_torch:
426
427
        working_directory: ~/transformers
        docker:
428
            - image: cimg/python:3.7.12
429
430
        environment:
            OMP_NUM_THREADS: 1
431
            TRANSFORMERS_IS_CI: yes
432
            PYTEST_TIMEOUT: 120
433
434
435
436
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
437
438
            - attach_workspace:
                at: ~/transformers/test_preparation
439
            - run: |
440
                if [ ! -s test_preparation/examples_test_list.txt ]; then
Sylvain Gugger's avatar
Sylvain Gugger committed
441
442
443
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
444
445
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
446
                      - v0.5-torch_examples-{{ checksum "setup.py" }}
447
                      - v0.5-torch_examples-
448
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
449
            - run: pip install --upgrade pip
450
            - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech]
451
452
            - run: pip install -r examples/pytorch/_tests_requirements.txt
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
453
                  key: v0.5-torch_examples-{{ checksum "setup.py" }}
454
455
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
456
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee tests_output.txt
457
458
459
460
            - store_artifacts:
                  path: ~/transformers/examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
461

Matt's avatar
Matt committed
462
463
464
465
466
467
468
469
470
471
472
473
    run_examples_tensorflow:
        working_directory: ~/transformers
        docker:
            - image: cimg/python:3.7.12
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
            PYTEST_TIMEOUT: 120
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
474
475
            - attach_workspace:
                at: ~/transformers/test_preparation
Matt's avatar
Matt committed
476
            - run: |
477
                if [ ! -s test_preparation/examples_test_list.txt ]; then
Sylvain Gugger's avatar
Sylvain Gugger committed
478
479
480
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
Matt's avatar
Matt committed
481
482
483
            - restore_cache:
                  keys:
                      - v0.5-tensorflow_examples-{{ checksum "setup.py" }}
484
                      - v0.5-tensorflow_examples-
Matt's avatar
Matt committed
485
486
487
488
489
490
491
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,tensorflow,sentencepiece,testing]
            - run: pip install -r examples/tensorflow/_tests_requirements.txt
            - save_cache:
                  key: v0.5-tensorflow_examples-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
492
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_tensorflow ./examples/tensorflow/ | tee tests_output.txt
Matt's avatar
Matt committed
493
494
495
496
497
            - store_artifacts:
                  path: ~/transformers/tensorflow_examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Suraj Patil's avatar
Suraj Patil committed
498
499
500
    run_examples_flax:
        working_directory: ~/transformers
        docker:
501
            - image: cimg/python:3.7.12
Suraj Patil's avatar
Suraj Patil committed
502
503
504
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
505
            PYTEST_TIMEOUT: 120
Suraj Patil's avatar
Suraj Patil committed
506
507
508
509
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
510
511
            - attach_workspace:
                at: ~/transformers/test_preparation
Suraj Patil's avatar
Suraj Patil committed
512
            - run: |
513
                if [ ! -s test_preparation/examples_test_list.txt ]; then
Sylvain Gugger's avatar
Sylvain Gugger committed
514
515
516
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
Suraj Patil's avatar
Suraj Patil committed
517
518
            - restore_cache:
                keys:
Yih-Dar's avatar
Yih-Dar committed
519
                    - v0.5-flax_examples-{{ checksum "setup.py" }}
520
                    - v0.5-flax_examples-
Suraj Patil's avatar
Suraj Patil committed
521
            - run: pip install --upgrade pip
522
            - run: pip install .[flax,testing,sentencepiece]
Suraj Patil's avatar
Suraj Patil committed
523
524
            - run: pip install -r examples/flax/_tests_requirements.txt
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
525
                  key: v0.5-flax_examples-{{ checksum "setup.py" }}
Suraj Patil's avatar
Suraj Patil committed
526
527
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
528
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_flax ./examples/flax/ | tee tests_output.txt
Suraj Patil's avatar
Suraj Patil committed
529
530
531
532
533
            - store_artifacts:
                  path: ~/transformers/flax_examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Sylvain Gugger's avatar
Sylvain Gugger committed
534
    run_tests_hub:
535
536
        working_directory: ~/transformers
        docker:
537
            - image: cimg/python:3.7.12
538
        environment:
Sylvain Gugger's avatar
Sylvain Gugger committed
539
            HUGGINGFACE_CO_STAGING: yes
540
541
            RUN_GIT_LFS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
542
            PYTEST_TIMEOUT: 120
543
544
545
546
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
547
548
            - attach_workspace:
                at: ~/transformers/test_preparation
549
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
550
551
552
553
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
554
555
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
556
                      - v0.5-hub-{{ checksum "setup.py" }}
557
                      - v0.5-hub-
558
            - run: sudo apt-get -y update && sudo apt-get install git-lfs
559
560
561
562
563
564
            - run: |
                git config --global user.email "ci@dummy.com"
                git config --global user.name "ci"
            - run: pip install --upgrade pip
            - run: pip install .[torch,sentencepiece,testing]
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
565
                  key: v0.5-hub-{{ checksum "setup.py" }}
566
567
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
568
            - run: python -m pytest --max-worker-restart=0 -sv --make-reports=tests_hub $(cat test_preparation/test_list.txt) -m is_staging_test | tee tests_output.txt
569
570
571
572
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
573

574
575
576
    run_tests_onnxruntime:
        working_directory: ~/transformers
        docker:
577
            - image: cimg/python:3.7.12
578
579
580
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
581
            PYTEST_TIMEOUT: 120
582
583
584
585
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
586
587
588
589
590
591
592
            - attach_workspace:
                at: ~/transformers/test_preparation
            - run: |
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
593
594
            - restore_cache:
                  keys:
595
596
                      - v0.5-onnx-{{ checksum "setup.py" }}
                      - v0.5-onnx-
597
            - run: pip install --upgrade pip
598
            - run: pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]
599
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
600
                  key: v0.5-onnx-{{ checksum "setup.py" }}
601
602
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
603
            - run: python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_onnx $(cat test_preparation/test_list.txt) -k onnx | tee tests_output.txt
604

605
606
607
608
609
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
610
611
612
    check_code_quality:
        working_directory: ~/transformers
        docker:
613
            - image: cimg/python:3.7.12
Lysandre's avatar
Lysandre committed
614
        resource_class: large
615
616
        environment:
            TRANSFORMERS_IS_CI: yes
617
            PYTEST_TIMEOUT: 120
Aymeric Augustin's avatar
Aymeric Augustin committed
618
619
620
        parallelism: 1
        steps:
            - checkout
621
622
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
623
                      - v0.5-code_quality-{{ checksum "setup.py" }}
624
                      - v0.5-code_quality-
625
            - run: pip install --upgrade pip
626
            - run: pip install .[all,quality]
627
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
628
                  key: v0.5-code_quality-{{ checksum "setup.py" }}
629
630
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
631
            - run: black --check --preview examples tests src utils
632
            - run: isort --check-only examples tests src utils
Sylvain Gugger's avatar
Sylvain Gugger committed
633
            - run: python utils/custom_init_isort.py --check_only
634
            - run: python utils/sort_auto_mappings.py --check_only
635
            - run: flake8 examples tests src utils
636
            - run: doc-builder style src/transformers docs/source --max_len 119 --check_only --path_to_docs docs/source
Sylvain Gugger's avatar
Sylvain Gugger committed
637
            - run: python utils/check_doc_toc.py
638

639
    check_repository_consistency:
R茅mi Louf's avatar
R茅mi Louf committed
640
641
        working_directory: ~/transformers
        docker:
642
            - image: cimg/python:3.7.12
Sylvain Gugger's avatar
Sylvain Gugger committed
643
644
645
        resource_class: large
        environment:
            TRANSFORMERS_IS_CI: yes
646
            PYTEST_TIMEOUT: 120
R茅mi Louf's avatar
R茅mi Louf committed
647
648
649
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
650
651
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
652
                      - v0.5-repository_consistency-{{ checksum "setup.py" }}
653
                      - v0.5-repository_consistency-
Sylvain Gugger's avatar
Sylvain Gugger committed
654
655
656
            - run: pip install --upgrade pip
            - run: pip install .[all,quality]
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
657
                  key: v0.5-repository_consistency-{{ checksum "setup.py" }}
Sylvain Gugger's avatar
Sylvain Gugger committed
658
659
660
661
662
663
664
                  paths:
                      - '~/.cache/pip'
            - run: python utils/check_copies.py
            - run: python utils/check_table.py
            - run: python utils/check_dummies.py
            - run: python utils/check_repo.py
            - run: python utils/check_inits.py
665
            - run: python utils/check_config_docstrings.py
Sylvain Gugger's avatar
Sylvain Gugger committed
666
667
            - run: make deps_table_check_updated
            - run: python utils/tests_fetcher.py --sanity_check
668
            - run: python utils/update_metadata.py --check-only
669

NielsRogge's avatar
NielsRogge committed
670
    run_tests_layoutlmv2_and_v3:
671
672
        working_directory: ~/transformers
        docker:
673
            - image: cimg/python:3.7.12
674
675
676
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
677
            PYTEST_TIMEOUT: 120
678
679
680
681
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
682
683
            - attach_workspace:
                at: ~/transformers/test_preparation
684
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
685
686
687
688
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
689
690
691
            - restore_cache:
                  keys:
                      - v0.5-torch-{{ checksum "setup.py" }}
692
                      - v0.5-torch-
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
            - run: pip install --upgrade pip
            - run: pip install .[torch,testing,vision]
            - run: pip install torchvision
            # The commit `36a65a0907d90ed591479b2ebaa8b61cfa0b4ef0` in `detectron2` break things.
            # See https://github.com/facebookresearch/detectron2/commit/36a65a0907d90ed591479b2ebaa8b61cfa0b4ef0#comments.
            # TODO: Revert this change back once the above issue is fixed.
            - run: python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'
            - run: sudo apt install tesseract-ocr
            - run: pip install pytesseract
            - save_cache:
                  key: v0.5-torch-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: python -m pytest -n 1 --max-worker-restart=0 tests/models/*layoutlmv* --dist=loadfile -s --make-reports=tests_layoutlmv2_and_v3 --durations=100
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

713
714
715
# TPU JOBS
    run_examples_tpu:
        docker:
716
            - image: cimg/python:3.7.12
717
718
        environment:
            OMP_NUM_THREADS: 1
719
            TRANSFORMERS_IS_CI: yes
720
721
722
723
724
725
726
727
728
729
730
731
732
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - go/install
            - *checkout_ml_testing
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - setup_remote_docker
            - *build_push_docker
            - *deploy_cluster
733

734
735
    cleanup-gke-jobs:
        docker:
736
            - image: cimg/python:3.7.12
737
738
739
740
741
742
        steps:
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - *delete_gke_jobs
743

LysandreJik's avatar
LysandreJik committed
744
745
746
747
workflow_filters: &workflow_filters
    filters:
        branches:
            only:
748
                - main
749
workflows:
LysandreJik's avatar
LysandreJik committed
750
751
752
    version: 2
    build_and_test:
        jobs:
Aymeric Augustin's avatar
Aymeric Augustin committed
753
            - check_code_quality
754
            - check_repository_consistency
Sylvain Gugger's avatar
Sylvain Gugger committed
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
            - fetch_tests
            - run_examples_torch:
                requires:
                  - fetch_tests
            - run_examples_tensorflow:
                requires:
                  - fetch_tests
            - run_examples_flax:
                requires:
                  - fetch_tests
            - run_tests_custom_tokenizers:
                requires:
                  - fetch_tests
            - run_tests_torch_and_tf:
                requires:
                  - fetch_tests
            - run_tests_torch_and_flax:
                requires:
                  - fetch_tests
            - run_tests_torch:
                requires:
                  - fetch_tests
            - run_tests_tf:
                requires:
                  - fetch_tests
            - run_tests_flax:
                requires:
                  - fetch_tests
            - run_tests_pipelines_torch:
                requires:
                  - fetch_tests
            - run_tests_pipelines_tf:
                requires:
                  - fetch_tests
            - run_tests_onnxruntime:
                requires:
                  - fetch_tests
            - run_tests_hub:
                requires:
                  - fetch_tests
            - run_tests_layoutlmv2_and_v3:
                requires:
                  - fetch_tests
798
799
800
801
802
803
804
    nightly:
        triggers:
            - schedule:
                cron: "0 0 * * *"
                filters:
                    branches:
                        only:
805
                            - main
806
        jobs:
Sylvain Gugger's avatar
Sylvain Gugger committed
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
            - fetch_all_tests
            - run_examples_torch:
                requires:
                  - fetch_all_tests
            - run_examples_tensorflow:
                requires:
                  - fetch_all_tests
            - run_examples_flax:
                requires:
                  - fetch_all_tests
            - run_tests_custom_tokenizers:
                requires:
                  - fetch_all_tests
            - run_tests_torch_and_tf:
                requires:
                  - fetch_all_tests
            - run_tests_torch_and_flax:
                requires:
                  - fetch_all_tests
            - run_tests_torch:
                requires:
                  - fetch_all_tests
            - run_tests_tf:
                requires:
                  - fetch_all_tests
            - run_tests_flax:
                requires:
                  - fetch_all_tests
            - run_tests_pipelines_torch:
                requires:
                  - fetch_all_tests
            - run_tests_pipelines_tf:
                requires:
                  - fetch_all_tests
            - run_tests_onnxruntime:
                requires:
                  - fetch_all_tests
            - run_tests_hub:
                requires:
                  - fetch_all_tests
            - run_tests_layoutlmv2_and_v3:
                requires:
                  - fetch_all_tests
850

851
852
853
854
855
856
857
858
#    tpu_testing_jobs:
#        triggers:
#            - schedule:
#                # Set to run at the first minute of every hour.
#                cron: "0 8 * * *"
#                filters:
#                    branches:
#                        only:
859
#                            - main
860
861
862
#        jobs:
#            - cleanup-gke-jobs
#            - run_examples_tpu