config.yml 35.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
version: 2.1
orbs:
    gcp-gke: circleci/gcp-gke@1.0.4
    go: circleci/go@1.3.0

# TPU REFERENCES
references:
    checkout_ml_testing: &checkout_ml_testing
        run:
            name: Checkout ml-testing-accelerators
            command: |
                git clone https://github.com/GoogleCloudPlatform/ml-testing-accelerators.git
                cd ml-testing-accelerators
                git fetch origin 5e88ac24f631c27045e62f0e8d5dfcf34e425e25:stable
                git checkout stable
    build_push_docker: &build_push_docker
        run:
            name: Configure Docker
            command: |
                gcloud --quiet auth configure-docker
                cd docker/transformers-pytorch-tpu
22
                if [ -z "$CIRCLE_PR_NUMBER" ]; then docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" . ; else docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" --build-arg "GITHUB_REF=pull/$CIRCLE_PR_NUMBER/head" . ; fi
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
                docker push "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID"
    deploy_cluster: &deploy_cluster
        run:
            name: Deploy the job on the kubernetes cluster
            command: |
                go get github.com/google/go-jsonnet/cmd/jsonnet && \
                export PATH=$PATH:$HOME/go/bin && \
                kubectl create -f docker/transformers-pytorch-tpu/dataset.yaml || true && \
                job_name=$(jsonnet -J ml-testing-accelerators/ docker/transformers-pytorch-tpu/bert-base-cased.jsonnet --ext-str image=$GCR_IMAGE_PATH --ext-str image-tag=$CIRCLE_WORKFLOW_JOB_ID | kubectl create -f -) && \
                job_name=${job_name#job.batch/} && \
                job_name=${job_name% created} && \
                echo "Waiting on kubernetes job: $job_name" && \
                i=0 && \
                # 30 checks spaced 30s apart = 900s total.
                max_checks=30 && \
                status_code=2 && \
                # Check on the job periodically. Set the status code depending on what
                # happened to the job in Kubernetes. If we try max_checks times and
                # still the job hasn't finished, give up and return the starting
                # non-zero status code.
                while [ $i -lt $max_checks ]; do ((i++)); if kubectl get jobs $job_name -o jsonpath='Failed:{.status.failed}' | grep "Failed:1"; then status_code=1 && break; elif kubectl get jobs $job_name -o jsonpath='Succeeded:{.status.succeeded}' | grep "Succeeded:1" ; then status_code=0 && break; else echo "Job not finished yet"; fi; sleep 30; done && \
                echo "Done waiting. Job status code: $status_code" && \
45
46
47
                pod_name=$(kubectl get po -l controller-uid=`kubectl get job $job_name -o "jsonpath={.metadata.labels.controller-uid}"` | awk 'match($0,!/NAME/) {print $1}') && \
                echo "GKE pod name: $pod_name" && \
                kubectl logs -f $pod_name --container=train
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
                echo "Done with log retrieval attempt." && \
                gcloud container images delete "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" --force-delete-tags && \
                exit $status_code
    delete_gke_jobs: &delete_gke_jobs
        run:
            name: Delete GKE Jobs
            command: |
                # Match jobs whose age matches patterns like '1h' or '1d', i.e. any job
                # that has been around longer than 1hr. First print all columns for
                # matches, then execute the delete.
                kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $0}'
                kubectl delete job $(kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $1}')




Julien Chaumond's avatar
Julien Chaumond committed
64
jobs:
Sylvain Gugger's avatar
Sylvain Gugger committed
65
66
    # Fetch the tests to run
    fetch_tests:
67
        working_directory: ~/transformers
68
        docker:
69
            - image: cimg/python:3.7.12
70
71
72
        parallelism: 1
        steps:
            - checkout
73
            - run: pip install --upgrade pip
Sylvain Gugger's avatar
Sylvain Gugger committed
74
75
76
77
            - run: pip install GitPython
            - run: pip install .
            - run: mkdir -p test_preparation
            - run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
78
            - store_artifacts:
Sylvain Gugger's avatar
Sylvain Gugger committed
79
                path: ~/transformers/tests_fetched_summary.txt
80
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
81
82
83
84
85
                if [ -f test_list.txt ]; then
                    mv test_list.txt test_preparation/test_list.txt
                else
                    touch test_preparation/test_list.txt
                fi
86
87
88
89
90
91
92
93
94
            - run: python utils/tests_fetcher.py --filters tests examples | tee examples_tests_fetched_summary.txt
            - store_artifacts:
                path: ~/transformers/examples_tests_fetched_summary.txt
            - run: |
                if [ -f test_list.txt ]; then
                    mv test_list.txt test_preparation/examples_test_list.txt
                else
                    touch test_preparation/examples_test_list.txt
                fi
Sylvain Gugger's avatar
Sylvain Gugger committed
95
96
97
98
99

            - persist_to_workspace:
                root: test_preparation/
                paths:
                    test_list.txt
100
                    examples_test_list.txt
101

Sylvain Gugger's avatar
Sylvain Gugger committed
102
103
104
105
106
107
108
109
110
111
    # To run all tests for the nightly build
    fetch_all_tests:
        working_directory: ~/transformers
        docker:
            - image: cimg/python:3.7.12
        parallelism: 1
        steps:
            - run: |
                  mkdir test_preparation
                  echo "tests" > test_preparation/test_list.txt
112
                  echo "tests" > test_preparation/examples_test_list.txt
Sylvain Gugger's avatar
Sylvain Gugger committed
113
114
115
116
117
118
119

            - persist_to_workspace:
                  root: test_preparation/
                  paths:
                      test_list.txt

    run_tests_torch_and_tf:
120
121
        working_directory: ~/transformers
        docker:
122
            - image: cimg/python:3.7.12
123
124
125
126
        environment:
            OMP_NUM_THREADS: 1
            RUN_PT_TF_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
127
            PYTEST_TIMEOUT: 120
128
129
130
131
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
132
133
134
135
136
137
138
            - attach_workspace:
                at: ~/transformers/test_preparation
            - run: |
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
139
140
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
141
                      - v0.5-torch_and_tf-{{ checksum "setup.py" }}
142
                      - v0.5-torch_and_tf-
143
144
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs
            - run: git lfs install
145
            - run: pip install --upgrade pip
146
            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]
147
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
Kamal Raj's avatar
Kamal Raj committed
148
            - run: pip install tensorflow_probability
149
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
150
            - run: pip install git+https://github.com/huggingface/accelerate
151
            - save_cache:
152
                key: v0.5-torch_and_tf-{{ checksum "setup.py" }}
153
154
                paths:
                    - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
155
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf $(cat test_preparation/test_list.txt) -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
156
157
158
159
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
160

161
162
163
    run_tests_torch_and_flax:
        working_directory: ~/transformers
        docker:
164
            - image: cimg/python:3.7.12
165
166
        environment:
            OMP_NUM_THREADS: 1
167
168
            RUN_PT_FLAX_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
169
            PYTEST_TIMEOUT: 120
170
171
172
173
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
174
175
            - attach_workspace:
                at: ~/transformers/test_preparation
176
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
177
178
179
180
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
181
182
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
183
                      - v0.5-torch_and_flax-{{ checksum "setup.py" }}
184
                      - v0.5-torch_and_flax-
185
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
186
            - run: pip install --upgrade pip
187
            - run: pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]
188
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
189
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
190
            - run: pip install git+https://github.com/huggingface/accelerate
191
            - save_cache:
192
                key: v0.5-torch_and_flax-{{ checksum "setup.py" }}
193
194
                paths:
                    - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
195
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax $(cat test_preparation/test_list.txt) -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt
196
197
198
199
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
200

Aymeric Augustin's avatar
Aymeric Augustin committed
201
    run_tests_torch:
202
        working_directory: ~/transformers
Julien Chaumond's avatar
Julien Chaumond committed
203
        docker:
204
            - image: cimg/python:3.7.12
205
206
        environment:
            OMP_NUM_THREADS: 1
207
            TRANSFORMERS_IS_CI: yes
208
            PYTEST_TIMEOUT: 120
209
        resource_class: xlarge
210
        parallelism: 1
Julien Chaumond's avatar
Julien Chaumond committed
211
212
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
213
214
            - attach_workspace:
                at: ~/transformers/test_preparation
215
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
216
217
218
219
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
220
221
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
222
                      - v0.5-torch-{{ checksum "setup.py" }}
223
                      - v0.5-torch-
Sylvain Gugger's avatar
Sylvain Gugger committed
224
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time
225
            - run: pip install --upgrade pip
226
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
227
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
228
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
229
            - run: pip install git+https://github.com/huggingface/accelerate
230
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
231
                  key: v0.5-torch-{{ checksum "setup.py" }}
232
233
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
234
            - run: python -m pytest -n 3 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_torch $(cat test_preparation/test_list.txt) | tee tests_output.txt
235
236
237
238
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
Lysandre Debut's avatar
Lysandre Debut committed
239

Aymeric Augustin's avatar
Aymeric Augustin committed
240
    run_tests_tf:
241
        working_directory: ~/transformers
thomwolf's avatar
thomwolf committed
242
        docker:
243
            - image: cimg/python:3.7.12
244
245
        environment:
            OMP_NUM_THREADS: 1
246
            TRANSFORMERS_IS_CI: yes
247
            PYTEST_TIMEOUT: 120
thomwolf's avatar
thomwolf committed
248
249
250
251
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
252
253
            - attach_workspace:
                at: ~/transformers/test_preparation
254
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
255
256
257
258
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
259
260
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
261
                      - v0.5-tf-{{ checksum "setup.py" }}
262
                      - v0.5-tf-
263
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
264
            - run: pip install --upgrade pip
265
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]
Kamal Raj's avatar
Kamal Raj committed
266
            - run: pip install tensorflow_probability
267
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
268
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
269
                  key: v0.5-tf-{{ checksum "setup.py" }}
270
271
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
272
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_tf $(cat test_preparation/test_list.txt) | tee tests_output.txt
273
274
275
276
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
277

278
279
280
    run_tests_flax:
        working_directory: ~/transformers
        docker:
281
            - image: cimg/python:3.7.12
282
283
        environment:
            OMP_NUM_THREADS: 1
284
            TRANSFORMERS_IS_CI: yes
285
            PYTEST_TIMEOUT: 120
286
287
288
289
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
290
291
            - attach_workspace:
                at: ~/transformers/test_preparation
292
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
293
294
295
296
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
297
298
            - restore_cache:
                keys:
Yih-Dar's avatar
Yih-Dar committed
299
                    - v0.5-flax-{{ checksum "setup.py" }}
300
                    - v0.5-flax-
301
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
302
            - run: pip install --upgrade pip
Sylvain Gugger's avatar
Sylvain Gugger committed
303
            - run: pip install .[flax,testing,sentencepiece,flax-speech,vision]
304
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
305
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
306
                  key: v0.5-flax-{{ checksum "setup.py" }}
307
308
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
309
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_flax $(cat test_preparation/test_list.txt) | tee tests_output.txt
310
311
312
313
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
314

315
316
317
    run_tests_pipelines_torch:
        working_directory: ~/transformers
        docker:
318
            - image: cimg/python:3.7.12
319
320
        environment:
            OMP_NUM_THREADS: 1
321
322
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
323
            PYTEST_TIMEOUT: 120
324
325
326
327
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
328
329
            - attach_workspace:
                at: ~/transformers/test_preparation
330
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
331
332
333
334
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
335
336
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
337
                      - v0.5-torch-{{ checksum "setup.py" }}
338
                      - v0.5-torch-
339
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
340
            - run: pip install --upgrade pip
341
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
342
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.12.0+cpu.html
343
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
344
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
345
                  key: v0.5-torch-{{ checksum "setup.py" }}
346
347
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
348
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test $(cat test_preparation/test_list.txt) | tee tests_output.txt
349
350
351
352
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
353

354
355
356
    run_tests_pipelines_tf:
        working_directory: ~/transformers
        docker:
357
            - image: cimg/python:3.7.12
358
359
        environment:
            OMP_NUM_THREADS: 1
360
361
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
362
            PYTEST_TIMEOUT: 120
363
364
365
366
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
367
368
            - attach_workspace:
                at: ~/transformers/test_preparation
369
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
370
371
372
373
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
374
375
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
376
                      - v0.5-tf-{{ checksum "setup.py" }}
377
                      - v0.5-tf-
378
379
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
Kamal Raj's avatar
Kamal Raj committed
380
            - run: pip install tensorflow_probability
381
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
382
                  key: v0.5-tf-{{ checksum "setup.py" }}
383
384
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
385
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf $(cat test_preparation/test_list.txt) -m is_pipeline_test | tee tests_output.txt
386
387
388
389
390
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
391
    run_tests_custom_tokenizers:
392
393
        working_directory: ~/transformers
        docker:
394
            - image: cimg/python:3.7.12
395
396
        environment:
            RUN_CUSTOM_TOKENIZERS: yes
397
            TRANSFORMERS_IS_CI: yes
398
            PYTEST_TIMEOUT: 120
399
400
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
401
402
            - attach_workspace:
                at: ~/transformers/test_preparation
403
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
404
405
406
407
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
408
409
410
            - restore_cache:
                  keys:
                      - v0.5-custom_tokenizers-{{ checksum "setup.py" }}
411
                      - v0.5-custom_tokenizers-
412
413
414
415
416
417
418
419
420
421
            - run: sudo apt-get -y update && sudo apt-get install -y cmake
            - run:
                name: install jumanpp
                command: |
                    wget https://github.com/ku-nlp/jumanpp/releases/download/v2.0.0-rc3/jumanpp-2.0.0-rc3.tar.xz
                    tar xvf jumanpp-2.0.0-rc3.tar.xz
                    mkdir jumanpp-2.0.0-rc3/bld
                    cd jumanpp-2.0.0-rc3/bld
                    sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local
                    sudo make install
422
423
424
425
426
427
428
429
430
431
432
433
434
            - run: pip install --upgrade pip
            - run: pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]
            - run: python -m unidic download
            - save_cache:
                  key: v0.5-custom_tokenizers-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: python -m pytest --max-worker-restart=0 -s --make-reports=tests_custom_tokenizers ./tests/models/bert_japanese/test_tokenization_bert_japanese.py ./tests/models/openai/test_tokenization_openai.py ./tests/models/clip/test_tokenization_clip.py | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
435
    run_examples_torch:
436
437
        working_directory: ~/transformers
        docker:
438
            - image: cimg/python:3.7.12
439
440
        environment:
            OMP_NUM_THREADS: 1
441
            TRANSFORMERS_IS_CI: yes
442
            PYTEST_TIMEOUT: 120
443
444
445
446
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
447
448
            - attach_workspace:
                at: ~/transformers/test_preparation
449
            - run: |
450
                if [ ! -s test_preparation/examples_test_list.txt ]; then
Sylvain Gugger's avatar
Sylvain Gugger committed
451
452
453
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
454
455
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
456
                      - v0.5-torch_examples-{{ checksum "setup.py" }}
457
                      - v0.5-torch_examples-
458
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
459
            - run: pip install --upgrade pip
460
            - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech]
461
462
            - run: pip install -r examples/pytorch/_tests_requirements.txt
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
463
                  key: v0.5-torch_examples-{{ checksum "setup.py" }}
464
465
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
466
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee tests_output.txt
467
468
469
470
            - store_artifacts:
                  path: ~/transformers/examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
471

Matt's avatar
Matt committed
472
473
474
475
476
477
478
479
480
481
482
483
    run_examples_tensorflow:
        working_directory: ~/transformers
        docker:
            - image: cimg/python:3.7.12
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
            PYTEST_TIMEOUT: 120
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
484
485
            - attach_workspace:
                at: ~/transformers/test_preparation
Matt's avatar
Matt committed
486
            - run: |
487
                if [ ! -s test_preparation/examples_test_list.txt ]; then
Sylvain Gugger's avatar
Sylvain Gugger committed
488
489
490
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
Matt's avatar
Matt committed
491
492
493
            - restore_cache:
                  keys:
                      - v0.5-tensorflow_examples-{{ checksum "setup.py" }}
494
                      - v0.5-tensorflow_examples-
Matt's avatar
Matt committed
495
496
497
498
499
500
501
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,tensorflow,sentencepiece,testing]
            - run: pip install -r examples/tensorflow/_tests_requirements.txt
            - save_cache:
                  key: v0.5-tensorflow_examples-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
502
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_tensorflow ./examples/tensorflow/ | tee tests_output.txt
Matt's avatar
Matt committed
503
504
505
506
507
            - store_artifacts:
                  path: ~/transformers/tensorflow_examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Suraj Patil's avatar
Suraj Patil committed
508
509
510
    run_examples_flax:
        working_directory: ~/transformers
        docker:
511
            - image: cimg/python:3.7.12
Suraj Patil's avatar
Suraj Patil committed
512
513
514
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
515
            PYTEST_TIMEOUT: 120
Suraj Patil's avatar
Suraj Patil committed
516
517
518
519
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
520
521
            - attach_workspace:
                at: ~/transformers/test_preparation
Suraj Patil's avatar
Suraj Patil committed
522
            - run: |
523
                if [ ! -s test_preparation/examples_test_list.txt ]; then
Sylvain Gugger's avatar
Sylvain Gugger committed
524
525
526
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
Suraj Patil's avatar
Suraj Patil committed
527
528
            - restore_cache:
                keys:
Yih-Dar's avatar
Yih-Dar committed
529
                    - v0.5-flax_examples-{{ checksum "setup.py" }}
530
                    - v0.5-flax_examples-
Suraj Patil's avatar
Suraj Patil committed
531
            - run: pip install --upgrade pip
532
            - run: pip install .[flax,testing,sentencepiece]
Suraj Patil's avatar
Suraj Patil committed
533
534
            - run: pip install -r examples/flax/_tests_requirements.txt
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
535
                  key: v0.5-flax_examples-{{ checksum "setup.py" }}
Suraj Patil's avatar
Suraj Patil committed
536
537
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
538
            - run: python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile -s --make-reports=examples_flax ./examples/flax/ | tee tests_output.txt
Suraj Patil's avatar
Suraj Patil committed
539
540
541
542
543
            - store_artifacts:
                  path: ~/transformers/flax_examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Sylvain Gugger's avatar
Sylvain Gugger committed
544
    run_tests_hub:
545
546
        working_directory: ~/transformers
        docker:
547
            - image: cimg/python:3.7.12
548
        environment:
Sylvain Gugger's avatar
Sylvain Gugger committed
549
            HUGGINGFACE_CO_STAGING: yes
550
551
            RUN_GIT_LFS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
552
            PYTEST_TIMEOUT: 120
553
554
555
556
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
557
558
            - attach_workspace:
                at: ~/transformers/test_preparation
559
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
560
561
562
563
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
564
565
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
566
                      - v0.5-hub-{{ checksum "setup.py" }}
567
                      - v0.5-hub-
568
            - run: sudo apt-get -y update && sudo apt-get install git-lfs
569
570
571
572
573
574
            - run: |
                git config --global user.email "ci@dummy.com"
                git config --global user.name "ci"
            - run: pip install --upgrade pip
            - run: pip install .[torch,sentencepiece,testing]
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
575
                  key: v0.5-hub-{{ checksum "setup.py" }}
576
577
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
578
            - run: python -m pytest --max-worker-restart=0 -sv --make-reports=tests_hub $(cat test_preparation/test_list.txt) -m is_staging_test | tee tests_output.txt
579
580
581
582
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
583

584
585
586
    run_tests_onnxruntime:
        working_directory: ~/transformers
        docker:
587
            - image: cimg/python:3.7.12
588
589
590
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
591
            PYTEST_TIMEOUT: 120
592
593
594
595
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
596
597
598
599
600
601
602
            - attach_workspace:
                at: ~/transformers/test_preparation
            - run: |
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
603
604
            - restore_cache:
                  keys:
605
606
                      - v0.5-onnx-{{ checksum "setup.py" }}
                      - v0.5-onnx-
607
            - run: pip install --upgrade pip
608
            - run: pip install .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]
609
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
610
                  key: v0.5-onnx-{{ checksum "setup.py" }}
611
612
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
613
            - run: python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s --make-reports=tests_onnx $(cat test_preparation/test_list.txt) -k onnx | tee tests_output.txt
614

615
616
617
618
619
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
620
621
622
    check_code_quality:
        working_directory: ~/transformers
        docker:
623
            - image: cimg/python:3.7.12
Lysandre's avatar
Lysandre committed
624
        resource_class: large
625
626
        environment:
            TRANSFORMERS_IS_CI: yes
627
            PYTEST_TIMEOUT: 120
Aymeric Augustin's avatar
Aymeric Augustin committed
628
629
630
        parallelism: 1
        steps:
            - checkout
631
632
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
633
                      - v0.5-code_quality-{{ checksum "setup.py" }}
634
                      - v0.5-code_quality-
635
            - run: pip install --upgrade pip
636
            - run: pip install .[all,quality]
637
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
638
                  key: v0.5-code_quality-{{ checksum "setup.py" }}
639
640
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
641
            - run: black --check --preview examples tests src utils
642
            - run: isort --check-only examples tests src utils
Sylvain Gugger's avatar
Sylvain Gugger committed
643
            - run: python utils/custom_init_isort.py --check_only
644
            - run: python utils/sort_auto_mappings.py --check_only
645
            - run: flake8 examples tests src utils
646
            - run: doc-builder style src/transformers docs/source --max_len 119 --check_only --path_to_docs docs/source
Sylvain Gugger's avatar
Sylvain Gugger committed
647
            - run: python utils/check_doc_toc.py
648

649
    check_repository_consistency:
R茅mi Louf's avatar
R茅mi Louf committed
650
651
        working_directory: ~/transformers
        docker:
652
            - image: cimg/python:3.7.12
Sylvain Gugger's avatar
Sylvain Gugger committed
653
654
655
        resource_class: large
        environment:
            TRANSFORMERS_IS_CI: yes
656
            PYTEST_TIMEOUT: 120
R茅mi Louf's avatar
R茅mi Louf committed
657
658
659
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
660
661
            - restore_cache:
                  keys:
Yih-Dar's avatar
Yih-Dar committed
662
                      - v0.5-repository_consistency-{{ checksum "setup.py" }}
663
                      - v0.5-repository_consistency-
Sylvain Gugger's avatar
Sylvain Gugger committed
664
665
666
            - run: pip install --upgrade pip
            - run: pip install .[all,quality]
            - save_cache:
Yih-Dar's avatar
Yih-Dar committed
667
                  key: v0.5-repository_consistency-{{ checksum "setup.py" }}
Sylvain Gugger's avatar
Sylvain Gugger committed
668
669
670
671
672
673
674
                  paths:
                      - '~/.cache/pip'
            - run: python utils/check_copies.py
            - run: python utils/check_table.py
            - run: python utils/check_dummies.py
            - run: python utils/check_repo.py
            - run: python utils/check_inits.py
675
            - run: python utils/check_config_docstrings.py
Sylvain Gugger's avatar
Sylvain Gugger committed
676
677
            - run: make deps_table_check_updated
            - run: python utils/tests_fetcher.py --sanity_check
678
            - run: python utils/update_metadata.py --check-only
679

NielsRogge's avatar
NielsRogge committed
680
    run_tests_layoutlmv2_and_v3:
681
682
        working_directory: ~/transformers
        docker:
683
            - image: cimg/python:3.7.12
684
685
686
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
687
            PYTEST_TIMEOUT: 120
688
689
690
691
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
692
693
            - attach_workspace:
                at: ~/transformers/test_preparation
694
            - run: |
Sylvain Gugger's avatar
Sylvain Gugger committed
695
696
697
698
                if [ ! -s test_preparation/test_list.txt ]; then
                    echo "No tests to run, exiting early!"
                    circleci-agent step halt
                fi
699
700
701
            - restore_cache:
                  keys:
                      - v0.5-torch-{{ checksum "setup.py" }}
702
                      - v0.5-torch-
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
            - run: pip install --upgrade pip
            - run: pip install .[torch,testing,vision]
            - run: pip install torchvision
            # The commit `36a65a0907d90ed591479b2ebaa8b61cfa0b4ef0` in `detectron2` break things.
            # See https://github.com/facebookresearch/detectron2/commit/36a65a0907d90ed591479b2ebaa8b61cfa0b4ef0#comments.
            # TODO: Revert this change back once the above issue is fixed.
            - run: python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'
            - run: sudo apt install tesseract-ocr
            - run: pip install pytesseract
            - save_cache:
                  key: v0.5-torch-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: python -m pytest -n 1 --max-worker-restart=0 tests/models/*layoutlmv* --dist=loadfile -s --make-reports=tests_layoutlmv2_and_v3 --durations=100
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

723
724
725
# TPU JOBS
    run_examples_tpu:
        docker:
726
            - image: cimg/python:3.7.12
727
728
        environment:
            OMP_NUM_THREADS: 1
729
            TRANSFORMERS_IS_CI: yes
730
731
732
733
734
735
736
737
738
739
740
741
742
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - go/install
            - *checkout_ml_testing
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - setup_remote_docker
            - *build_push_docker
            - *deploy_cluster
743

744
745
    cleanup-gke-jobs:
        docker:
746
            - image: cimg/python:3.7.12
747
748
749
750
751
752
        steps:
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - *delete_gke_jobs
753

LysandreJik's avatar
LysandreJik committed
754
755
756
757
workflow_filters: &workflow_filters
    filters:
        branches:
            only:
758
                - main
759
workflows:
LysandreJik's avatar
LysandreJik committed
760
761
762
    version: 2
    build_and_test:
        jobs:
Aymeric Augustin's avatar
Aymeric Augustin committed
763
            - check_code_quality
764
            - check_repository_consistency
Sylvain Gugger's avatar
Sylvain Gugger committed
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
            - fetch_tests
            - run_examples_torch:
                requires:
                  - fetch_tests
            - run_examples_tensorflow:
                requires:
                  - fetch_tests
            - run_examples_flax:
                requires:
                  - fetch_tests
            - run_tests_custom_tokenizers:
                requires:
                  - fetch_tests
            - run_tests_torch_and_tf:
                requires:
                  - fetch_tests
            - run_tests_torch_and_flax:
                requires:
                  - fetch_tests
            - run_tests_torch:
                requires:
                  - fetch_tests
            - run_tests_tf:
                requires:
                  - fetch_tests
            - run_tests_flax:
                requires:
                  - fetch_tests
            - run_tests_pipelines_torch:
                requires:
                  - fetch_tests
            - run_tests_pipelines_tf:
                requires:
                  - fetch_tests
            - run_tests_onnxruntime:
                requires:
                  - fetch_tests
            - run_tests_hub:
                requires:
                  - fetch_tests
            - run_tests_layoutlmv2_and_v3:
                requires:
                  - fetch_tests
808
809
810
811
812
813
814
    nightly:
        triggers:
            - schedule:
                cron: "0 0 * * *"
                filters:
                    branches:
                        only:
815
                            - main
816
        jobs:
Sylvain Gugger's avatar
Sylvain Gugger committed
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
            - fetch_all_tests
            - run_examples_torch:
                requires:
                  - fetch_all_tests
            - run_examples_tensorflow:
                requires:
                  - fetch_all_tests
            - run_examples_flax:
                requires:
                  - fetch_all_tests
            - run_tests_custom_tokenizers:
                requires:
                  - fetch_all_tests
            - run_tests_torch_and_tf:
                requires:
                  - fetch_all_tests
            - run_tests_torch_and_flax:
                requires:
                  - fetch_all_tests
            - run_tests_torch:
                requires:
                  - fetch_all_tests
            - run_tests_tf:
                requires:
                  - fetch_all_tests
            - run_tests_flax:
                requires:
                  - fetch_all_tests
            - run_tests_pipelines_torch:
                requires:
                  - fetch_all_tests
            - run_tests_pipelines_tf:
                requires:
                  - fetch_all_tests
            - run_tests_onnxruntime:
                requires:
                  - fetch_all_tests
            - run_tests_hub:
                requires:
                  - fetch_all_tests
            - run_tests_layoutlmv2_and_v3:
                requires:
                  - fetch_all_tests
860

861
862
863
864
865
866
867
868
#    tpu_testing_jobs:
#        triggers:
#            - schedule:
#                # Set to run at the first minute of every hour.
#                cron: "0 8 * * *"
#                filters:
#                    branches:
#                        only:
869
#                            - main
870
871
872
#        jobs:
#            - cleanup-gke-jobs
#            - run_examples_tpu