config.yml 42.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
version: 2.1
orbs:
    gcp-gke: circleci/gcp-gke@1.0.4
    go: circleci/go@1.3.0

# TPU REFERENCES
references:
    checkout_ml_testing: &checkout_ml_testing
        run:
            name: Checkout ml-testing-accelerators
            command: |
                git clone https://github.com/GoogleCloudPlatform/ml-testing-accelerators.git
                cd ml-testing-accelerators
                git fetch origin 5e88ac24f631c27045e62f0e8d5dfcf34e425e25:stable
                git checkout stable
    build_push_docker: &build_push_docker
        run:
            name: Configure Docker
            command: |
                gcloud --quiet auth configure-docker
                cd docker/transformers-pytorch-tpu
22
                if [ -z "$CIRCLE_PR_NUMBER" ]; then docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" . ; else docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" --build-arg "GITHUB_REF=pull/$CIRCLE_PR_NUMBER/head" . ; fi
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
                docker push "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID"
    deploy_cluster: &deploy_cluster
        run:
            name: Deploy the job on the kubernetes cluster
            command: |
                go get github.com/google/go-jsonnet/cmd/jsonnet && \
                export PATH=$PATH:$HOME/go/bin && \
                kubectl create -f docker/transformers-pytorch-tpu/dataset.yaml || true && \
                job_name=$(jsonnet -J ml-testing-accelerators/ docker/transformers-pytorch-tpu/bert-base-cased.jsonnet --ext-str image=$GCR_IMAGE_PATH --ext-str image-tag=$CIRCLE_WORKFLOW_JOB_ID | kubectl create -f -) && \
                job_name=${job_name#job.batch/} && \
                job_name=${job_name% created} && \
                echo "Waiting on kubernetes job: $job_name" && \
                i=0 && \
                # 30 checks spaced 30s apart = 900s total.
                max_checks=30 && \
                status_code=2 && \
                # Check on the job periodically. Set the status code depending on what
                # happened to the job in Kubernetes. If we try max_checks times and
                # still the job hasn't finished, give up and return the starting
                # non-zero status code.
                while [ $i -lt $max_checks ]; do ((i++)); if kubectl get jobs $job_name -o jsonpath='Failed:{.status.failed}' | grep "Failed:1"; then status_code=1 && break; elif kubectl get jobs $job_name -o jsonpath='Succeeded:{.status.succeeded}' | grep "Succeeded:1" ; then status_code=0 && break; else echo "Job not finished yet"; fi; sleep 30; done && \
                echo "Done waiting. Job status code: $status_code" && \
45
46
47
                pod_name=$(kubectl get po -l controller-uid=`kubectl get job $job_name -o "jsonpath={.metadata.labels.controller-uid}"` | awk 'match($0,!/NAME/) {print $1}') && \
                echo "GKE pod name: $pod_name" && \
                kubectl logs -f $pod_name --container=train
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
                echo "Done with log retrieval attempt." && \
                gcloud container images delete "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" --force-delete-tags && \
                exit $status_code
    delete_gke_jobs: &delete_gke_jobs
        run:
            name: Delete GKE Jobs
            command: |
                # Match jobs whose age matches patterns like '1h' or '1d', i.e. any job
                # that has been around longer than 1hr. First print all columns for
                # matches, then execute the delete.
                kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $0}'
                kubectl delete job $(kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $1}')




Julien Chaumond's avatar
Julien Chaumond committed
64
jobs:
Aymeric Augustin's avatar
Aymeric Augustin committed
65
    run_tests_torch_and_tf:
66
        working_directory: ~/transformers
67
        docker:
Kamal Raj's avatar
Kamal Raj committed
68
            - image: circleci/python:3.7
69
70
        environment:
            OMP_NUM_THREADS: 1
71
72
            RUN_PT_TF_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
73
74
75
76
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
77
78
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
79
80
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
81
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
82
            - run: pip install --upgrade pip
83
            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]
84
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
Kamal Raj's avatar
Kamal Raj committed
85
            - run: pip install tensorflow_probability
86
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
87
            - run: pip install git+https://github.com/huggingface/accelerate
88
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
89
                key: v0.4-{{ checksum "setup.py" }}
90
91
                paths:
                    - '~/.cache/pip'
92
93
94
95
96
97
98
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf $(cat test_list.txt) -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
                  fi
99
            - store_artifacts:
100
101
102
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
103

104
105
106
    run_tests_torch_and_tf_all:
        working_directory: ~/transformers
        docker:
107
            - image: circleci/python:3.7
108
109
110
111
112
113
114
115
116
117
118
119
        environment:
            OMP_NUM_THREADS: 1
            RUN_PT_TF_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
120
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
121
            - run: pip install --upgrade pip
122
            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]
123
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
Kamal Raj's avatar
Kamal Raj committed
124
            - run: pip install tensorflow_probability
125
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
126
            - run: pip install git+https://github.com/huggingface/accelerate
127
128
129
130
131
132
133
134
135
136
            - save_cache:
                key: v0.4-{{ checksum "setup.py" }}
                paths:
                    - '~/.cache/pip'
            - run: |
                  python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf tests -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
137

138
139
140
    run_tests_torch_and_flax:
        working_directory: ~/transformers
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
141
            - image: circleci/python:3.7
142
143
        environment:
            OMP_NUM_THREADS: 1
144
145
            RUN_PT_FLAX_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
146
147
148
149
150
151
152
153
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch_and_flax-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
154
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
155
            - run: pip install --upgrade pip
156
            - run: pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]
157
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
158
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
159
            - run: pip install git+https://github.com/huggingface/accelerate
160
161
162
163
            - save_cache:
                key: v0.4-{{ checksum "setup.py" }}
                paths:
                    - '~/.cache/pip'
164
165
166
167
168
169
170
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax $(cat test_list.txt) -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt
                  fi
171
172
173
174
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
175

176
177
178
    run_tests_torch_and_flax_all:
        working_directory: ~/transformers
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
179
            - image: circleci/python:3.7
180
181
182
183
184
185
186
187
188
189
190
191
        environment:
            OMP_NUM_THREADS: 1
            RUN_PT_FLAX_CROSS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch_and_flax-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
192
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
193
            - run: pip install --upgrade pip
194
            - run: pip install .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]
195
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
196
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
197
            - run: pip install git+https://github.com/huggingface/accelerate
198
199
200
201
202
203
204
205
206
207
            - save_cache:
                key: v0.4-{{ checksum "setup.py" }}
                paths:
                    - '~/.cache/pip'
            - run: |
                  python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax tests -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
208

Aymeric Augustin's avatar
Aymeric Augustin committed
209
    run_tests_torch:
210
        working_directory: ~/transformers
Julien Chaumond's avatar
Julien Chaumond committed
211
        docker:
212
            - image: circleci/python:3.7
213
214
        environment:
            OMP_NUM_THREADS: 1
215
            TRANSFORMERS_IS_CI: yes
216
        resource_class: xlarge
217
        parallelism: 1
Julien Chaumond's avatar
Julien Chaumond committed
218
219
        steps:
            - checkout
220
221
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
222
223
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
224
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time
225
            - run: pip install --upgrade pip
226
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
227
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
228
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
229
            - run: pip install git+https://github.com/huggingface/accelerate
230
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
231
                  key: v0.4-torch-{{ checksum "setup.py" }}
232
233
                  paths:
                      - '~/.cache/pip'
234
235
236
237
238
239
240
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 3 --dist=loadfile -s --make-reports=tests_torch $(cat test_list.txt) | tee tests_output.txt
                  fi
241
            - store_artifacts:
242
243
244
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
245

246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
    run_tests_torch_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
261
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
262
            - run: pip install --upgrade pip
263
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
264
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
265
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
266
            - run: pip install git+https://github.com/huggingface/accelerate
267
268
269
270
271
272
273
274
275
276
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  python -m pytest -n 3 --dist=loadfile -s --make-reports=tests_torch tests | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
Lysandre Debut's avatar
Lysandre Debut committed
277

Aymeric Augustin's avatar
Aymeric Augustin committed
278
    run_tests_tf:
279
        working_directory: ~/transformers
thomwolf's avatar
thomwolf committed
280
        docker:
281
            - image: circleci/python:3.7
282
283
        environment:
            OMP_NUM_THREADS: 1
284
            TRANSFORMERS_IS_CI: yes
thomwolf's avatar
thomwolf committed
285
286
287
288
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
289
290
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
291
292
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
293
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
294
            - run: pip install --upgrade pip
295
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]
Kamal Raj's avatar
Kamal Raj committed
296
            - run: pip install tensorflow_probability
297
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
298
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
299
                  key: v0.4-tf-{{ checksum "setup.py" }}
300
301
                  paths:
                      - '~/.cache/pip'
302
303
304
305
306
307
308
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf $(cat test_list.txt) | tee tests_output.txt
                  fi
309
            - store_artifacts:
310
311
312
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
313

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
    run_tests_tf_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
329
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
330
            - run: pip install --upgrade pip
331
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]
Kamal Raj's avatar
Kamal Raj committed
332
            - run: pip install tensorflow_probability
333
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
334
335
336
337
338
339
340
341
342
343
            - save_cache:
                  key: v0.4-tf-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf tests | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
344

345
346
347
348
349
350
    run_tests_flax:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
351
            TRANSFORMERS_IS_CI: yes
352
353
354
355
356
357
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                keys:
Lysandre Debut's avatar
Lysandre Debut committed
358
359
                    - v0.4-flax-{{ checksum "setup.py" }}
                    - v0.4-{{ checksum "setup.py" }}
360
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
361
            - run: pip install --upgrade pip
362
363
            - run: pip install .[flax,testing,sentencepiece,flax-speech,vision]
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
364
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
365
                  key: v0.4-flax-{{ checksum "setup.py" }}
366
367
                  paths:
                      - '~/.cache/pip'
368
369
370
371
372
373
374
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax $(cat test_list.txt) | tee tests_output.txt
                  fi
375
            - store_artifacts:
376
377
378
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
379

380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
    run_tests_flax_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                keys:
                    - v0.4-flax-{{ checksum "setup.py" }}
                    - v0.4-{{ checksum "setup.py" }}
395
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
396
            - run: pip install --upgrade pip
397
398
            - run: pip install .[flax,testing,sentencepiece,vision,flax-speech]
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
399
400
401
402
403
404
405
406
407
408
            - save_cache:
                  key: v0.4-flax-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax tests | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
409

410
411
412
413
414
415
    run_tests_pipelines_torch:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
416
417
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
418
419
420
421
422
423
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
424
425
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
426
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
427
            - run: pip install --upgrade pip
428
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
429
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
430
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
431
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
432
                  key: v0.4-torch-{{ checksum "setup.py" }}
433
434
                  paths:
                      - '~/.cache/pip'
435
436
437
438
439
440
441
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test $(cat test_list.txt) | tee tests_output.txt
                  fi
442
            - store_artifacts:
443
444
445
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
446

447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
    run_tests_pipelines_torch_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
463
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
464
            - run: pip install --upgrade pip
465
            - run: pip install .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]
466
            - run: pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.11.0+cpu.html
467
            - run: pip install https://github.com/kpu/kenlm/archive/master.zip
468
469
470
471
472
473
474
475
476
477
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test tests | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
478

479
480
481
482
483
484
    run_tests_pipelines_tf:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
485
486
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
487
488
489
490
491
492
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
493
494
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
495
            - run: pip install --upgrade pip
496
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
Kamal Raj's avatar
Kamal Raj committed
497
            - run: pip install tensorflow_probability
498
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
499
                  key: v0.4-tf-{{ checksum "setup.py" }}
500
501
                  paths:
                      - '~/.cache/pip'
502
503
504
505
506
507
508
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf $(cat test_list.txt) -m is_pipeline_test | tee tests_output.txt
                  fi
509
510
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
511
            - store_artifacts:
512
513
                  path: ~/transformers/reports

514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
    run_tests_pipelines_tf_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            RUN_PIPELINE_TESTS: yes
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
Kamal Raj's avatar
Kamal Raj committed
532
            - run: pip install tensorflow_probability
533
534
535
536
537
            - save_cache:
                  key: v0.4-tf-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
538
                  python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf tests -m is_pipeline_test | tee tests_output.txt
539
540
541
542
543
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
544
    run_tests_custom_tokenizers:
545
546
        working_directory: ~/transformers
        docker:
547
            - image: circleci/python:3.7
548
549
        environment:
            RUN_CUSTOM_TOKENIZERS: yes
550
            TRANSFORMERS_IS_CI: yes
551
552
        steps:
            - checkout
553
554
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
555
556
                      - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
557
            - run: pip install --upgrade pip
558
            - run: pip install .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]
559
            - run: python -m unidic download
560
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
561
                  key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
562
563
                  paths:
                      - '~/.cache/pip'
564
565
            - run: |
                  if [ -f test_list.txt ]; then
566
                    python -m pytest -s --make-reports=tests_custom_tokenizers ./tests/test_tokenization_bert_japanese.py ./tests/test_tokenization_openai.py | tee tests_output.txt
567
                  fi
568
569
570
571
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 1 tests/test_tokenization_clip.py --dist=loadfile -s --make-reports=tests_tokenization_clip --durations=100 | tee tests_output.txt
                  fi
572
573
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
574
            - store_artifacts:
575
576
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
577
    run_examples_torch:
578
579
        working_directory: ~/transformers
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
580
            - image: circleci/python:3.7
581
582
        environment:
            OMP_NUM_THREADS: 1
583
            TRANSFORMERS_IS_CI: yes
584
585
586
587
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
588
589
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
590
591
                      - v0.4-torch_examples-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
592
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
593
            - run: pip install --upgrade pip
594
            - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech]
Sylvain Gugger's avatar
Sylvain Gugger committed
595
            - run: pip install -r examples/pytorch/_tests_requirements.txt
596
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
597
                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
598
599
                  paths:
                      - '~/.cache/pip'
600
            - run: python utils/tests_fetcher.py --filters examples tests | tee test_preparation.txt
601
602
603
604
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
605
                    python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee tests_output.txt
606
                  fi
607
            - store_artifacts:
608
609
610
                  path: ~/transformers/examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
611

612
613
614
    run_examples_torch_all:
        working_directory: ~/transformers
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
615
            - image: circleci/python:3.7
616
617
618
619
620
621
622
623
624
625
626
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch_examples-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
627
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng
628
            - run: pip install --upgrade pip
629
            - run: pip install .[sklearn,torch,sentencepiece,testing,torch-speech]
630
631
632
633
634
635
636
637
638
639
640
            - run: pip install -r examples/pytorch/_tests_requirements.txt
            - save_cache:
                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  TRANSFORMERS_IS_CI=1 python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/pytorch/ | tee examples_output.txt
            - store_artifacts:
                  path: ~/transformers/examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
641

Suraj Patil's avatar
Suraj Patil committed
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
    run_examples_flax:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                keys:
                    - v0.4-flax_examples-{{ checksum "setup.py" }}
                    - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: sudo pip install .[flax,testing,sentencepiece]
            - run: pip install -r examples/flax/_tests_requirements.txt
            - save_cache:
                  key: v0.4-flax_examples-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: python utils/tests_fetcher.py --filters examples tests | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
                    python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_flax ./examples/flax/ | tee tests_output.txt
                  fi
            - store_artifacts:
                  path: ~/transformers/flax_examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
675

Suraj Patil's avatar
Suraj Patil committed
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
    run_examples_flax_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                keys:
                    - v0.4-flax_examples-{{ checksum "setup.py" }}
                    - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: sudo pip install .[flax,testing,sentencepiece]
            - run: pip install -r examples/flax/_tests_requirements.txt
            - save_cache:
                  key: v0.4-flax_examples-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  TRANSFORMERS_IS_CI=1 python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_flax ./examples/flax/ | tee examples_output.txt
            - store_artifacts:
                  path: ~/transformers/flax_examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Sylvain Gugger's avatar
Sylvain Gugger committed
705
    run_tests_hub:
706
707
708
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
709
        environment:
Sylvain Gugger's avatar
Sylvain Gugger committed
710
            HUGGINGFACE_CO_STAGING: yes
711
712
            RUN_GIT_LFS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
713
714
715
716
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
717
718
719
720
            - restore_cache:
                  keys:
                      - v0.4-hub-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
721
722
723
724
725
            - run: sudo apt-get install git-lfs
            - run: |
                git config --global user.email "ci@dummy.com"
                git config --global user.name "ci"
            - run: pip install --upgrade pip
Sylvain Gugger's avatar
Sylvain Gugger committed
726
727
728
729
730
            - run: pip install .[torch,sentencepiece,testing]
            - save_cache:
                  key: v0.4-hub-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
731
732
733
734
735
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
736
                    python -m pytest -sv --make-reports=tests_hub $(cat test_list.txt) -m is_staging_test | tee tests_output.txt
737
                  fi
738
739
740
741
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
742

743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
    run_tests_hub_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            HUGGINGFACE_CO_STAGING: yes
            RUN_GIT_LFS_TESTS: yes
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-hub-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: sudo apt-get install git-lfs
            - run: |
                git config --global user.email "ci@dummy.com"
                git config --global user.name "ci"
            - run: pip install --upgrade pip
            - run: pip install .[torch,sentencepiece,testing]
            - save_cache:
                  key: v0.4-hub-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  python -m pytest -sv --make-reports=tests_hub tests -m is_staging_test | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
775

776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
    run_tests_onnxruntime:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
792
            - run: pip install .[torch,testing,sentencepiece,onnxruntime,vision,rjieba]
793
794
795
796
            - save_cache:
                  key: v0.4-onnx-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
797
798
799
800
801
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
802
                    python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_onnx $(cat test_list.txt) -k onnx | tee tests_output.txt
803
                  fi
804
805
806
807
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
808

809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
    run_tests_onnxruntime_all:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
lewtun's avatar
lewtun committed
825
            - run: pip install .[torch,testing,sentencepiece,onnxruntime,vision]
826
827
828
829
830
831
832
833
834
835
836
            - save_cache:
                  key: v0.4-onnx-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: |
                  python -m pytest -n 1 --dist=loadfile -s --make-reports=tests_onnx tests -k onnx | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
837
838
839
    check_code_quality:
        working_directory: ~/transformers
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
840
            - image: circleci/python:3.7
Lysandre's avatar
Lysandre committed
841
        resource_class: large
842
843
        environment:
            TRANSFORMERS_IS_CI: yes
Aymeric Augustin's avatar
Aymeric Augustin committed
844
845
846
        parallelism: 1
        steps:
            - checkout
847
848
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
849
850
                      - v0.4-code_quality-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
851
            - run: pip install --upgrade pip
852
            - run: pip install .[all,quality]
853
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
854
                  key: v0.4-code_quality-{{ checksum "setup.py" }}
855
856
                  paths:
                      - '~/.cache/pip'
Sylvain Gugger's avatar
Sylvain Gugger committed
857
            - run: black --check --preview examples tests src utils
858
            - run: isort --check-only examples tests src utils
Sylvain Gugger's avatar
Sylvain Gugger committed
859
            - run: python utils/custom_init_isort.py --check_only
860
            - run: python utils/sort_auto_mappings.py --check_only
861
            - run: flake8 examples tests src utils
862
            - run: doc-builder style src/transformers docs/source --max_len 119 --check_only --path_to_docs docs/source
863

864
    check_repository_consistency:
R茅mi Louf's avatar
R茅mi Louf committed
865
866
        working_directory: ~/transformers
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
867
            - image: circleci/python:3.7
Sylvain Gugger's avatar
Sylvain Gugger committed
868
869
870
        resource_class: large
        environment:
            TRANSFORMERS_IS_CI: yes
R茅mi Louf's avatar
R茅mi Louf committed
871
872
873
        parallelism: 1
        steps:
            - checkout
Sylvain Gugger's avatar
Sylvain Gugger committed
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
            - restore_cache:
                  keys:
                      - v0.4-repository_consistency-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: pip install --upgrade pip
            - run: pip install .[all,quality]
            - save_cache:
                  key: v0.4-repository_consistency-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: python utils/check_copies.py
            - run: python utils/check_table.py
            - run: python utils/check_dummies.py
            - run: python utils/check_repo.py
            - run: python utils/check_inits.py
889
            - run: python utils/check_config_docstrings.py
Sylvain Gugger's avatar
Sylvain Gugger committed
890
891
            - run: make deps_table_check_updated
            - run: python utils/tests_fetcher.py --sanity_check
892

893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
    run_tests_layoutlmv2:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
            TRANSFORMERS_IS_CI: yes
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
            - run: pip install --upgrade pip
            - run: pip install .[torch,testing,vision]
            - run: pip install torchvision
            - run: python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'
            - run: sudo apt install tesseract-ocr
            - run: pip install pytesseract
            - save_cache:
                  key: v0.4-torch-{{ checksum "setup.py" }}
                  paths:
                      - '~/.cache/pip'
            - run: python utils/tests_fetcher.py | tee test_preparation.txt
            - store_artifacts:
                  path: ~/transformers/test_preparation.txt
            - run: |
                  if [ -f test_list.txt ]; then
Yih-Dar's avatar
Yih-Dar committed
924
                    python -m pytest -n 1 tests/models/*layoutlmv2* --dist=loadfile -s --make-reports=tests_layoutlmv2 --durations=100
925
926
927
928
929
930
                  fi
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

931
932
933
# TPU JOBS
    run_examples_tpu:
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
934
            - image: circleci/python:3.7
935
936
        environment:
            OMP_NUM_THREADS: 1
937
            TRANSFORMERS_IS_CI: yes
938
939
940
941
942
943
944
945
946
947
948
949
950
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - go/install
            - *checkout_ml_testing
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - setup_remote_docker
            - *build_push_docker
            - *deploy_cluster
951

952
953
    cleanup-gke-jobs:
        docker:
Sylvain Gugger's avatar
Sylvain Gugger committed
954
            - image: circleci/python:3.7
955
956
957
958
959
960
        steps:
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - *delete_gke_jobs
961

LysandreJik's avatar
LysandreJik committed
962
963
964
965
workflow_filters: &workflow_filters
    filters:
        branches:
            only:
966
                - main
967
workflows:
LysandreJik's avatar
LysandreJik committed
968
969
970
    version: 2
    build_and_test:
        jobs:
Aymeric Augustin's avatar
Aymeric Augustin committed
971
            - check_code_quality
972
            - check_repository_consistency
Aymeric Augustin's avatar
Aymeric Augustin committed
973
            - run_examples_torch
974
            - run_examples_flax
Aymeric Augustin's avatar
Aymeric Augustin committed
975
976
            - run_tests_custom_tokenizers
            - run_tests_torch_and_tf
977
            - run_tests_torch_and_flax
Aymeric Augustin's avatar
Aymeric Augustin committed
978
979
            - run_tests_torch
            - run_tests_tf
980
            - run_tests_flax
981
982
            - run_tests_pipelines_torch
            - run_tests_pipelines_tf
983
            - run_tests_onnxruntime
Sylvain Gugger's avatar
Sylvain Gugger committed
984
            - run_tests_hub
985
            - run_tests_layoutlmv2
986
987
988
989
990
991
992
    nightly:
        triggers:
            - schedule:
                cron: "0 0 * * *"
                filters:
                    branches:
                        only:
993
                            - main
994
        jobs:
995
            - run_examples_torch_all
996
            - run_examples_flax_all
997
998
999
1000
1001
1002
1003
1004
1005
1006
            - run_tests_torch_and_tf_all
            - run_tests_torch_and_flax_all
            - run_tests_torch_all
            - run_tests_tf_all
            - run_tests_flax_all
            - run_tests_pipelines_torch_all
            - run_tests_pipelines_tf_all
            - run_tests_onnxruntime_all
            - run_tests_hub_all

1007
1008
1009
1010
1011
1012
1013
1014
#    tpu_testing_jobs:
#        triggers:
#            - schedule:
#                # Set to run at the first minute of every hour.
#                cron: "0 8 * * *"
#                filters:
#                    branches:
#                        only:
1015
#                            - main
1016
1017
1018
#        jobs:
#            - cleanup-gke-jobs
#            - run_examples_tpu