config.yml 18.8 KB
Newer Older
1
2
3
4
5
version: 2.1
orbs:
    gcp-gke: circleci/gcp-gke@1.0.4
    go: circleci/go@1.3.0

6

7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# TPU REFERENCES
references:
    checkout_ml_testing: &checkout_ml_testing
        run:
            name: Checkout ml-testing-accelerators
            command: |
                git clone https://github.com/GoogleCloudPlatform/ml-testing-accelerators.git
                cd ml-testing-accelerators
                git fetch origin 5e88ac24f631c27045e62f0e8d5dfcf34e425e25:stable
                git checkout stable
    build_push_docker: &build_push_docker
        run:
            name: Configure Docker
            command: |
                gcloud --quiet auth configure-docker
                cd docker/transformers-pytorch-tpu
23
                if [ -z "$CIRCLE_PR_NUMBER" ]; then docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" . ; else docker build --tag "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" -f Dockerfile --build-arg "TEST_IMAGE=1" --build-arg "GITHUB_REF=pull/$CIRCLE_PR_NUMBER/head" . ; fi
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
                docker push "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID"
    deploy_cluster: &deploy_cluster
        run:
            name: Deploy the job on the kubernetes cluster
            command: |
                go get github.com/google/go-jsonnet/cmd/jsonnet && \
                export PATH=$PATH:$HOME/go/bin && \
                kubectl create -f docker/transformers-pytorch-tpu/dataset.yaml || true && \
                job_name=$(jsonnet -J ml-testing-accelerators/ docker/transformers-pytorch-tpu/bert-base-cased.jsonnet --ext-str image=$GCR_IMAGE_PATH --ext-str image-tag=$CIRCLE_WORKFLOW_JOB_ID | kubectl create -f -) && \
                job_name=${job_name#job.batch/} && \
                job_name=${job_name% created} && \
                echo "Waiting on kubernetes job: $job_name" && \
                i=0 && \
                # 30 checks spaced 30s apart = 900s total.
                max_checks=30 && \
                status_code=2 && \
                # Check on the job periodically. Set the status code depending on what
                # happened to the job in Kubernetes. If we try max_checks times and
                # still the job hasn't finished, give up and return the starting
                # non-zero status code.
                while [ $i -lt $max_checks ]; do ((i++)); if kubectl get jobs $job_name -o jsonpath='Failed:{.status.failed}' | grep "Failed:1"; then status_code=1 && break; elif kubectl get jobs $job_name -o jsonpath='Succeeded:{.status.succeeded}' | grep "Succeeded:1" ; then status_code=0 && break; else echo "Job not finished yet"; fi; sleep 30; done && \
                echo "Done waiting. Job status code: $status_code" && \
46
47
48
                pod_name=$(kubectl get po -l controller-uid=`kubectl get job $job_name -o "jsonpath={.metadata.labels.controller-uid}"` | awk 'match($0,!/NAME/) {print $1}') && \
                echo "GKE pod name: $pod_name" && \
                kubectl logs -f $pod_name --container=train
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
                echo "Done with log retrieval attempt." && \
                gcloud container images delete "$GCR_IMAGE_PATH:$CIRCLE_WORKFLOW_JOB_ID" --force-delete-tags && \
                exit $status_code
    delete_gke_jobs: &delete_gke_jobs
        run:
            name: Delete GKE Jobs
            command: |
                # Match jobs whose age matches patterns like '1h' or '1d', i.e. any job
                # that has been around longer than 1hr. First print all columns for
                # matches, then execute the delete.
                kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $0}'
                kubectl delete job $(kubectl get job | awk 'match($4,/[0-9]+[dh]/) {print $1}')




Julien Chaumond's avatar
Julien Chaumond committed
65
jobs:
Aymeric Augustin's avatar
Aymeric Augustin committed
66
    run_tests_torch_and_tf:
67
        working_directory: ~/transformers
68
        docker:
69
            - image: circleci/python:3.6
70
71
        environment:
            OMP_NUM_THREADS: 1
72
73
74
75
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
76
77
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
78
79
                      - v0.4-torch_and_tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
Suraj Patil's avatar
Suraj Patil committed
80
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
81
            - run: pip install --upgrade pip
Suraj Patil's avatar
Suraj Patil committed
82
            - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece,speech]
83
            - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.0+cpu.html
84
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
85
                key: v0.4-{{ checksum "setup.py" }}
86
87
                paths:
                    - '~/.cache/pip'
88
            - run: RUN_PT_TF_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_tf ./tests/ -m is_pt_tf_cross_test --durations=0 | tee tests_output.txt
89
            - store_artifacts:
90
91
92
93
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    run_tests_torch_and_flax:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.6
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
                      - v0.4-torch_and_flax-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
            - run: pip install --upgrade pip
            - run: pip install .[sklearn,flax,torch,testing,sentencepiece,speech]
            - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.0+cpu.html
            - save_cache:
                key: v0.4-{{ checksum "setup.py" }}
                paths:
                    - '~/.cache/pip'
            - run: RUN_PT_FLAX_CROSS_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_torch_and_flax ./tests/ -m is_pt_flax_cross_test --durations=0 | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
122
    run_tests_torch:
123
        working_directory: ~/transformers
Julien Chaumond's avatar
Julien Chaumond committed
124
        docker:
125
            - image: circleci/python:3.7
126
127
        environment:
            OMP_NUM_THREADS: 1
128
        resource_class: xlarge
129
        parallelism: 1
Julien Chaumond's avatar
Julien Chaumond committed
130
131
        steps:
            - checkout
132
133
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
134
135
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
Suraj Patil's avatar
Suraj Patil committed
136
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
137
            - run: pip install --upgrade pip
Suraj Patil's avatar
Suraj Patil committed
138
            - run: pip install .[sklearn,torch,testing,sentencepiece,speech]
139
            - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.0+cpu.html
140
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
141
                  key: v0.4-torch-{{ checksum "setup.py" }}
142
143
                  paths:
                      - '~/.cache/pip'
144
            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=tests_torch ./tests/ | tee tests_output.txt
145
            - store_artifacts:
146
147
148
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
Lysandre Debut's avatar
Lysandre Debut committed
149

Aymeric Augustin's avatar
Aymeric Augustin committed
150
    run_tests_tf:
151
        working_directory: ~/transformers
thomwolf's avatar
thomwolf committed
152
        docker:
153
            - image: circleci/python:3.7
154
155
        environment:
            OMP_NUM_THREADS: 1
thomwolf's avatar
thomwolf committed
156
157
158
159
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
160
161
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
162
163
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
164
            - run: pip install --upgrade pip
165
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
166
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
167
                  key: v0.4-tf-{{ checksum "setup.py" }}
168
169
                  paths:
                      - '~/.cache/pip'
170
            - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_tf ./tests/ | tee tests_output.txt
171
            - store_artifacts:
172
173
174
175
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

176
177
178
179
180
181
182
183
184
185
186
187
    run_tests_flax:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                keys:
Lysandre Debut's avatar
Lysandre Debut committed
188
189
                    - v0.4-flax-{{ checksum "setup.py" }}
                    - v0.4-{{ checksum "setup.py" }}
190
            - run: pip install --upgrade pip
191
            - run: sudo pip install .[flax,testing,sentencepiece]
192
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
193
                  key: v0.4-flax-{{ checksum "setup.py" }}
194
195
                  paths:
                      - '~/.cache/pip'
196
            - run: python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_flax ./tests/ | tee tests_output.txt
197
            - store_artifacts:
198
199
200
201
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

202
203
204
205
206
207
208
209
210
211
212
213
    run_tests_pipelines_torch:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
214
215
                      - v0.4-torch-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
Suraj Patil's avatar
Suraj Patil committed
216
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
217
            - run: pip install --upgrade pip
Suraj Patil's avatar
Suraj Patil committed
218
            - run: pip install .[sklearn,torch,testing,sentencepiece,speech]
219
            - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.0+cpu.html
220
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
221
                  key: v0.4-torch-{{ checksum "setup.py" }}
222
223
                  paths:
                      - '~/.cache/pip'
224
            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_torch -m is_pipeline_test ./tests/ | tee tests_output.txt
225
            - store_artifacts:
226
227
228
229
                  path: ~/transformers/tests_output.txt
            - store_artifacts:
                  path: ~/transformers/reports

230
231
232
233
234
235
236
237
238
239
240
241
    run_tests_pipelines_tf:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
242
243
                      - v0.4-tf-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
244
            - run: pip install --upgrade pip
245
            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece]
246
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
247
                  key: v0.4-tf-{{ checksum "setup.py" }}
248
249
                  paths:
                      - '~/.cache/pip'
250
251
252
            - run: RUN_PIPELINE_TESTS=1 python -m pytest -n 8 --dist=loadfile -rA -s --make-reports=tests_pipelines_tf ./tests/ -m is_pipeline_test | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
253
            - store_artifacts:
254
255
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
256
    run_tests_custom_tokenizers:
257
258
        working_directory: ~/transformers
        docker:
259
            - image: circleci/python:3.7
260
261
        environment:
            RUN_CUSTOM_TOKENIZERS: yes
262
263
        steps:
            - checkout
264
265
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
266
267
                      - v0.4-custom_tokenizers-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
268
            - run: pip install --upgrade pip
269
            - run: pip install .[ja,testing,sentencepiece]
270
            - run: python -m unidic download
271
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
272
                  key: v0.4-custom_tokenizers-{{ checksum "setup.py" }}
273
274
                  paths:
                      - '~/.cache/pip'
275
276
277
            - run: python -m pytest -s --make-reports=tests_custom_tokenizers ./tests/test_tokenization_bert_japanese.py | tee tests_output.txt
            - store_artifacts:
                  path: ~/transformers/tests_output.txt
278
            - store_artifacts:
279
280
                  path: ~/transformers/reports

Aymeric Augustin's avatar
Aymeric Augustin committed
281
    run_examples_torch:
282
283
        working_directory: ~/transformers
        docker:
284
            - image: circleci/python:3.6
285
286
287
288
289
290
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
291
292
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
293
294
                      - v0.4-torch_examples-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
295
            - run: pip install --upgrade pip
296
            - run: pip install .[sklearn,torch,sentencepiece,testing]
Sylvain Gugger's avatar
Sylvain Gugger committed
297
            - run: pip install -r examples/_tests_requirements.txt
298
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
299
                  key: v0.4-torch_examples-{{ checksum "setup.py" }}
300
301
                  paths:
                      - '~/.cache/pip'
302
            - run: python -m pytest -n 8 --dist=loadfile -s --make-reports=examples_torch ./examples/ | tee examples_output.txt
303
            - store_artifacts:
304
305
306
                  path: ~/transformers/examples_output.txt
            - store_artifacts:
                  path: ~/transformers/reports
307

308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
    run_tests_git_lfs:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.7
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - run: sudo apt-get install git-lfs
            - run: |
                git config --global user.email "ci@dummy.com"
                git config --global user.name "ci"
            - run: pip install --upgrade pip
            - run: pip install .[testing]
            - run: RUN_GIT_LFS_TESTS=1 python -m pytest -sv ./tests/test_hf_api.py -k "HfLargefilesTest"

324
325
326
327
328
329
    build_doc:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.6
        steps:
            - checkout
330
331
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
332
333
                      - v0.4-build_doc-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
Suraj Patil's avatar
Suraj Patil committed
334
            - run: sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev
335
            - run: pip install --upgrade pip
336
            - run: pip install ."[all, docs]"
337
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
338
                  key: v0.4-build_doc-{{ checksum "setup.py" }}
339
340
                  paths:
                      - '~/.cache/pip'
Stas Bekman's avatar
Stas Bekman committed
341
            - run: cd docs && make html SPHINXOPTS="-W -j 4"
342
343
            - store_artifacts:
                path: ./docs/_build
344

LysandreJik's avatar
LysandreJik committed
345
    deploy_doc:
346
        working_directory: ~/transformers
LysandreJik's avatar
LysandreJik committed
347
        docker:
348
            - image: circleci/python:3.6
LysandreJik's avatar
LysandreJik committed
349
350
        steps:
            - add_ssh_keys:
351
352
                fingerprints:
                    - "5b:7a:95:18:07:8c:aa:76:4c:60:35:88:ad:60:56:71"
LysandreJik's avatar
LysandreJik committed
353
            - checkout
354
355
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
356
357
                      - v0.4-deploy_doc-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
358
            - run: pip install ."[all,docs]"
359
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
360
                  key: v0.4-deploy_doc-{{ checksum "setup.py" }}
361
362
                  paths:
                      - '~/.cache/pip'
Lysandre's avatar
Lysandre committed
363
            - run: ./.circleci/deploy.sh
364

Aymeric Augustin's avatar
Aymeric Augustin committed
365
366
367
368
    check_code_quality:
        working_directory: ~/transformers
        docker:
            - image: circleci/python:3.6
Aymeric Augustin's avatar
Aymeric Augustin committed
369
        resource_class: medium
Aymeric Augustin's avatar
Aymeric Augustin committed
370
371
372
        parallelism: 1
        steps:
            - checkout
373
374
            - restore_cache:
                  keys:
Lysandre Debut's avatar
Lysandre Debut committed
375
376
                      - v0.4-code_quality-{{ checksum "setup.py" }}
                      - v0.4-{{ checksum "setup.py" }}
377
            - run: pip install --upgrade pip
378
            - run: pip install isort
379
            - run: pip install .[all,quality]
380
            - save_cache:
Lysandre Debut's avatar
Lysandre Debut committed
381
                  key: v0.4-code_quality-{{ checksum "setup.py" }}
382
383
                  paths:
                      - '~/.cache/pip'
384
385
            - run: black --check examples tests src utils
            - run: isort --check-only examples tests src utils
Sylvain Gugger's avatar
Sylvain Gugger committed
386
            - run: python utils/custom_init_isort.py --check_only
387
            - run: flake8 examples tests src utils
Sylvain Gugger's avatar
Sylvain Gugger committed
388
            - run: python utils/style_doc.py src/transformers docs/source --max_len 119 --check_only
389
            - run: python utils/check_copies.py
390
            - run: python utils/check_table.py
391
            - run: python utils/check_dummies.py
392
            - run: python utils/check_repo.py
393

394
    check_repository_consistency:
R茅mi Louf's avatar
R茅mi Louf committed
395
396
        working_directory: ~/transformers
        docker:
397
            - image: circleci/python:3.6
R茅mi Louf's avatar
R茅mi Louf committed
398
399
400
401
        resource_class: small
        parallelism: 1
        steps:
            - checkout
402
            - run: pip install requests
R茅mi Louf's avatar
R茅mi Louf committed
403
            - run: python ./utils/link_tester.py
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423

# TPU JOBS
    run_examples_tpu:
        docker:
            - image: circleci/python:3.6
        environment:
            OMP_NUM_THREADS: 1
        resource_class: xlarge
        parallelism: 1
        steps:
            - checkout
            - go/install
            - *checkout_ml_testing
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - setup_remote_docker
            - *build_push_docker
            - *deploy_cluster
424

425
426
427
428
429
430
431
432
433
    cleanup-gke-jobs:
        docker:
            - image: circleci/python:3.6
        steps:
            - gcp-gke/install
            - gcp-gke/update-kubeconfig-with-credentials:
                  cluster: $GKE_CLUSTER
                  perform-login: true
            - *delete_gke_jobs
434

LysandreJik's avatar
LysandreJik committed
435
436
437
438
workflow_filters: &workflow_filters
    filters:
        branches:
            only:
Lysandre's avatar
Lysandre committed
439
                - master
440
workflows:
LysandreJik's avatar
LysandreJik committed
441
442
443
    version: 2
    build_and_test:
        jobs:
Aymeric Augustin's avatar
Aymeric Augustin committed
444
            - check_code_quality
445
            - check_repository_consistency
Aymeric Augustin's avatar
Aymeric Augustin committed
446
447
448
            - run_examples_torch
            - run_tests_custom_tokenizers
            - run_tests_torch_and_tf
449
            - run_tests_torch_and_flax
Aymeric Augustin's avatar
Aymeric Augustin committed
450
451
            - run_tests_torch
            - run_tests_tf
452
            - run_tests_flax
453
454
            - run_tests_pipelines_torch
            - run_tests_pipelines_tf
455
            - run_tests_git_lfs
456
            - build_doc
Lysandre's avatar
Lysandre committed
457
            - deploy_doc: *workflow_filters
458
459
460
461
462
463
464
465
466
467
468
469
#    tpu_testing_jobs:
#        triggers:
#            - schedule:
#                # Set to run at the first minute of every hour.
#                cron: "0 8 * * *"
#                filters:
#                    branches:
#                        only:
#                            - master
#        jobs:
#            - cleanup-gke-jobs
#            - run_examples_tpu