config.yml.in 33.4 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
Edward Z. Yang's avatar
Edward Z. Yang committed
5
6
7
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
#       Job names are 'name:' key.
8

9
executors:
10
11
12
13
14
15
16
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
17
    machine:
18
19
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
20
21
      shell: bash.exe

22
23
24
25
26
commands:
  checkout_merge:
    description: "checkout merge branch"
    steps:
      - checkout
27
28
29
30
31
#     - run:
#         name: Checkout merge branch
#         command: |
#           set -ex
#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
32
#           if [[ "$BRANCH" != "main" ]]; then
33
34
35
#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
#             git checkout "merged/$CIRCLE_BRANCH"
#           fi
36
37
38
39
40
41
42
43
44
45
46
47
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
48

49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
  brew_update:
    description: "Update Homebrew and install base formulae"
    steps:
      - run:
          name: Update Homebrew
          no_output_timeout: "10m"
          command: |
            set -ex

            # Update repositories manually.
            # Running `brew update` produces a comparison between the
            # current checkout and the updated checkout, which takes a
            # very long time because the existing checkout is 2y old.
            for path in $(find /usr/local/Homebrew -type d -name .git)
            do
            cd $path/..
            git fetch --depth=1 origin
            git reset --hard origin/master
            done

            export HOMEBREW_NO_AUTO_UPDATE=1

            # Install expect and moreutils so that we can call `unbuffer` and `ts`.
            # moreutils installs a `parallel` executable by default, which conflicts
            # with the executable from the GNU `parallel`, so we must unlink GNU
            # `parallel` first, and relink it afterwards.
            brew install coreutils
            brew unlink parallel
            brew install moreutils
            brew link parallel --overwrite
            brew install expect

  brew_install:
    description: "Install Homebrew formulae"
    parameters:
      formulae:
        type: string
        default: ""
    steps:
      - run:
          name: Install << parameters.formulae >>
          no_output_timeout: "10m"
          command: |
            set -ex
            export HOMEBREW_NO_AUTO_UPDATE=1
            brew install << parameters.formulae >>

  run_brew_for_ios_build:
    steps:
      - brew_update
      - brew_install:
          formulae: libtool

102
103
binary_common: &binary_common
  parameters:
104
    # Edit these defaults to do a release
105
106
107
108
109
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
110
      description: "PyTorch version to build against; by default, use a nightly"
111
      type: string
112
      default: ""
113
114
115
116
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
117
118
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
119
      type: string
guyang3532's avatar
guyang3532 committed
120
      default: "cpu"
121
122
123
124
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
Edward Z. Yang's avatar
Edward Z. Yang committed
125
126
127
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
128
      default: "pytorch/manylinux-cuda102"
129
130
131
132
    conda_docker_image:
      description: "Conda only: what docker image to use"
      type: string
      default: "pytorch/conda-builder:cpu"
133
134
135
136
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
Edward Z. Yang's avatar
Edward Z. Yang committed
137
    CU_VERSION: << parameters.cu_version >>
138

139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
torchvision_ios_params: &torchvision_ios_params
  parameters:
    build_environment:
      type: string
      default: ""
    ios_arch:
      type: string
      default: ""
    ios_platform:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    IOS_ARCH: << parameters.ios_arch >>
    IOS_PLATFORM: << parameters.ios_platform >>

155
156
157
158
159
160
161
162
torchvision_android_params: &torchvision_android_params
  parameters:
    build_environment:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>

guyang3532's avatar
guyang3532 committed
163
164
165
166
167
smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: torchvision/smoke_test:latest

168
169
170
171
172
173
174
175
jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
176
            pip install --user --progress-bar off jinja2 pyyaml
177
178
179
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

180
181
182
183
184
185
186
  python_lint:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
187
188
189
            pip install --user --progress-bar off pre-commit
            pre-commit install-hooks
      - run: pre-commit run --all-files
190
191
192
193
194
195
196
197

  python_type_check:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
198
199
            sudo apt-get update -y
            sudo apt install -y libturbojpeg-dev
200
            pip install --user --progress-bar off mypy
201
            pip install --user --progress-bar off types-requests
202
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
203
            pip install --user --progress-bar off git+https://github.com/pytorch/data.git
204
            pip install --user --progress-bar off --no-build-isolation --editable .
205
            mypy --config-file mypy.ini
206

207
208
209
210
211
212
213
214
215
216
  docstring_parameters_sync:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user pydocstyle
            pydocstyle

217
218
219
220
221
222
223
  clang_format:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
224
225
226
            curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format
            chmod +x clang-format
            sudo mv clang-format /opt/clang-format
227
            ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format
228

Francisco Massa's avatar
Francisco Massa committed
229
230
231
232
233
234
235
236
237
  torchhub_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
238
            pip install --user --progress-bar off --no-build-isolation .
Anirudh's avatar
Anirudh committed
239
            pip install pytest
Francisco Massa's avatar
Francisco Massa committed
240
241
            python test/test_hub.py

242
243
244
245
246
247
248
249
250
  torch_onnx_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
251
            pip install --user --progress-bar off --no-build-isolation .
252
            pip install --user onnx
253
            pip install --user onnxruntime
254
            pip install --user pytest
255
256
            python test/test_onnx.py

257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
  prototype_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - run:
          name: Install torch
          command: pip install --user --progress-bar=off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
      - run:
          name: Install prototype dependencies
          command: pip install --user --progress-bar=off git+https://github.com/pytorch/data.git
      - checkout
      - run:
          name: Install torchvision
          command: pip install --user --progress-bar off --no-build-isolation .
      - run:
          name: Install test utilities
          command: pip install --user --progress-bar=off pytest
      - run:
          name: Run tests
          command: pytest test/test_prototype_*.py

278
279
280
  binary_linux_wheel:
    <<: *binary_common
    docker:
Edward Z. Yang's avatar
Edward Z. Yang committed
281
      - image: << parameters.wheel_docker_image >>
282
283
    resource_class: 2xlarge+
    steps:
284
      - checkout_merge
285
      - designate_upload_channel
286
287
288
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
289
290
291
292
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
293
294
295
296

  binary_linux_conda:
    <<: *binary_common
    docker:
297
      - image: "<< parameters.conda_docker_image >>"
298
299
    resource_class: 2xlarge+
    steps:
300
      - checkout_merge
301
      - designate_upload_channel
302
303
304
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
305
306
307
308
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"
309
310
      - store_test_results:
          path: build_results/
311

Francisco Massa's avatar
Francisco Massa committed
312
  binary_win_conda:
313
314
    <<: *binary_common
    executor: windows-cpu
315
316
    steps:
      - checkout_merge
317
      - designate_upload_channel
318
319
      - run:
          name: Build conda packages
320
          no_output_timeout: 20m
321
          command: |
322
323
324
325
326
327
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
328
329
330
331
            # cudatoolkit >= 11 isn't available for windows in the nvidia channel
            if [[ "${CU_VERSION}" =~ cu11.* ]]; then
              export CONDA_CHANNEL_FLAGS="-c conda-forge"
            fi
332
            packaging/build_conda.sh
333
            rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
334
      - store_artifacts:
335
          path: C:/tools/miniconda3/conda-bld/win-64
336
      - persist_to_workspace:
337
          root: C:/tools/miniconda3/conda-bld/win-64
338
339
340
341
342
          paths:
            - "*"
      - store_test_results:
          path: build_results/

343
  binary_win_wheel:
344
345
    <<: *binary_common
    executor: windows-cpu
346
347
    steps:
      - checkout_merge
348
      - designate_upload_channel
349
350
351
      - run:
          name: Build wheel packages
          command: |
352
353
354
355
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_wheel.sh
356
      - store_artifacts:
357
          path: dist
358
      - persist_to_workspace:
359
          root: dist
360
361
362
363
364
          paths:
            - "*"
      - store_test_results:
          path: build_results/

365
366
367
  binary_macos_wheel:
    <<: *binary_common
    macos:
368
      xcode: "12.0"
369
    steps:
370
      - checkout_merge
371
      - designate_upload_channel
372
373
374
375
376
377
378
379
380
381
382
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
383
384
385
386
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
387

388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
  binary_ios_build:
    <<: *torchvision_ios_params
    macos:
      xcode: "12.0"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
          cat "$script"
          source "$script"
    - persist_to_workspace:
        root: /Users/distiller/workspace/
        paths: ios

  binary_ios_upload:
    <<: *torchvision_ios_params
    macos:
      xcode: "12.0"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Upload
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
          cat "$script"
          source "$script"

425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
  binary_android_build:
    <<: *torchvision_android_params
    docker:
      - image: circleci/android:api-29-ndk
    resource_class: xlarge
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh"
          cat "$script"
          source "$script"
    - store_artifacts:
        path: ~/workspace/artifacts

  binary_android_upload:
    <<: *torchvision_android_params
    docker:
      - image: circleci/android:api-29-ndk
    resource_class: xlarge
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run:
        name: Upload
        no_output_timeout: "1h"
        command: |
          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh"
          cat "$script"
          source "$script"

461
462
463
  binary_macos_conda:
    <<: *binary_common
    macos:
464
      xcode: "12.0"
465
    steps:
466
      - checkout_merge
467
      - designate_upload_channel
468
469
470
471
472
473
474
475
476
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
477
478
479
480
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
481
482
      - store_test_results:
          path: build_results/
483

Edward Z. Yang's avatar
Edward Z. Yang committed
484
485
486
487
488
489
490
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
491
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
492
493
494
495
496
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
497
            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
Edward Z. Yang's avatar
Edward Z. Yang committed
498
499
500

  # Requires org-member context
  binary_wheel_upload:
Edward Z. Yang's avatar
Edward Z. Yang committed
501
502
503
504
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
505
506
507
508
509
    docker:
      - image: circleci/python:3.7
    steps:
      - attach_workspace:
          at: ~/workspace
510
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
511
512
513
514
515
516
517
518
519
520
      - checkout
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
Edward Z. Yang's avatar
Edward Z. Yang committed
521
            for pkg in ~/workspace/*.whl; do
522
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
Edward Z. Yang's avatar
Edward Z. Yang committed
523
            done
Edward Z. Yang's avatar
Edward Z. Yang committed
524

guyang3532's avatar
guyang3532 committed
525
526
527
528
529
  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
530
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
549
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_docker_image_build:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchvision/smoke_test
    steps:
      - checkout
570
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
      - run:
          name: Build and push Docker image
          no_output_timeout: "1h"
          command: |
            set +x
            echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin
            set -x
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest
            docker push ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push ${image_name}:latest

  smoke_test_win_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
590
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
591
592
593
594
595
596
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
597
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
guyang3532's avatar
guyang3532 committed
598
            conda activate python${PYTHON_VERSION}
599
            conda install Pillow>=5.3.0
guyang3532's avatar
guyang3532 committed
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_win_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
616
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

632
633
634
635
636
637
638
  unittest_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout
639
      - designate_upload_channel
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_linux_gpu:
    <<: *binary_common
    machine:
674
      image: ubuntu-1604-cuda-10.2:202012-01
675
    resource_class: gpu.nvidia.medium
676
    environment:
677
      image_name: "pytorch/manylinux-cuda102"
678
      PYTHON_VERSION: << parameters.python_version >>
679
680
    steps:
      - checkout
681
      - designate_upload_channel
682
683
684
685
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
686
687
688
689
690
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
691
692
      - run:
          name: Setup
693
          command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
694
695
      - save_cache:
          {% raw %}
696
          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
697
698
699
700
          {% endraw %}
          paths:
            - conda
            - env
701
702
703
704
705
706
707
708
      - run:
          # Here we create an envlist file that contains some env variables that we want the docker container to be aware of.
          # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables.
          # They're avaiable in all the other workflows (OSX and Windows).
          # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible.
          # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run".
          name: export CIRCLECI env var
          command: echo "CIRCLECI=true" >> ./env.list
709
710
      - run:
          name: Install torchvision
711
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh
712
713
      - run:
          name: Run tests
714
          command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
715
716
717
718
719
720
721
722
723
724
725
726
      - run:
          name: Post Process
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
727
      - designate_upload_channel
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
764
      CUDA_VERSION: "11.1"
765
      PYTHON_VERSION: << parameters.python_version >>
766
767
    steps:
      - checkout
768
      - designate_upload_channel
769
770
771
772
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
773
774
775
776
      - restore_cache:
          {% raw %}
          keys:
            - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
777
          {% endraw %}
778
779
780
781
782
783
784
785
786
787
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
788
789
790
791
792
793
      - run:
          name: Install CUDA
          command: packaging/windows/internal/cuda_install.bat
      - run:
          name: Update CUDA driver
          command: packaging/windows/internal/driver_update.bat
794
795
796
797
798
799
800
801
802
803
804
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results
Edward Z. Yang's avatar
Edward Z. Yang committed
805

Francisco Massa's avatar
Francisco Massa committed
806
807
808
  unittest_macos_cpu:
    <<: *binary_common
    macos:
809
      xcode: "12.0"
Francisco Massa's avatar
Francisco Massa committed
810
811
812
    resource_class: large
    steps:
      - checkout
813
      - designate_upload_channel
Francisco Massa's avatar
Francisco Massa committed
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
      - run:
          name: Install wget
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
          # Disable brew auto update which is very slow
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

849
850
851
852
853
854
855
  cmake_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout_merge
856
      - designate_upload_channel
857
858
859
860
861
862
863
864
      - run:
          name: Setup conda
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run: packaging/build_cmake.sh

  cmake_linux_gpu:
    <<: *binary_common
    machine:
865
      image: ubuntu-1604-cuda-10.2:202012-01
866
    resource_class: gpu.nvidia.small
867
868
869
870
871
872
873
    environment:
      PYTHON_VERSION: << parameters.python_version >>
      PYTORCH_VERSION: << parameters.pytorch_version >>
      UNICODE_ABI: << parameters.unicode_abi >>
      CU_VERSION: << parameters.cu_version >>
    steps:
      - checkout_merge
874
      - designate_upload_channel
875
876
877
878
879
      - run:
          name: Setup conda
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Build torchvision C++ distribution and test
880
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
881
882
883
884

  cmake_macos_cpu:
    <<: *binary_common
    macos:
885
      xcode: "12.0"
886
887
    steps:
      - checkout_merge
888
      - designate_upload_channel
889
890
891
892
893
894
895
896
897
898
899
900
901
902
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build cmake
            packaging/build_cmake.sh

  cmake_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout_merge
903
      - designate_upload_channel
904
905
906
907
908
909
910
911
912
913
914
915
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/build_cmake.sh

  cmake_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    steps:
      - checkout_merge
916
      - designate_upload_channel
917
918
919
920
921
922
923
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_cmake.sh

924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
  build_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - designate_upload_channel
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Build docs
          command: |
            set -ex
            tag=${CIRCLE_TAG:1:5}
945
            VERSION=${tag:-main}
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
            eval "$(./conda/bin/conda shell.bash hook)"
            conda activate ./env
            pushd docs
            pip install -r requirements.txt
            make html
            popd
      - persist_to_workspace:
          root: ./
          paths:
            - "*"
      - store_artifacts:
          path: ./docs/build/html
          destination: docs

  upload_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - run:
          name: Generate netrc
          command: |
            # set credentials for https pushing
            # requires the org-member context
            cat > ~/.netrc \<<DONE
              machine github.com
              login pytorchbot
              password ${GITHUB_PYTORCHBOT_TOKEN}
            DONE
      - run:
          name: Upload docs
          command: |
            # Don't use "checkout" step since it uses ssh, which cannot git push
            # https://circleci.com/docs/2.0/configuration-reference/#checkout
            set -ex
            tag=${CIRCLE_TAG:1:5}
985
            target=${tag:-main}
986
987
988
            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target


Edward Z. Yang's avatar
Edward Z. Yang committed
989
990
991
992
993
workflows:
  build:
{%- if True %}
    jobs:
      - circleci_consistency
994
      {{ build_workflows(windows_latest_only=True) }}
995
      - python_lint
996
      - python_type_check
997
      - docstring_parameters_sync
998
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
999
      - torchhub_test
1000
      - torch_onnx_test
1001
      - prototype_test
1002
      {{ ios_workflows() }}
1003
      {{ android_workflows() }}
Edward Z. Yang's avatar
Edward Z. Yang committed
1004

1005
1006
1007
  unittest:
    jobs:
      {{ unittest_workflows() }}
1008
1009
1010
1011
1012

  cmake:
    jobs:
      {{ cmake_workflows() }}

Edward Z. Yang's avatar
Edward Z. Yang committed
1013
1014
1015
1016
  nightly:
{%- endif %}
    jobs:
      - circleci_consistency
1017
      - python_lint
1018
      - python_type_check
1019
      - docstring_parameters_sync
1020
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
1021
      - torchhub_test
1022
      - torch_onnx_test
1023
      - prototype_test
1024
      {{ ios_workflows(nightly=True) }}
1025
      {{ android_workflows(nightly=True) }}
1026
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
guyang3532's avatar
guyang3532 committed
1027
1028
1029
1030
1031
1032
1033
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
1034
                - main
guyang3532's avatar
guyang3532 committed
1035
1036
1037
    jobs:
      - smoke_test_docker_image_build:
          context: org-member