config.yml.in 33.5 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
Edward Z. Yang's avatar
Edward Z. Yang committed
5
6
7
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
#       Job names are 'name:' key.
8

9
executors:
10
11
12
13
14
15
16
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
17
    machine:
18
19
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
20
21
      shell: bash.exe

22
23
24
25
26
commands:
  checkout_merge:
    description: "checkout merge branch"
    steps:
      - checkout
27
28
29
30
31
32
33
34
35
#     - run:
#         name: Checkout merge branch
#         command: |
#           set -ex
#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
#           if [[ "$BRANCH" != "master" ]]; then
#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
#             git checkout "merged/$CIRCLE_BRANCH"
#           fi
36
37
38
39
40
41
42
43
44
45
46
47
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
48
49
50
51
52
53
54
55
  install_cuda_compatible_cmath:
    description: "Install CUDA compatible cmath"
    steps:
      - run:
          name: _HACK_ Install CUDA compatible cmath
          no_output_timeout: 1m
          command: |
              powershell .circleci/scripts/vs_install_cmath.ps1
56

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
  brew_update:
    description: "Update Homebrew and install base formulae"
    steps:
      - run:
          name: Update Homebrew
          no_output_timeout: "10m"
          command: |
            set -ex

            # Update repositories manually.
            # Running `brew update` produces a comparison between the
            # current checkout and the updated checkout, which takes a
            # very long time because the existing checkout is 2y old.
            for path in $(find /usr/local/Homebrew -type d -name .git)
            do
            cd $path/..
            git fetch --depth=1 origin
            git reset --hard origin/master
            done

            export HOMEBREW_NO_AUTO_UPDATE=1

            # Install expect and moreutils so that we can call `unbuffer` and `ts`.
            # moreutils installs a `parallel` executable by default, which conflicts
            # with the executable from the GNU `parallel`, so we must unlink GNU
            # `parallel` first, and relink it afterwards.
            brew install coreutils
            brew unlink parallel
            brew install moreutils
            brew link parallel --overwrite
            brew install expect

  brew_install:
    description: "Install Homebrew formulae"
    parameters:
      formulae:
        type: string
        default: ""
    steps:
      - run:
          name: Install << parameters.formulae >>
          no_output_timeout: "10m"
          command: |
            set -ex
            export HOMEBREW_NO_AUTO_UPDATE=1
            brew install << parameters.formulae >>

  run_brew_for_ios_build:
    steps:
      - brew_update
      - brew_install:
          formulae: libtool

110
111
binary_common: &binary_common
  parameters:
112
    # Edit these defaults to do a release
113
114
115
116
117
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
118
      description: "PyTorch version to build against; by default, use a nightly"
119
      type: string
120
      default: ""
121
122
123
124
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
125
126
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
127
      type: string
guyang3532's avatar
guyang3532 committed
128
      default: "cpu"
129
130
131
132
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
Edward Z. Yang's avatar
Edward Z. Yang committed
133
134
135
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
136
      default: "pytorch/manylinux-cuda102"
137
138
139
140
    conda_docker_image:
      description: "Conda only: what docker image to use"
      type: string
      default: "pytorch/conda-builder:cpu"
141
142
143
144
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
Edward Z. Yang's avatar
Edward Z. Yang committed
145
    CU_VERSION: << parameters.cu_version >>
146

147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
torchvision_ios_params: &torchvision_ios_params
  parameters:
    build_environment:
      type: string
      default: ""
    ios_arch:
      type: string
      default: ""
    ios_platform:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    IOS_ARCH: << parameters.ios_arch >>
    IOS_PLATFORM: << parameters.ios_platform >>

163
164
165
166
167
168
169
170
torchvision_android_params: &torchvision_android_params
  parameters:
    build_environment:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>

guyang3532's avatar
guyang3532 committed
171
172
173
174
175
smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: torchvision/smoke_test:latest

176
177
178
179
180
181
182
183
jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
184
            pip install --user --progress-bar off jinja2 pyyaml
185
186
187
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

188
189
190
191
192
193
194
195
  python_lint:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off flake8 typing
196
197
198
199
200
201
202
203
204
            flake8 --config=setup.cfg .

  python_type_check:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
205
206
            sudo apt-get update -y
            sudo apt install -y libturbojpeg-dev
207
            pip install --user --progress-bar off mypy
208
            pip install --user --progress-bar off types-requests
209
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
210
            pip install --user --progress-bar off --editable .
211
            mypy --config-file mypy.ini
212

213
214
215
216
217
218
219
220
221
222
  docstring_parameters_sync:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user pydocstyle
            pydocstyle

223
224
225
226
227
228
229
  clang_format:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
230
231
232
            curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format
            chmod +x clang-format
            sudo mv clang-format /opt/clang-format
233
            ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format
234

Francisco Massa's avatar
Francisco Massa committed
235
236
237
238
239
240
241
242
243
244
  torchhub_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
Anirudh's avatar
Anirudh committed
245
            pip install pytest
Francisco Massa's avatar
Francisco Massa committed
246
247
            python test/test_hub.py

248
249
250
251
252
253
254
255
256
257
258
  torch_onnx_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
            pip install --user onnx
259
            pip install --user onnxruntime
260
261
            python test/test_onnx.py

262
263
264
  binary_linux_wheel:
    <<: *binary_common
    docker:
Edward Z. Yang's avatar
Edward Z. Yang committed
265
      - image: << parameters.wheel_docker_image >>
266
267
    resource_class: 2xlarge+
    steps:
268
      - checkout_merge
269
      - designate_upload_channel
270
271
272
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
273
274
275
276
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
277
278
279
280

  binary_linux_conda:
    <<: *binary_common
    docker:
281
      - image: "<< parameters.conda_docker_image >>"
282
283
    resource_class: 2xlarge+
    steps:
284
      - checkout_merge
285
      - designate_upload_channel
286
287
288
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
289
290
291
292
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"
293
294
      - store_test_results:
          path: build_results/
295

Francisco Massa's avatar
Francisco Massa committed
296
  binary_win_conda:
297
298
    <<: *binary_common
    executor: windows-cpu
299
300
    steps:
      - checkout_merge
301
      - designate_upload_channel
302
      - install_cuda_compatible_cmath
303
304
      - run:
          name: Build conda packages
305
          no_output_timeout: 20m
306
          command: |
307
308
309
310
311
312
313
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
            packaging/build_conda.sh
314
            rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
315
      - store_artifacts:
316
          path: C:/tools/miniconda3/conda-bld/win-64
317
      - persist_to_workspace:
318
          root: C:/tools/miniconda3/conda-bld/win-64
319
320
321
322
323
          paths:
            - "*"
      - store_test_results:
          path: build_results/

324
  binary_win_wheel:
325
326
    <<: *binary_common
    executor: windows-cpu
327
328
    steps:
      - checkout_merge
329
      - designate_upload_channel
330
      - install_cuda_compatible_cmath
331
332
333
      - run:
          name: Build wheel packages
          command: |
334
335
336
337
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_wheel.sh
338
      - store_artifacts:
339
          path: dist
340
      - persist_to_workspace:
341
          root: dist
342
343
344
345
346
          paths:
            - "*"
      - store_test_results:
          path: build_results/

347
348
349
  binary_macos_wheel:
    <<: *binary_common
    macos:
350
      xcode: "12.0"
351
    steps:
352
      - checkout_merge
353
      - designate_upload_channel
354
355
356
357
358
359
360
361
362
363
364
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
365
366
367
368
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
369

370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
  binary_ios_build:
    <<: *torchvision_ios_params
    macos:
      xcode: "12.0"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
          cat "$script"
          source "$script"
    - persist_to_workspace:
        root: /Users/distiller/workspace/
        paths: ios

  binary_ios_upload:
    <<: *torchvision_ios_params
    macos:
      xcode: "12.0"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Upload
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
          cat "$script"
          source "$script"

407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
  binary_android_build:
    <<: *torchvision_android_params
    docker:
      - image: circleci/android:api-29-ndk
    resource_class: xlarge
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh"
          cat "$script"
          source "$script"
    - store_artifacts:
        path: ~/workspace/artifacts

  binary_android_upload:
    <<: *torchvision_android_params
    docker:
      - image: circleci/android:api-29-ndk
    resource_class: xlarge
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run:
        name: Upload
        no_output_timeout: "1h"
        command: |
          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh"
          cat "$script"
          source "$script"

443
444
445
  binary_macos_conda:
    <<: *binary_common
    macos:
446
      xcode: "12.0"
447
    steps:
448
      - checkout_merge
449
      - designate_upload_channel
450
451
452
453
454
455
456
457
458
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
459
460
461
462
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
463
464
      - store_test_results:
          path: build_results/
465

Edward Z. Yang's avatar
Edward Z. Yang committed
466
467
468
469
470
471
472
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
473
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
474
475
476
477
478
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
479
            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
Edward Z. Yang's avatar
Edward Z. Yang committed
480
481
482

  # Requires org-member context
  binary_wheel_upload:
Edward Z. Yang's avatar
Edward Z. Yang committed
483
484
485
486
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
487
488
489
490
491
    docker:
      - image: circleci/python:3.7
    steps:
      - attach_workspace:
          at: ~/workspace
492
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
493
494
495
496
497
498
499
500
501
502
      - checkout
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
Edward Z. Yang's avatar
Edward Z. Yang committed
503
            for pkg in ~/workspace/*.whl; do
504
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
Edward Z. Yang's avatar
Edward Z. Yang committed
505
            done
Edward Z. Yang's avatar
Edward Z. Yang committed
506

guyang3532's avatar
guyang3532 committed
507
508
509
510
511
  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
512
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
531
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_docker_image_build:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchvision/smoke_test
    steps:
      - checkout
552
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
      - run:
          name: Build and push Docker image
          no_output_timeout: "1h"
          command: |
            set +x
            echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin
            set -x
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest
            docker push ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push ${image_name}:latest

  smoke_test_win_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
572
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
573
574
575
576
577
578
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
579
580
581
582
583
            CONDA_CHANNEL_FLAGS=""
            if [[ "${PYTHON_VERSION}" = 3.9 ]]; then
              CONDA_CHANNEL_FLAGS="-c=conda-forge"
            fi
            conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
guyang3532's avatar
guyang3532 committed
584
            conda activate python${PYTHON_VERSION}
585
            conda install Pillow>=5.3.0
guyang3532's avatar
guyang3532 committed
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_win_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
602
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
603
604
605
606
607
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
608
609
610
611
612
            CONDA_CHANNEL_FLAGS=""
            if [[ "${PYTHON_VERSION}" = 3.9 ]]; then
              CONDA_CHANNEL_FLAGS="-c=conda-forge"
            fi
            conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
guyang3532's avatar
guyang3532 committed
613
614
615
616
617
618
619
620
621
622
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

623
624
625
626
627
628
629
  unittest_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout
630
      - designate_upload_channel
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_linux_gpu:
    <<: *binary_common
    machine:
665
      image: ubuntu-1604-cuda-10.2:202012-01
666
    resource_class: gpu.nvidia.medium
667
    environment:
668
      image_name: "pytorch/manylinux-cuda102"
669
      PYTHON_VERSION: << parameters.python_version >>
670
671
    steps:
      - checkout
672
      - designate_upload_channel
673
674
675
676
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
677
678
679
680
681
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
682
683
      - run:
          name: Setup
684
          command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
685
686
      - save_cache:
          {% raw %}
687
          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
688
689
690
691
          {% endraw %}
          paths:
            - conda
            - env
692
693
694
695
696
697
698
699
      - run:
          # Here we create an envlist file that contains some env variables that we want the docker container to be aware of.
          # Normally, the CIRCLECI variable is set and available on all CI workflows: https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables.
          # They're avaiable in all the other workflows (OSX and Windows).
          # But here, we're running the unittest_linux_gpu workflows in a docker container, where those variables aren't accessible.
          # So instead we dump the variables we need in env.list and we pass that file when invoking "docker run".
          name: export CIRCLECI env var
          command: echo "CIRCLECI=true" >> ./env.list
700
701
      - run:
          name: Install torchvision
702
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh
703
704
      - run:
          name: Run tests
705
          command: docker run --env-file ./env.list -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
706
707
708
709
710
711
712
713
714
715
716
717
      - run:
          name: Post Process
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
718
      - designate_upload_channel
719
      - install_cuda_compatible_cmath
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
756
      CUDA_VERSION: "10.2"
757
      PYTHON_VERSION: << parameters.python_version >>
758
759
    steps:
      - checkout
760
      - designate_upload_channel
761
      - install_cuda_compatible_cmath
762
763
764
765
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
766
767
768
769
      - restore_cache:
          {% raw %}
          keys:
            - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
770
          {% endraw %}
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results
Edward Z. Yang's avatar
Edward Z. Yang committed
792

Francisco Massa's avatar
Francisco Massa committed
793
794
795
  unittest_macos_cpu:
    <<: *binary_common
    macos:
796
      xcode: "12.0"
Francisco Massa's avatar
Francisco Massa committed
797
798
799
    resource_class: large
    steps:
      - checkout
800
      - designate_upload_channel
Francisco Massa's avatar
Francisco Massa committed
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
      - run:
          name: Install wget
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
          # Disable brew auto update which is very slow
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

836
837
838
839
840
841
842
  cmake_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout_merge
843
      - designate_upload_channel
844
845
846
847
848
849
850
851
      - run:
          name: Setup conda
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run: packaging/build_cmake.sh

  cmake_linux_gpu:
    <<: *binary_common
    machine:
852
      image: ubuntu-1604-cuda-10.2:202012-01
853
854
855
856
857
858
859
860
    resource_class: gpu.small
    environment:
      PYTHON_VERSION: << parameters.python_version >>
      PYTORCH_VERSION: << parameters.pytorch_version >>
      UNICODE_ABI: << parameters.unicode_abi >>
      CU_VERSION: << parameters.cu_version >>
    steps:
      - checkout_merge
861
      - designate_upload_channel
862
863
864
865
866
      - run:
          name: Setup conda
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Build torchvision C++ distribution and test
867
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
868
869
870
871

  cmake_macos_cpu:
    <<: *binary_common
    macos:
872
      xcode: "12.0"
873
874
    steps:
      - checkout_merge
875
      - designate_upload_channel
876
877
878
879
880
881
882
883
884
885
886
887
888
889
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build cmake
            packaging/build_cmake.sh

  cmake_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout_merge
890
      - designate_upload_channel
891
      - install_cuda_compatible_cmath
892
893
894
895
896
897
898
899
900
901
902
903
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/build_cmake.sh

  cmake_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    steps:
      - checkout_merge
904
      - designate_upload_channel
905
      - install_cuda_compatible_cmath
906
907
908
909
910
911
912
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_cmake.sh

913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
  build_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - designate_upload_channel
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
929
930
931
932
933
934
935
936
937
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - sphinx-gallery-{{ checksum "./docs/source/conf.py" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
938
939
940
941
942
943
944
945
946
947
948
949
      - run:
          name: Build docs
          command: |
            set -ex
            tag=${CIRCLE_TAG:1:5}
            VERSION=${tag:-master}
            eval "$(./conda/bin/conda shell.bash hook)"
            conda activate ./env
            pushd docs
            pip install -r requirements.txt
            make html
            popd
950
951
952
953
954
955
      - save_cache:
          {% raw %}
          key: sphinx-gallery-{{ checksum "./docs/source/conf.py" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - ./docs/source/auto_examples
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
      - persist_to_workspace:
          root: ./
          paths:
            - "*"
      - store_artifacts:
          path: ./docs/build/html
          destination: docs

  upload_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - run:
          name: Generate netrc
          command: |
            # set credentials for https pushing
            # requires the org-member context
            cat > ~/.netrc \<<DONE
              machine github.com
              login pytorchbot
              password ${GITHUB_PYTORCHBOT_TOKEN}
            DONE
      - run:
          name: Upload docs
          command: |
            # Don't use "checkout" step since it uses ssh, which cannot git push
            # https://circleci.com/docs/2.0/configuration-reference/#checkout
            set -ex
            tag=${CIRCLE_TAG:1:5}
            target=${tag:-master}
            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target


Edward Z. Yang's avatar
Edward Z. Yang committed
993
994
995
996
997
workflows:
  build:
{%- if True %}
    jobs:
      - circleci_consistency
998
      {{ build_workflows(windows_latest_only=True) }}
999
      - python_lint
1000
      - python_type_check
1001
      - docstring_parameters_sync
1002
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
1003
      - torchhub_test
1004
      - torch_onnx_test
1005
      {{ ios_workflows() }}
1006
      {{ android_workflows() }}
Edward Z. Yang's avatar
Edward Z. Yang committed
1007

1008
1009
1010
  unittest:
    jobs:
      {{ unittest_workflows() }}
1011
1012
1013
1014
1015

  cmake:
    jobs:
      {{ cmake_workflows() }}

Edward Z. Yang's avatar
Edward Z. Yang committed
1016
1017
1018
1019
  nightly:
{%- endif %}
    jobs:
      - circleci_consistency
1020
      - python_lint
1021
      - python_type_check
1022
      - docstring_parameters_sync
1023
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
1024
      - torchhub_test
1025
      - torch_onnx_test
1026
      {{ ios_workflows(nightly=True) }}
1027
      {{ android_workflows(nightly=True) }}
1028
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
guyang3532's avatar
guyang3532 committed
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
                - master
    jobs:
      - smoke_test_docker_image_build:
          context: org-member