config.yml.in 27.9 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
Edward Z. Yang's avatar
Edward Z. Yang committed
5
6
7
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
#       Job names are 'name:' key.
8

9
executors:
10
11
12
13
14
15
16
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
17
    machine:
18
19
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
20
21
      shell: bash.exe

22
23
24
25
26
commands:
  checkout_merge:
    description: "checkout merge branch"
    steps:
      - checkout
27
28
29
30
31
32
33
34
35
#     - run:
#         name: Checkout merge branch
#         command: |
#           set -ex
#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
#           if [[ "$BRANCH" != "master" ]]; then
#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
#             git checkout "merged/$CIRCLE_BRANCH"
#           fi
36
37
38
39
40
41
42
43
44
45
46
47
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
48
49
50
51
52
53
54
55
  install_cuda_compatible_cmath:
    description: "Install CUDA compatible cmath"
    steps:
      - run:
          name: _HACK_ Install CUDA compatible cmath
          no_output_timeout: 1m
          command: |
              powershell .circleci/scripts/vs_install_cmath.ps1
56

57
58
binary_common: &binary_common
  parameters:
59
    # Edit these defaults to do a release
60
61
62
63
64
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
65
      description: "PyTorch version to build against; by default, use a nightly"
66
      type: string
67
      default: ""
68
69
70
71
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
72
73
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
74
      type: string
guyang3532's avatar
guyang3532 committed
75
      default: "cpu"
76
77
78
79
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
Edward Z. Yang's avatar
Edward Z. Yang committed
80
81
82
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
83
      default: "pytorch/manylinux-cuda101"
84
85
86
87
    conda_docker_image:
      description: "Conda only: what docker image to use"
      type: string
      default: "pytorch/conda-builder:cpu"
88
89
90
91
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
Edward Z. Yang's avatar
Edward Z. Yang committed
92
    CU_VERSION: << parameters.cu_version >>
93

guyang3532's avatar
guyang3532 committed
94
95
96
97
98
smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: torchvision/smoke_test:latest

99
100
101
102
103
104
105
106
jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
107
            pip install --user --progress-bar off jinja2 pyyaml
108
109
110
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

111
112
113
114
115
116
117
118
  python_lint:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off flake8 typing
119
120
121
122
123
124
125
126
127
            flake8 --config=setup.cfg .

  python_type_check:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
128
129
            sudo apt-get update -y
            sudo apt install -y libturbojpeg-dev
130
            pip install --user --progress-bar off mypy
131
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
132
            pip install --user --progress-bar off --editable .
133
            mypy --config-file mypy.ini
134

135
136
137
138
139
140
141
142
143
144
  docstring_parameters_sync:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user pydocstyle
            pydocstyle

145
146
147
148
149
150
151
  clang_format:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
152
153
154
            curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format
            chmod +x clang-format
            sudo mv clang-format /opt/clang-format
155
            ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format
156

Francisco Massa's avatar
Francisco Massa committed
157
158
159
160
161
162
163
164
165
166
167
168
  torchhub_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
            python test/test_hub.py

169
170
171
172
173
174
175
176
177
178
179
  torch_onnx_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
            pip install --user onnx
180
            pip install --user onnxruntime
181
182
            python test/test_onnx.py

183
184
185
  binary_linux_wheel:
    <<: *binary_common
    docker:
Edward Z. Yang's avatar
Edward Z. Yang committed
186
      - image: << parameters.wheel_docker_image >>
187
188
    resource_class: 2xlarge+
    steps:
189
      - checkout_merge
190
      - designate_upload_channel
191
192
193
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
194
195
196
197
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
198
199
200
201

  binary_linux_conda:
    <<: *binary_common
    docker:
202
      - image: "<< parameters.conda_docker_image >>"
203
204
    resource_class: 2xlarge+
    steps:
205
      - checkout_merge
206
      - designate_upload_channel
207
208
209
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
210
211
212
213
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"
214
215
      - store_test_results:
          path: build_results/
216

Francisco Massa's avatar
Francisco Massa committed
217
  binary_win_conda:
218
219
    <<: *binary_common
    executor: windows-cpu
220
221
    steps:
      - checkout_merge
222
      - designate_upload_channel
223
      - install_cuda_compatible_cmath
224
225
      - run:
          name: Build conda packages
226
          no_output_timeout: 20m
227
          command: |
228
229
230
231
232
233
234
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
            packaging/build_conda.sh
235
            rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
236
      - store_artifacts:
237
          path: C:/tools/miniconda3/conda-bld/win-64
238
      - persist_to_workspace:
239
          root: C:/tools/miniconda3/conda-bld/win-64
240
241
242
243
244
          paths:
            - "*"
      - store_test_results:
          path: build_results/

245
  binary_win_wheel:
246
247
    <<: *binary_common
    executor: windows-cpu
248
249
    steps:
      - checkout_merge
250
      - designate_upload_channel
251
      - install_cuda_compatible_cmath
252
253
254
      - run:
          name: Build wheel packages
          command: |
255
256
257
258
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_wheel.sh
259
      - store_artifacts:
260
          path: dist
261
      - persist_to_workspace:
262
          root: dist
263
264
265
266
267
          paths:
            - "*"
      - store_test_results:
          path: build_results/

268
269
270
  binary_macos_wheel:
    <<: *binary_common
    macos:
271
      xcode: "9.4.1"
272
    steps:
273
      - checkout_merge
274
      - designate_upload_channel
275
276
277
278
279
280
281
282
283
284
285
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
286
287
288
289
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
290
291
292
293

  binary_macos_conda:
    <<: *binary_common
    macos:
294
      xcode: "9.4.1"
295
    steps:
296
      - checkout_merge
297
      - designate_upload_channel
298
299
300
301
302
303
304
305
306
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
307
308
309
310
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
311
312
      - store_test_results:
          path: build_results/
313

Edward Z. Yang's avatar
Edward Z. Yang committed
314
315
316
317
318
319
320
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
321
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
322
323
324
325
326
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
327
            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
Edward Z. Yang's avatar
Edward Z. Yang committed
328
329
330

  # Requires org-member context
  binary_wheel_upload:
Edward Z. Yang's avatar
Edward Z. Yang committed
331
332
333
334
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
335
336
337
338
339
    docker:
      - image: circleci/python:3.7
    steps:
      - attach_workspace:
          at: ~/workspace
340
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
341
342
343
344
345
346
347
348
349
350
      - checkout
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
Edward Z. Yang's avatar
Edward Z. Yang committed
351
            for pkg in ~/workspace/*.whl; do
352
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
Edward Z. Yang's avatar
Edward Z. Yang committed
353
            done
Edward Z. Yang's avatar
Edward Z. Yang committed
354

guyang3532's avatar
guyang3532 committed
355
356
357
358
359
  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
360
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
379
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_docker_image_build:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchvision/smoke_test
    steps:
      - checkout
400
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
      - run:
          name: Build and push Docker image
          no_output_timeout: "1h"
          command: |
            set +x
            echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin
            set -x
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest
            docker push ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push ${image_name}:latest

  smoke_test_win_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
420
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
421
422
423
424
425
426
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
427
428
429
430
431
            CONDA_CHANNEL_FLAGS=""
            if [[ "${PYTHON_VERSION}" = 3.9 ]]; then
              CONDA_CHANNEL_FLAGS="-c=conda-forge"
            fi
            conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
guyang3532's avatar
guyang3532 committed
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
            conda activate python${PYTHON_VERSION}
            conda install Pillow
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_win_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
450
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
451
452
453
454
455
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
456
457
458
459
460
            CONDA_CHANNEL_FLAGS=""
            if [[ "${PYTHON_VERSION}" = 3.9 ]]; then
              CONDA_CHANNEL_FLAGS="-c=conda-forge"
            fi
            conda create ${CONDA_CHANNEL_FLAGS} -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
guyang3532's avatar
guyang3532 committed
461
462
463
464
465
466
467
468
469
470
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

471
472
473
474
475
476
477
  unittest_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout
478
      - designate_upload_channel
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
      image_name: "pytorch/manylinux-cuda101"
517
      PYTHON_VERSION: << parameters.python_version >>
518
519
    steps:
      - checkout
520
      - designate_upload_channel
521
522
523
524
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
525
526
527
528
529
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
530
531
      - run:
          name: Setup
532
          command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
533
534
      - save_cache:
          {% raw %}
535
          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
536
537
538
539
540
541
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
542
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL -e CU_VERSION "${image_name}" .circleci/unittest/linux/scripts/install.sh
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
      - run:
          name: Run tests
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post Process
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
558
      - designate_upload_channel
559
      - install_cuda_compatible_cmath
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
      CUDA_VERSION: "10.1"
597
      PYTHON_VERSION: << parameters.python_version >>
598
599
    steps:
      - checkout
600
      - designate_upload_channel
601
      - install_cuda_compatible_cmath
602
603
604
605
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
606
607
608
609
610
      - restore_cache:
          {% raw %}
          keys:
            - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %} 
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results
Edward Z. Yang's avatar
Edward Z. Yang committed
632

Francisco Massa's avatar
Francisco Massa committed
633
634
635
636
637
638
639
  unittest_macos_cpu:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    resource_class: large
    steps:
      - checkout
640
      - designate_upload_channel
Francisco Massa's avatar
Francisco Massa committed
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
      - run:
          name: Install wget
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
          # Disable brew auto update which is very slow
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

676
677
678
679
680
681
682
  cmake_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout_merge
683
      - designate_upload_channel
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
      - run:
          name: Setup conda
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run: packaging/build_cmake.sh

  cmake_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
      PYTHON_VERSION: << parameters.python_version >>
      PYTORCH_VERSION: << parameters.pytorch_version >>
      UNICODE_ABI: << parameters.unicode_abi >>
      CU_VERSION: << parameters.cu_version >>
    steps:
      - checkout_merge
701
      - designate_upload_channel
702
703
704
705
706
      - run:
          name: Setup conda
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Build torchvision C++ distribution and test
707
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
708
709
710
711

  cmake_macos_cpu:
    <<: *binary_common
    macos:
712
      xcode: "9.4.1"
713
714
    steps:
      - checkout_merge
715
      - designate_upload_channel
716
717
718
719
720
721
722
723
724
725
726
727
728
729
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build cmake
            packaging/build_cmake.sh

  cmake_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout_merge
730
      - designate_upload_channel
731
      - install_cuda_compatible_cmath
732
733
734
735
736
737
738
739
740
741
742
743
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/build_cmake.sh

  cmake_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    steps:
      - checkout_merge
744
      - designate_upload_channel
745
      - install_cuda_compatible_cmath
746
747
748
749
750
751
752
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_cmake.sh

753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
  build_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - designate_upload_channel
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Build docs
          command: |
            set -ex
            tag=${CIRCLE_TAG:1:5}
            VERSION=${tag:-master}
            eval "$(./conda/bin/conda shell.bash hook)"
            conda activate ./env
            pushd docs
            pip install -r requirements.txt
            make html
            popd
      - persist_to_workspace:
          root: ./
          paths:
            - "*"
      - store_artifacts:
          path: ./docs/build/html
          destination: docs

  upload_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - run:
          name: Generate netrc
          command: |
            # set credentials for https pushing
            # requires the org-member context
            cat > ~/.netrc \<<DONE
              machine github.com
              login pytorchbot
              password ${GITHUB_PYTORCHBOT_TOKEN}
            DONE
      - run:
          name: Upload docs
          command: |
            # Don't use "checkout" step since it uses ssh, which cannot git push
            # https://circleci.com/docs/2.0/configuration-reference/#checkout
            set -ex
            tag=${CIRCLE_TAG:1:5}
            target=${tag:-master}
            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target


Edward Z. Yang's avatar
Edward Z. Yang committed
818
819
820
821
822
workflows:
  build:
{%- if True %}
    jobs:
      - circleci_consistency
823
      {{ build_workflows(windows_latest_only=True) }}
824
      - python_lint
825
      - python_type_check
826
      - docstring_parameters_sync
827
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
828
      - torchhub_test
829
      - torch_onnx_test
Edward Z. Yang's avatar
Edward Z. Yang committed
830

831
832
833
  unittest:
    jobs:
      {{ unittest_workflows() }}
834
835
836
837
838

  cmake:
    jobs:
      {{ cmake_workflows() }}

Edward Z. Yang's avatar
Edward Z. Yang committed
839
840
841
842
  nightly:
{%- endif %}
    jobs:
      - circleci_consistency
843
      - python_lint
844
      - python_type_check
845
      - docstring_parameters_sync
846
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
847
      - torchhub_test
848
      - torch_onnx_test
849
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
guyang3532's avatar
guyang3532 committed
850
851
852
853
854
855
856
857
858
859
860
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
                - master
    jobs:
      - smoke_test_docker_image_build:
          context: org-member