config.yml.in 26.4 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
Edward Z. Yang's avatar
Edward Z. Yang committed
5
6
7
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
#       Job names are 'name:' key.
8

9
executors:
10
11
12
13
14
15
16
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
17
    machine:
18
19
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
20
21
      shell: bash.exe

22
23
24
25
26
commands:
  checkout_merge:
    description: "checkout merge branch"
    steps:
      - checkout
27
28
29
30
31
32
33
34
35
#     - run:
#         name: Checkout merge branch
#         command: |
#           set -ex
#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
#           if [[ "$BRANCH" != "master" ]]; then
#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
#             git checkout "merged/$CIRCLE_BRANCH"
#           fi
36
37
38
39
40
41
42
43
44
45
46
47
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
48

49
50
binary_common: &binary_common
  parameters:
51
    # Edit these defaults to do a release
52
53
54
55
56
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
57
      description: "PyTorch version to build against; by default, use a nightly"
58
      type: string
59
      default: ""
60
61
62
63
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
64
65
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
66
      type: string
guyang3532's avatar
guyang3532 committed
67
      default: "cpu"
68
69
70
71
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
Edward Z. Yang's avatar
Edward Z. Yang committed
72
73
74
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
75
      default: "pytorch/manylinux-cuda101"
76
77
78
79
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
Edward Z. Yang's avatar
Edward Z. Yang committed
80
    CU_VERSION: << parameters.cu_version >>
81

guyang3532's avatar
guyang3532 committed
82
83
84
85
86
smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: torchvision/smoke_test:latest

87
88
89
90
91
92
93
94
jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
95
            pip install --user --progress-bar off jinja2 pyyaml
96
97
98
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

99
100
101
102
103
104
105
106
  python_lint:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off flake8 typing
107
108
109
110
111
112
113
114
115
            flake8 --config=setup.cfg .

  python_type_check:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
116
117
            sudo apt-get update -y
            sudo apt install -y libturbojpeg-dev
118
119
            pip install --user --progress-bar off numpy mypy
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
120
            pip install --user --progress-bar off --editable .
121
            mypy --config-file mypy.ini
122

123
124
125
126
127
128
129
130
131
132
  docstring_parameters_sync:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user pydocstyle
            pydocstyle

133
134
135
136
137
138
139
  clang_format:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
140
141
142
            curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format
            chmod +x clang-format
            sudo mv clang-format /opt/clang-format
143
            ./.circleci/unittest/linux/scripts/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format
144

Francisco Massa's avatar
Francisco Massa committed
145
146
147
148
149
150
151
152
153
154
155
156
157
  torchhub_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off numpy
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
            python test/test_hub.py

158
159
160
161
162
163
164
165
166
167
168
169
  torch_onnx_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off numpy
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
            pip install --user onnx
170
            pip install --user onnxruntime
171
172
            python test/test_onnx.py

173
174
175
  binary_linux_wheel:
    <<: *binary_common
    docker:
Edward Z. Yang's avatar
Edward Z. Yang committed
176
      - image: << parameters.wheel_docker_image >>
177
178
    resource_class: 2xlarge+
    steps:
179
      - checkout_merge
180
      - designate_upload_channel
181
182
183
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
184
185
186
187
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
188
189
190
191

  binary_linux_conda:
    <<: *binary_common
    docker:
192
      - image: "pytorch/conda-cuda"
193
194
    resource_class: 2xlarge+
    steps:
195
      - checkout_merge
196
      - designate_upload_channel
197
198
199
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
200
201
202
203
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"
204
205
      - store_test_results:
          path: build_results/
206

Francisco Massa's avatar
Francisco Massa committed
207
  binary_win_conda:
208
209
    <<: *binary_common
    executor: windows-cpu
210
211
    steps:
      - checkout_merge
212
      - designate_upload_channel
213
214
215
      - run:
          name: Build conda packages
          command: |
216
217
218
219
220
221
222
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
            packaging/build_conda.sh
223
            rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
224
      - store_artifacts:
225
          path: C:/tools/miniconda3/conda-bld/win-64
226
      - persist_to_workspace:
227
          root: C:/tools/miniconda3/conda-bld/win-64
228
229
230
231
232
          paths:
            - "*"
      - store_test_results:
          path: build_results/

233
  binary_win_wheel:
234
235
    <<: *binary_common
    executor: windows-cpu
236
237
    steps:
      - checkout_merge
238
      - designate_upload_channel
239
240
241
      - run:
          name: Build wheel packages
          command: |
242
243
244
245
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_wheel.sh
246
      - store_artifacts:
247
          path: dist
248
      - persist_to_workspace:
249
          root: dist
250
251
252
253
254
          paths:
            - "*"
      - store_test_results:
          path: build_results/

255
256
257
  binary_macos_wheel:
    <<: *binary_common
    macos:
258
      xcode: "9.4.1"
259
    steps:
260
      - checkout_merge
261
      - designate_upload_channel
262
263
264
265
266
267
268
269
270
271
272
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
273
274
275
276
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
277
278
279
280

  binary_macos_conda:
    <<: *binary_common
    macos:
281
      xcode: "9.4.1"
282
    steps:
283
      - checkout_merge
284
      - designate_upload_channel
285
286
287
288
289
290
291
292
293
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
294
295
296
297
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
298
299
      - store_test_results:
          path: build_results/
300

Edward Z. Yang's avatar
Edward Z. Yang committed
301
302
303
304
305
306
307
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
308
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
309
310
311
312
313
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
314
            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
Edward Z. Yang's avatar
Edward Z. Yang committed
315
316
317

  # Requires org-member context
  binary_wheel_upload:
Edward Z. Yang's avatar
Edward Z. Yang committed
318
319
320
321
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
322
323
324
325
326
    docker:
      - image: circleci/python:3.7
    steps:
      - attach_workspace:
          at: ~/workspace
327
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
328
329
330
331
332
333
334
335
336
337
      - checkout
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
Edward Z. Yang's avatar
Edward Z. Yang committed
338
            for pkg in ~/workspace/*.whl; do
339
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
Edward Z. Yang's avatar
Edward Z. Yang committed
340
            done
Edward Z. Yang's avatar
Edward Z. Yang committed
341

guyang3532's avatar
guyang3532 committed
342
343
344
345
346
  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
347
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
366
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_docker_image_build:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchvision/smoke_test
    steps:
      - checkout
387
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
      - run:
          name: Build and push Docker image
          no_output_timeout: "1h"
          command: |
            set +x
            echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin
            set -x
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest
            docker push ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push ${image_name}:latest

  smoke_test_win_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
407
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            conda install Pillow
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_win_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
433
      - designate_upload_channel
guyang3532's avatar
guyang3532 committed
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

450
451
452
453
454
455
456
  unittest_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout
457
      - designate_upload_channel
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
      image_name: "pytorch/manylinux-cuda101"
496
      PYTHON_VERSION: << parameters.python_version >>
497
498
    steps:
      - checkout
499
      - designate_upload_channel
500
501
502
503
504
505
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - run:
          name: Setup
506
          command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
507
508
509
510
511
512
513
514
515
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
516
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
      - run:
          name: Run tests
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post Process
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
532
      - designate_upload_channel
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
      CUDA_VERSION: "10.1"
570
      PYTHON_VERSION: << parameters.python_version >>
571
572
    steps:
      - checkout
573
      - designate_upload_channel
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results
Edward Z. Yang's avatar
Edward Z. Yang committed
599

Francisco Massa's avatar
Francisco Massa committed
600
601
602
603
604
605
606
  unittest_macos_cpu:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    resource_class: large
    steps:
      - checkout
607
      - designate_upload_channel
Francisco Massa's avatar
Francisco Massa committed
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
      - run:
          name: Install wget
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
          # Disable brew auto update which is very slow
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

643
644
645
646
647
648
649
  cmake_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout_merge
650
      - designate_upload_channel
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
      - run:
          name: Setup conda
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run: packaging/build_cmake.sh

  cmake_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
      PYTHON_VERSION: << parameters.python_version >>
      PYTORCH_VERSION: << parameters.pytorch_version >>
      UNICODE_ABI: << parameters.unicode_abi >>
      CU_VERSION: << parameters.cu_version >>
    steps:
      - checkout_merge
668
      - designate_upload_channel
669
670
671
672
673
      - run:
          name: Setup conda
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Build torchvision C++ distribution and test
674
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
675
676
677
678

  cmake_macos_cpu:
    <<: *binary_common
    macos:
679
      xcode: "9.4.1"
680
681
    steps:
      - checkout_merge
682
      - designate_upload_channel
683
684
685
686
687
688
689
690
691
692
693
694
695
696
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build cmake
            packaging/build_cmake.sh

  cmake_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout_merge
697
      - designate_upload_channel
698
699
700
701
702
703
704
705
706
707
708
709
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/build_cmake.sh

  cmake_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    steps:
      - checkout_merge
710
      - designate_upload_channel
711
712
713
714
715
716
717
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_cmake.sh

718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
  build_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - designate_upload_channel
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Build docs
          command: |
            set -ex
            tag=${CIRCLE_TAG:1:5}
            VERSION=${tag:-master}
            eval "$(./conda/bin/conda shell.bash hook)"
            conda activate ./env
            pushd docs
            pip install -r requirements.txt
            make html
            popd
      - persist_to_workspace:
          root: ./
          paths:
            - "*"
      - store_artifacts:
          path: ./docs/build/html
          destination: docs

  upload_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - run:
          name: Generate netrc
          command: |
            # set credentials for https pushing
            # requires the org-member context
            cat > ~/.netrc \<<DONE
              machine github.com
              login pytorchbot
              password ${GITHUB_PYTORCHBOT_TOKEN}
            DONE
      - run:
          name: Upload docs
          command: |
            # Don't use "checkout" step since it uses ssh, which cannot git push
            # https://circleci.com/docs/2.0/configuration-reference/#checkout
            set -ex
            tag=${CIRCLE_TAG:1:5}
            target=${tag:-master}
            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target


Edward Z. Yang's avatar
Edward Z. Yang committed
783
784
785
786
787
workflows:
  build:
{%- if True %}
    jobs:
      - circleci_consistency
788
      {{ build_workflows(windows_latest_only=True) }}
789
      - python_lint
790
      - python_type_check
791
      - docstring_parameters_sync
792
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
793
      - torchhub_test
794
      - torch_onnx_test
Edward Z. Yang's avatar
Edward Z. Yang committed
795

796
797
798
  unittest:
    jobs:
      {{ unittest_workflows() }}
799
800
801
802
803

  cmake:
    jobs:
      {{ cmake_workflows() }}

Edward Z. Yang's avatar
Edward Z. Yang committed
804
805
806
807
  nightly:
{%- endif %}
    jobs:
      - circleci_consistency
808
      - python_lint
809
      - python_type_check
810
      - docstring_parameters_sync
811
      - clang_format
Francisco Massa's avatar
Francisco Massa committed
812
      - torchhub_test
813
      - torch_onnx_test
814
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
guyang3532's avatar
guyang3532 committed
815
816
817
818
819
820
821
822
823
824
825
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
                - master
    jobs:
      - smoke_test_docker_image_build:
          context: org-member