config.yml.in 13 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
Edward Z. Yang's avatar
Edward Z. Yang committed
5
6
7
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
#       Job names are 'name:' key.
8

9
executors:
10
11
12
13
14
15
16
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
17
    machine:
18
19
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
20
21
      shell: bash.exe

22
23
24
25
26
commands:
  checkout_merge:
    description: "checkout merge branch"
    steps:
      - checkout
27
28
29
30
31
32
33
34
35
#     - run:
#         name: Checkout merge branch
#         command: |
#           set -ex
#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
#           if [[ "$BRANCH" != "master" ]]; then
#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
#             git checkout "merged/$CIRCLE_BRANCH"
#           fi
36
37
38
39
40
41
42
43
44
45
46
47
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
48

49
50
51
52
53
54
55
56
binary_common: &binary_common
  parameters:
    # Edit these defaults to do a release`
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
57
      description: "PyTorch version to build against; by default, use a nightly"
58
      type: string
59
      default: ""
60
61
62
63
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
64
65
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
66
67
68
69
70
      type: string
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
Edward Z. Yang's avatar
Edward Z. Yang committed
71
72
73
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
74
      default: "pytorch/manylinux-cuda101"
75
76
77
78
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
Edward Z. Yang's avatar
Edward Z. Yang committed
79
    CU_VERSION: << parameters.cu_version >>
80
81
82
83
84
85
86
87
88

jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
89
            pip install --user --progress-bar off jinja2 pyyaml
90
91
92
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

93
94
95
96
97
98
99
100
  python_lint:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off flake8 typing
101
102
103
104
105
106
107
108
109
110
111
            flake8 --config=setup.cfg .

  python_type_check:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off numpy mypy
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
112
            pip install --user --progress-bar off --editable .
113
            mypy --config-file mypy.ini
114

115
116
117
118
119
120
121
122
123
124
125
  clang_format:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            sudo apt-get update -y
            sudo apt-get install -y clang-format
            ./travis-scripts/run-clang-format/run-clang-format.py -r torchvision/csrc

126
127
128
  binary_linux_wheel:
    <<: *binary_common
    docker:
Edward Z. Yang's avatar
Edward Z. Yang committed
129
      - image: << parameters.wheel_docker_image >>
130
131
    resource_class: 2xlarge+
    steps:
132
      - checkout_merge
133
134
135
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
136
137
138
139
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
140
141
142
143

  binary_linux_conda:
    <<: *binary_common
    docker:
144
      - image: "pytorch/conda-cuda"
145
146
    resource_class: 2xlarge+
    steps:
147
      - checkout_merge
148
149
150
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
151
152
153
154
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"
155
156
      - store_test_results:
          path: build_results/
157

Francisco Massa's avatar
Francisco Massa committed
158
159
160
161
162
163
  binary_linux_conda_cuda:
    <<: *binary_common
    machine:
      image: ubuntu-1604:201903-01
    resource_class: gpu.medium
    steps:
164
    - checkout_merge
Francisco Massa's avatar
Francisco Massa committed
165
166
167
    - run:
        name: Setup environment
        command: |
168
          set -ex
Francisco Massa's avatar
Francisco Massa committed
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190

          curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
          curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -

          sudo apt-get update

          sudo apt-get install \
              apt-transport-https \
              ca-certificates \
              curl \
              gnupg-agent \
              software-properties-common

          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

          sudo add-apt-repository \
             "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
             $(lsb_release -cs) \
             stable"

          sudo apt-get update
          export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial"
Francisco Massa's avatar
Francisco Massa committed
191
          sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io=1.2.6-3
Francisco Massa's avatar
Francisco Massa committed
192
193
194
195
196
197
198
199
200
201

          # Add the package repositories
          distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
          curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
          curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list

          export NVIDIA_CONTAINER_VERSION="1.0.3-1"
          sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION}
          sudo systemctl restart docker

202
          DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
Francisco Massa's avatar
Francisco Massa committed
203
204
205
206
207
208
209
          wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
          sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
          nvidia-smi

    - run:
        name: Pull docker image
        command: |
210
          set -ex
211
          export DOCKER_IMAGE=pytorch/conda-cuda
Francisco Massa's avatar
Francisco Massa committed
212
213
214
215
216
217
          echo Pulling docker image $DOCKER_IMAGE
          docker pull $DOCKER_IMAGE >/dev/null

    - run:
        name: Build and run tests
        command: |
218
          set -ex
Francisco Massa's avatar
Francisco Massa committed
219
220
221

          cd ${HOME}/project/

222
          export DOCKER_IMAGE=pytorch/conda-cuda
Francisco Massa's avatar
Francisco Massa committed
223
224
225
226
227
228
          export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e UNICODE_ABI -e CU_VERSION"

          docker run --gpus all  --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh

  binary_win_conda:
    <<: *binary_common
229
    executor: windows-cpu
Francisco Massa's avatar
Francisco Massa committed
230
    steps:
231
      - checkout_merge
Francisco Massa's avatar
Francisco Massa committed
232
233
      - run:
          command: |
234
235
236
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
237
238
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
239
            packaging/build_conda.sh
240
241
      - store_test_results:
          path: build_results/
Francisco Massa's avatar
Francisco Massa committed
242

243
244
  binary_win_conda_cuda:
    <<: *binary_common
245
    executor: windows-gpu
246
247
248
249
    steps:
      - checkout_merge
      - run:
          command: |
250
251
252
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
253
254
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
255
            packaging/build_conda.sh
256

257
  binary_win_conda_release:
258
259
    <<: *binary_common
    executor: windows-cpu
260
261
262
263
264
    steps:
      - checkout_merge
      - run:
          name: Build conda packages
          command: |
265
266
267
268
269
270
271
272
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
            packaging/build_conda.sh
            rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2
273
      - store_artifacts:
274
          path: C:/tools/miniconda3/conda-bld/win-64
275
      - persist_to_workspace:
276
          root: C:/tools/miniconda3/conda-bld/win-64
277
278
279
280
281
282
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  binary_win_wheel_release:
283
284
    <<: *binary_common
    executor: windows-cpu
285
286
287
288
289
    steps:
      - checkout_merge
      - run:
          name: Build wheel packages
          command: |
290
291
292
293
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_wheel.sh
294
      - store_artifacts:
295
          path: dist
296
      - persist_to_workspace:
297
          root: dist
298
299
300
301
302
          paths:
            - "*"
      - store_test_results:
          path: build_results/

303
304
305
306
307
  binary_macos_wheel:
    <<: *binary_common
    macos:
      xcode: "9.0"
    steps:
308
      - checkout_merge
309
310
311
312
313
314
315
316
317
318
319
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
320
321
322
323
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
324
325
326
327
328
329

  binary_macos_conda:
    <<: *binary_common
    macos:
      xcode: "9.0"
    steps:
330
      - checkout_merge
331
332
333
334
335
336
337
338
339
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
340
341
342
343
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
344
345
      - store_test_results:
          path: build_results/
346

Edward Z. Yang's avatar
Edward Z. Yang committed
347
348
349
350
351
352
353
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
354
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
355
356
357
358
359
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
360
            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
Edward Z. Yang's avatar
Edward Z. Yang committed
361
362
363

  # Requires org-member context
  binary_wheel_upload:
Edward Z. Yang's avatar
Edward Z. Yang committed
364
365
366
367
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
368
369
370
371
372
    docker:
      - image: circleci/python:3.7
    steps:
      - attach_workspace:
          at: ~/workspace
373
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
374
375
376
377
378
379
380
381
382
383
      - checkout
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
Edward Z. Yang's avatar
Edward Z. Yang committed
384
            for pkg in ~/workspace/*.whl; do
385
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
Edward Z. Yang's avatar
Edward Z. Yang committed
386
            done
Edward Z. Yang's avatar
Edward Z. Yang committed
387
388
389
390
391
392
393


workflows:
  build:
{%- if True %}
    jobs:
      - circleci_consistency
394
      {{ workflows(windows_latest_only=True) }}
395
      - binary_linux_conda_cuda:
396
397
398
          name: torchvision_linux_py3.8_cu102_cuda
          python_version: "3.8"
          cu_version: "cu102"
Francisco Massa's avatar
Francisco Massa committed
399
400
401
402
      - binary_win_conda:
          name: torchvision_win_py3.6_cpu
          python_version: "3.6"
          cu_version: "cpu"
403
404
405
406
      - binary_win_conda_cuda:
          name: torchvision_win_py3.6_cu101
          python_version: "3.6"
          cu_version: "cu101"
407
      - python_lint
408
      - python_type_check
409
      - clang_format
Edward Z. Yang's avatar
Edward Z. Yang committed
410
411
412
413
414

  nightly:
{%- endif %}
    jobs:
      - circleci_consistency
415
      - python_lint
416
      - python_type_check
417
      - clang_format
418
      {{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}