config.yml 12.6 KB
Newer Older
facebook-github-bot's avatar
facebook-github-bot committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
version: 2.1

#examples:
#https://github.com/facebookresearch/ParlAI/blob/master/.circleci/config.yml
#https://github.com/facebookresearch/hydra/blob/master/.circleci/config.yml
#https://github.com/facebookresearch/habitat-api/blob/master/.circleci/config.yml

#drive tests with nox or tox or pytest?

# -------------------------------------------------------------------------------------
# environments where we run our jobs
# -------------------------------------------------------------------------------------


setupcuda: &setupcuda
  run:
    name: Setup CUDA
    working_directory: ~/
    command: |
      # download and install nvidia drivers, cuda, etc
      wget --no-verbose --no-clobber -P ~/nvidia-downloads 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-430.40.run'
      wget --no-verbose --no-clobber -P ~/nvidia-downloads http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run
      sudo /bin/bash ~/nvidia-downloads/NVIDIA-Linux-x86_64-430.40.run --no-drm -q --ui=none
      sudo sh ~/nvidia-downloads/cuda_10.2.89_440.33.01_linux.run --silent
      echo "Done installing CUDA."
      pyenv versions
      nvidia-smi
      pyenv global 3.7.0

gpu: &gpu
  environment:
    CUDA_VERSION: "10.2"
  machine:
    image: default
  resource_class: gpu.medium # tesla m60

binary_common: &binary_common
  parameters:
    # Edit these defaults to do a release`
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
      type: string
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
      default: "pytorch/manylinux-cuda101"
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    BUILD_VERSION: << parameters.build_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    CU_VERSION: << parameters.cu_version >>

jobs:
  main:
    <<: *gpu
    machine:
      image: ubuntu-1604:201903-01
    steps:
      - checkout
      - <<: *setupcuda
      - run: pip3 install --progress-bar off wheel matplotlib 'pillow<7'
      - run: pip3 install --progress-bar off torch torchvision
      # - run: conda create -p ~/conda_env python=3.7 numpy
      # - run: conda activate ~/conda_env
      # - run: conda install -c pytorch pytorch torchvision

      - run: pip3 install --progress-bar off 'git+https://github.com/facebookresearch/fvcore'
      - run: LD_LIBRARY_PATH=$LD_LIBARY_PATH:/usr/local/cuda-10.2/lib64 python3 setup.py build_ext --inplace
      - run: LD_LIBRARY_PATH=$LD_LIBARY_PATH:/usr/local/cuda-10.2/lib64 python -m unittest discover -v -s tests
      - run: python3 setup.py bdist_wheel

  binary_linux_wheel:
    <<: *binary_common
    docker:
      - image: << parameters.wheel_docker_image >>
    resource_class: 2xlarge+
    steps:
      - checkout
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_linux_conda:
    <<: *binary_common
    docker:
      - image: "pytorch/conda-cuda"
    resource_class: 2xlarge+
    steps:
      - checkout
      # This is building with cuda but no gpu present,
      # so we aren't running the tests.
      - run: TEST_FLAG=--no-test packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"

  binary_linux_conda_cuda:
    <<: *binary_common
    machine:
      image: ubuntu-1604:201903-01
    resource_class: gpu.medium
    steps:
    - checkout
    - run:
        name: Setup environment
        command: |
          set -e

          curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
          curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -

          sudo apt-get update

          sudo apt-get install \
              apt-transport-https \
              ca-certificates \
              curl \
              gnupg-agent \
              software-properties-common

          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

          sudo add-apt-repository \
             "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
             $(lsb_release -cs) \
             stable"

          sudo apt-get update
          export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial"
          sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io=1.2.6-3

          # Add the package repositories
          distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
          curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
          curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list

          export NVIDIA_CONTAINER_VERSION="1.0.3-1"
          sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION}
          sudo systemctl restart docker

160
          DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
facebook-github-bot's avatar
facebook-github-bot committed
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
          wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
          sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
          nvidia-smi

    - run:
        name: Pull docker image
        command: |
          set -e
          export DOCKER_IMAGE=pytorch/conda-cuda
          echo Pulling docker image $DOCKER_IMAGE
          docker pull $DOCKER_IMAGE >/dev/null

    - run:
        name: Build and run tests
        command: |
          set -e

          cd ${HOME}/project/

          export DOCKER_IMAGE=pytorch/conda-cuda
          export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e UNICODE_ABI -e CU_VERSION"

          docker run --gpus all  --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh

185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
  binary_macos_wheel:
    <<: *binary_common
    macos:
      xcode: "9.0"
    steps:
      - checkout
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist

facebook-github-bot's avatar
facebook-github-bot committed
203
204
205
206
207
208
209
workflows:
  version: 2
  build_and_test:
    jobs:
      - main
      - binary_linux_conda:
          cu_version: cu92
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
210
          name: linux_conda_py36_cu92_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
211
212
213
214
          python_version: '3.6'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu100
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
215
          name: linux_conda_py36_cu100_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
216
217
218
219
          python_version: '3.6'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu101
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
220
          name: linux_conda_py36_cu101_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
221
222
223
224
          python_version: '3.6'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu92
225
          name: linux_conda_py36_cu92_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
226
          python_version: '3.6'
227
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
228
229
      - binary_linux_conda:
          cu_version: cu101
230
          name: linux_conda_py36_cu101_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
231
          python_version: '3.6'
232
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
233
234
      - binary_linux_conda:
          cu_version: cu102
235
          name: linux_conda_py36_cu102_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
236
          python_version: '3.6'
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
          pytorch_version: 1.5.0
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py36_cu92_pyt151
          python_version: '3.6'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py36_cu101_pyt151
          python_version: '3.6'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py36_cu102_pyt151
          python_version: '3.6'
          pytorch_version: 1.5.1
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
253
254
255
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py37_cu92_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
256
257
258
259
          python_version: '3.7'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu100
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
260
          name: linux_conda_py37_cu100_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
261
262
263
264
          python_version: '3.7'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu101
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
265
          name: linux_conda_py37_cu101_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
266
267
268
269
          python_version: '3.7'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu92
270
          name: linux_conda_py37_cu92_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
271
          python_version: '3.7'
272
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
273
274
      - binary_linux_conda:
          cu_version: cu101
275
          name: linux_conda_py37_cu101_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
276
          python_version: '3.7'
277
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
278
279
      - binary_linux_conda:
          cu_version: cu102
280
          name: linux_conda_py37_cu102_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
281
          python_version: '3.7'
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
          pytorch_version: 1.5.0
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py37_cu92_pyt151
          python_version: '3.7'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py37_cu101_pyt151
          python_version: '3.7'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py37_cu102_pyt151
          python_version: '3.7'
          pytorch_version: 1.5.1
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
298
299
300
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py38_cu92_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
301
302
303
304
          python_version: '3.8'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu100
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
305
          name: linux_conda_py38_cu100_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
306
307
308
309
          python_version: '3.8'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu101
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
310
          name: linux_conda_py38_cu101_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
311
312
          python_version: '3.8'
          pytorch_version: '1.4'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
313
314
      - binary_linux_conda:
          cu_version: cu92
315
          name: linux_conda_py38_cu92_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
316
          python_version: '3.8'
317
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
318
319
      - binary_linux_conda:
          cu_version: cu101
320
          name: linux_conda_py38_cu101_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
321
          python_version: '3.8'
322
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
323
324
      - binary_linux_conda:
          cu_version: cu102
325
          name: linux_conda_py38_cu102_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
326
          python_version: '3.8'
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
          pytorch_version: 1.5.0
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py38_cu92_pyt151
          python_version: '3.8'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py38_cu101_pyt151
          python_version: '3.8'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py38_cu102_pyt151
          python_version: '3.8'
          pytorch_version: 1.5.1
343
344
      - binary_linux_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
345
          name: linux_wheel_py36_cpu_pyt15
346
          python_version: '3.6'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
347
          pytorch_version: '1.5'
348
349
      - binary_linux_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
350
          name: linux_wheel_py37_cpu_pyt15
351
          python_version: '3.7'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
352
          pytorch_version: '1.5'
353
354
      - binary_linux_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
355
          name: linux_wheel_py38_cpu_pyt15
356
          python_version: '3.8'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
357
          pytorch_version: '1.5'
facebook-github-bot's avatar
facebook-github-bot committed
358
      - binary_linux_conda_cuda:
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
359
          name: testrun_conda_cuda_py37_cu100_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
360
361
          python_version: "3.7"
          pytorch_version: "1.4"
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
362
          cu_version: "cu100"
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
363
      - binary_linux_conda_cuda:
364
          name: testrun_conda_cuda_py37_cu102_pyt151
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
365
          python_version: "3.7"
366
          pytorch_version: "1.5.1"
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
367
          cu_version: "cu102"
368
369
      - binary_macos_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
370
          name: macos_wheel_py36_cpu
371
          python_version: '3.6'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
372
          pytorch_version: '1.5'
373
374
      - binary_macos_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
375
          name: macos_wheel_py37_cpu
376
          python_version: '3.7'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
377
          pytorch_version: '1.5'
378
379
      - binary_macos_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
380
          name: macos_wheel_py38_cpu
381
          python_version: '3.8'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
382
          pytorch_version: '1.5'