config.yml 15 KB
Newer Older
facebook-github-bot's avatar
facebook-github-bot committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
version: 2.1

#examples:
#https://github.com/facebookresearch/ParlAI/blob/master/.circleci/config.yml
#https://github.com/facebookresearch/hydra/blob/master/.circleci/config.yml
#https://github.com/facebookresearch/habitat-api/blob/master/.circleci/config.yml

#drive tests with nox or tox or pytest?

# -------------------------------------------------------------------------------------
# environments where we run our jobs
# -------------------------------------------------------------------------------------


setupcuda: &setupcuda
  run:
    name: Setup CUDA
    working_directory: ~/
    command: |
      # download and install nvidia drivers, cuda, etc
      wget --no-verbose --no-clobber -P ~/nvidia-downloads http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run
      sudo sh ~/nvidia-downloads/cuda_10.2.89_440.33.01_linux.run --silent
      echo "Done installing CUDA."
      pyenv versions
      nvidia-smi
      pyenv global 3.7.0

gpu: &gpu
  environment:
    CUDA_VERSION: "10.2"
  machine:
    image: default
  resource_class: gpu.medium # tesla m60

binary_common: &binary_common
  parameters:
    # Edit these defaults to do a release`
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
      type: string
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
56
      default: "pytorch/manylinux-cuda101"
facebook-github-bot's avatar
facebook-github-bot committed
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    BUILD_VERSION: << parameters.build_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    CU_VERSION: << parameters.cu_version >>

jobs:
  main:
    <<: *gpu
    machine:
      image: ubuntu-1604:201903-01
    steps:
      - checkout
      - <<: *setupcuda
      - run: pip3 install --progress-bar off wheel matplotlib 'pillow<7'
      - run: pip3 install --progress-bar off torch torchvision
      # - run: conda create -p ~/conda_env python=3.7 numpy
      # - run: conda activate ~/conda_env
      # - run: conda install -c pytorch pytorch torchvision

      - run: pip3 install --progress-bar off 'git+https://github.com/facebookresearch/fvcore'
      - run: LD_LIBRARY_PATH=$LD_LIBARY_PATH:/usr/local/cuda-10.2/lib64 python3 setup.py build_ext --inplace
      - run: LD_LIBRARY_PATH=$LD_LIBARY_PATH:/usr/local/cuda-10.2/lib64 python -m unittest discover -v -s tests
      - run: python3 setup.py bdist_wheel

  binary_linux_wheel:
    <<: *binary_common
    docker:
      - image: << parameters.wheel_docker_image >>
    resource_class: 2xlarge+
    steps:
      - checkout
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_linux_conda:
    <<: *binary_common
    docker:
      - image: "pytorch/conda-cuda"
    resource_class: 2xlarge+
    steps:
      - checkout
      # This is building with cuda but no gpu present,
      # so we aren't running the tests.
      - run: TEST_FLAG=--no-test packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"

  binary_linux_conda_cuda:
    <<: *binary_common
    machine:
      image: ubuntu-1604:201903-01
    resource_class: gpu.medium
    steps:
    - checkout
    - run:
        name: Setup environment
        command: |
          set -e

          curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
          curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -

          sudo apt-get update

          sudo apt-get install \
              apt-transport-https \
              ca-certificates \
              curl \
              gnupg-agent \
              software-properties-common

          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

          sudo add-apt-repository \
             "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
             $(lsb_release -cs) \
             stable"

          sudo apt-get update
          export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial"
          sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io=1.2.6-3

          # Add the package repositories
          distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
          curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
          curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list

          export NVIDIA_CONTAINER_VERSION="1.0.3-1"
          sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION}
          sudo systemctl restart docker

Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
158
159
          DRIVER_FN="NVIDIA-Linux-x86_64-450.80.02.run"
          wget "https://us.download.nvidia.com/XFree86/Linux-x86_64/450.80.02/$DRIVER_FN"
facebook-github-bot's avatar
facebook-github-bot committed
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
          sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
          nvidia-smi

    - run:
        name: Pull docker image
        command: |
          set -e
          export DOCKER_IMAGE=pytorch/conda-cuda
          echo Pulling docker image $DOCKER_IMAGE
          docker pull $DOCKER_IMAGE >/dev/null

    - run:
        name: Build and run tests
        command: |
          set -e

          cd ${HOME}/project/

          export DOCKER_IMAGE=pytorch/conda-cuda
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
179
          export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e CU_VERSION"
facebook-github-bot's avatar
facebook-github-bot committed
180
181
182

          docker run --gpus all  --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh

183
184
185
  binary_macos_wheel:
    <<: *binary_common
    macos:
186
      xcode: "9.4.1"
187
188
189
190
191
192
193
194
195
196
197
198
199
200
    steps:
      - checkout
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist

facebook-github-bot's avatar
facebook-github-bot committed
201
202
203
204
205
206
207
workflows:
  version: 2
  build_and_test:
    jobs:
      - main
      - binary_linux_conda:
          cu_version: cu92
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
208
          name: linux_conda_py36_cu92_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
209
210
211
212
          python_version: '3.6'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu101
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
213
          name: linux_conda_py36_cu101_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
214
215
216
217
          python_version: '3.6'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu92
218
          name: linux_conda_py36_cu92_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
219
          python_version: '3.6'
220
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
221
222
      - binary_linux_conda:
          cu_version: cu101
223
          name: linux_conda_py36_cu101_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
224
          python_version: '3.6'
225
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
226
227
      - binary_linux_conda:
          cu_version: cu102
228
          name: linux_conda_py36_cu102_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
229
          python_version: '3.6'
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
          pytorch_version: 1.5.0
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py36_cu92_pyt151
          python_version: '3.6'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py36_cu101_pyt151
          python_version: '3.6'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py36_cu102_pyt151
          python_version: '3.6'
          pytorch_version: 1.5.1
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py36_cu92_pyt160
          python_version: '3.6'
          pytorch_version: 1.6.0
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py36_cu101_pyt160
          python_version: '3.6'
          pytorch_version: 1.6.0
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py36_cu102_pyt160
          python_version: '3.6'
          pytorch_version: 1.6.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py36_cu101_pyt170
          python_version: '3.6'
          pytorch_version: 1.7.0
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py36_cu102_pyt170
          python_version: '3.6'
          pytorch_version: 1.7.0
      - binary_linux_conda:
          cu_version: cu110
          name: linux_conda_py36_cu110_pyt170
          python_version: '3.6'
          pytorch_version: 1.7.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
276
277
278
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py37_cu92_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
279
280
281
282
          python_version: '3.7'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu101
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
283
          name: linux_conda_py37_cu101_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
284
285
286
287
          python_version: '3.7'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu92
288
          name: linux_conda_py37_cu92_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
289
          python_version: '3.7'
290
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
291
292
      - binary_linux_conda:
          cu_version: cu101
293
          name: linux_conda_py37_cu101_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
294
          python_version: '3.7'
295
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
296
297
      - binary_linux_conda:
          cu_version: cu102
298
          name: linux_conda_py37_cu102_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
299
          python_version: '3.7'
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
          pytorch_version: 1.5.0
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py37_cu92_pyt151
          python_version: '3.7'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py37_cu101_pyt151
          python_version: '3.7'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py37_cu102_pyt151
          python_version: '3.7'
          pytorch_version: 1.5.1
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py37_cu92_pyt160
          python_version: '3.7'
          pytorch_version: 1.6.0
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py37_cu101_pyt160
          python_version: '3.7'
          pytorch_version: 1.6.0
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py37_cu102_pyt160
          python_version: '3.7'
          pytorch_version: 1.6.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py37_cu101_pyt170
          python_version: '3.7'
          pytorch_version: 1.7.0
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py37_cu102_pyt170
          python_version: '3.7'
          pytorch_version: 1.7.0
      - binary_linux_conda:
          cu_version: cu110
          name: linux_conda_py37_cu110_pyt170
          python_version: '3.7'
          pytorch_version: 1.7.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
346
347
348
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py38_cu92_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
349
350
351
352
          python_version: '3.8'
          pytorch_version: '1.4'
      - binary_linux_conda:
          cu_version: cu101
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
353
          name: linux_conda_py38_cu101_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
354
355
          python_version: '3.8'
          pytorch_version: '1.4'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
356
357
      - binary_linux_conda:
          cu_version: cu92
358
          name: linux_conda_py38_cu92_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
359
          python_version: '3.8'
360
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
361
362
      - binary_linux_conda:
          cu_version: cu101
363
          name: linux_conda_py38_cu101_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
364
          python_version: '3.8'
365
          pytorch_version: 1.5.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
366
367
      - binary_linux_conda:
          cu_version: cu102
368
          name: linux_conda_py38_cu102_pyt150
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
369
          python_version: '3.8'
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
          pytorch_version: 1.5.0
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py38_cu92_pyt151
          python_version: '3.8'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py38_cu101_pyt151
          python_version: '3.8'
          pytorch_version: 1.5.1
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py38_cu102_pyt151
          python_version: '3.8'
          pytorch_version: 1.5.1
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
      - binary_linux_conda:
          cu_version: cu92
          name: linux_conda_py38_cu92_pyt160
          python_version: '3.8'
          pytorch_version: 1.6.0
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py38_cu101_pyt160
          python_version: '3.8'
          pytorch_version: 1.6.0
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py38_cu102_pyt160
          python_version: '3.8'
          pytorch_version: 1.6.0
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
      - binary_linux_conda:
          cu_version: cu101
          name: linux_conda_py38_cu101_pyt170
          python_version: '3.8'
          pytorch_version: 1.7.0
      - binary_linux_conda:
          cu_version: cu102
          name: linux_conda_py38_cu102_pyt170
          python_version: '3.8'
          pytorch_version: 1.7.0
      - binary_linux_conda:
          cu_version: cu110
          name: linux_conda_py38_cu110_pyt170
          python_version: '3.8'
          pytorch_version: 1.7.0
416
      - binary_linux_wheel:
417
418
          cu_version: cu101
          name: linux_wheel_py36_cu101_pyt160
419
          python_version: '3.6'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
420
          pytorch_version: 1.6.0
421
      - binary_linux_wheel:
422
423
          cu_version: cu101
          name: linux_wheel_py37_cu101_pyt160
424
          python_version: '3.7'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
425
          pytorch_version: 1.6.0
426
      - binary_linux_wheel:
427
428
          cu_version: cu101
          name: linux_wheel_py38_cu101_pyt160
429
          python_version: '3.8'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
430
          pytorch_version: 1.6.0
facebook-github-bot's avatar
facebook-github-bot committed
431
      - binary_linux_conda_cuda:
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
432
          name: testrun_conda_cuda_py37_cu101_pyt14
facebook-github-bot's avatar
facebook-github-bot committed
433
434
          python_version: "3.7"
          pytorch_version: "1.4"
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
435
          cu_version: "cu101"
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
436
      - binary_linux_conda_cuda:
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
437
          name: testrun_conda_cuda_py37_cu102_pyt160
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
438
          python_version: "3.7"
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
439
          pytorch_version: '1.6.0'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
440
          cu_version: "cu102"
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
441
442
443
444
445
      - binary_linux_conda_cuda:
          name: testrun_conda_cuda_py37_cu110_pyt170
          python_version: "3.7"
          pytorch_version: '1.7.0'
          cu_version: "cu110"
446
447
      - binary_macos_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
448
          name: macos_wheel_py36_cpu
449
          python_version: '3.6'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
450
          pytorch_version: '1.6.0'
451
452
      - binary_macos_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
453
          name: macos_wheel_py37_cpu
454
          python_version: '3.7'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
455
          pytorch_version: '1.6.0'
456
457
      - binary_macos_wheel:
          cu_version: cpu
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
458
          name: macos_wheel_py38_cpu
459
          python_version: '3.8'
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
460
          pytorch_version: '1.6.0'