config.yml 7.93 KB
Newer Older
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
# Adopted from
# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml

version: 2

# -------------------------------------------------------------------------------------
# Environments to run the jobs in
# -------------------------------------------------------------------------------------
cpu: &cpu
  docker:
    - image: circleci/python:3.7
  resource_class: medium

gpu: &gpu
  environment:
    CUDA_VERSION: "10.1"
  machine:
    image: ubuntu-1604-cuda-10.1:201909-23
  resource_class: gpu.large

# -------------------------------------------------------------------------------------
# Re-usable commands
# -------------------------------------------------------------------------------------
setup_venv: &setup_venv
  - run:
      name: Setup Virtual Env
      working_directory: ~/
      command: |
        python -m venv ~/venv
        echo ". ~/venv/bin/activate" >> $BASH_ENV
        . ~/venv/bin/activate
        python --version
        which python
        which pip
        pip install --upgrade pip

41
install_dep_15: &install_dep_15
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
42
43
44
  - run:
      name: Install Dependencies
      command: |
45
        sudo apt-get install -y libopenmpi-dev
46
        pip install --progress-bar off torch==1.5.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
47
48
49
50
        pip install --progress-bar off -r requirements-test.txt
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -m torch.utils.collect_env

51
52
53
54
install_dep_16: &install_dep_16
  - run:
      name: Install Dependencies
      command: |
55
        sudo apt-get install -y libopenmpi-dev
56
57
58
59
60
        pip install --progress-bar off torch==1.6.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -m torch.utils.collect_env

61
62
63
64
65
66
67
68
69
70
install_dep_17: &install_dep_17
  - run:
      name: Install Dependencies
      command: |
        sudo apt-get install -y libopenmpi-dev
        pip install --progress-bar off torch==1.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -m torch.utils.collect_env

Jun Ru Anderson's avatar
Jun Ru Anderson committed
71
install_repo_cpu: &install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
72
73
74
75
76
  - run:
      name: Install Repository
      command: |
        python setup.py build develop

Jun Ru Anderson's avatar
Jun Ru Anderson committed
77
78
79
80
81
82
83
install_repo_gpu: &install_repo_gpu
  - run:
      name: Install Repository
      command: |
        export CUDA_HOME=/usr/local/cuda-10.1
        python setup.py build develop

msbaines's avatar
msbaines committed
84
85
86
87
88
89
90
91
92

run_coverage: &run_coverage
  - run:
      name: Run Unit Tests With Coverage
      command: |
        pytest --cov-report=xml --cov=./
        #Uploading test coverage for Python code
        bash <(curl -s https://codecov.io/bash) -f coverage.xml -cF Python

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
93
94
95
96
97
98
run_unittests: &run_unittests
  - run:
      name: Run Unit Tests
      command: |
        pytest --junitxml=test-results/junit.xml --verbose

99
100
101
102
103
104
run_mpi_unittests: &run_mpi_unittests
  - run:
      name: Run MPI Unit Tests
      command: |
        mpirun -n4 python -m pytest -only-mpi --junitxml=test-results/junit.xml --verbose

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
105
106
107
108
109
110
run_flake8: &run_flake8
  - run:
      name: Run Linter (flake8)
      command: |
        flake8 --show-source --statistics

Jun Ru Anderson's avatar
Jun Ru Anderson committed
111
run_pipe_benchmark: &run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
112
  - run:
Jun Ru Anderson's avatar
Jun Ru Anderson committed
113
      name: Run Pipe Benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
114
      command: |
Jun Ru Anderson's avatar
Jun Ru Anderson committed
115
        python benchmarks/pipe.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
116

117
118
119
120
run_oss_benchmark: &run_oss_benchmark
  - run:
      name: Run OSS Benchmark
      command: |
121
        python benchmarks/oss.py --check_regression --world_size 4 --reference_speed 760 --reference_memory 1120 --reference_loss 0.023
122
123
124
125
126

run_oss_gloo: &run_oss_gloo
- run:
    name: Run OSS with Gloo
    command: |
127
      python benchmarks/oss.py --gloo --optim_type oss_ddp --epochs 3
128

129

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
# -------------------------------------------------------------------------------------
# Jobs to run
# -------------------------------------------------------------------------------------

jobs:
  cpu_tests:
    <<: *cpu

    working_directory: ~/fairscale

    steps:
      - checkout
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-cpu-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

149
      - <<: *install_dep_17
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
150
151
152
153
154
155

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-cpu-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

Jun Ru Anderson's avatar
Jun Ru Anderson committed
156
      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
157
158
159
160

      - run:
          name: Run Linter (isort)
          command: |
161
            isort . --check
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
162
163
164
165
166
167
168
169
170

      - run:
          name: Run Linter (black)
          command: |
            black --check .

      - run:
          name: Run type-checking (mypy)
          command: |
Tom Birch's avatar
Tom Birch committed
171
            mypy --ignore-missing-imports --scripts-are-modules --pretty .
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
172
173
174
175
176
177
178
179

      - <<: *run_flake8

      - <<: *run_unittests

      - store_test_results:
          path: test-results

180
  gpu_tests_15:
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
197
            - cache-key-gpu15-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
198

199
      - <<: *install_dep_15
200
201
202
203

      - save_cache:
          paths:
            - ~/venv
204
          key: cache-key-gpu15-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
205
206
207
208
209
210
211
212
213

      - <<: *install_repo_gpu

      - <<: *run_unittests

      - store_test_results:
          path: test-results

  gpu_tests_16:
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
230
            - cache-key-gpu16-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
231

232
      - <<: *install_dep_16
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
233
234
235
236

      - save_cache:
          paths:
            - ~/venv
237
          key: cache-key-gpu16-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
238

Jun Ru Anderson's avatar
Jun Ru Anderson committed
239
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
240
241
242

      - <<: *run_unittests

msbaines's avatar
msbaines committed
243
244
      - <<: *run_coverage

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
245
246
247
      - store_test_results:
          path: test-results

248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
  gpu_tests_17:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-gpu17-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_dep_17

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-gpu17-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo_gpu

      - <<: *run_unittests

      - store_test_results:
          path: test-results

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
281
282
283
284
285
286
287
288
289
290
  benchmarks:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

291
292
293
294
      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
295
296
297
298
299
300
301
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
302
            - cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
303

304
      - <<: *install_dep_17
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
305
306
307
308

      - save_cache:
          paths:
            - ~/venv
309
          key: cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
310

Jun Ru Anderson's avatar
Jun Ru Anderson committed
311
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
312

Jun Ru Anderson's avatar
Jun Ru Anderson committed
313
      - <<: *run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
314

315
316
      - <<: *run_oss_benchmark

317
318
319
      - <<: *run_oss_gloo


320

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
321
322
323
324
325
326

workflows:
  version: 2
  build:
    jobs:
      - cpu_tests
327
      - gpu_tests_15
328
      - gpu_tests_16
329
      - gpu_tests_17
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
330
      - benchmarks