config.yml 8.38 KB
Newer Older
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
# Adopted from
# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml

version: 2

# -------------------------------------------------------------------------------------
# Environments to run the jobs in
# -------------------------------------------------------------------------------------
cpu: &cpu
  docker:
    - image: circleci/python:3.7
  resource_class: medium

gpu: &gpu
  environment:
    CUDA_VERSION: "10.1"
  machine:
    image: ubuntu-1604-cuda-10.1:201909-23
  resource_class: gpu.large

# -------------------------------------------------------------------------------------
# Re-usable commands
# -------------------------------------------------------------------------------------
setup_venv: &setup_venv
  - run:
      name: Setup Virtual Env
      working_directory: ~/
      command: |
        python -m venv ~/venv
        echo ". ~/venv/bin/activate" >> $BASH_ENV
        . ~/venv/bin/activate
        python --version
        which python
        which pip
        pip install --upgrade pip

41
install_dep_15: &install_dep_15
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
42
43
44
  - run:
      name: Install Dependencies
      command: |
45
        sudo apt-get install -y libopenmpi-dev
46
        pip install --progress-bar off torch==1.5.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
47
48
49
50
        pip install --progress-bar off -r requirements-test.txt
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -m torch.utils.collect_env

51
52
53
54
install_dep_16: &install_dep_16
  - run:
      name: Install Dependencies
      command: |
55
        sudo apt-get install -y libopenmpi-dev
56
57
        pip install --progress-bar off torch==1.6.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
58
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
59
60
61
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -m torch.utils.collect_env

62
63
64
65
66
67
68
install_dep_17: &install_dep_17
  - run:
      name: Install Dependencies
      command: |
        sudo apt-get install -y libopenmpi-dev
        pip install --progress-bar off torch==1.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
69
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
70
71
72
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -m torch.utils.collect_env

Jun Ru Anderson's avatar
Jun Ru Anderson committed
73
install_repo_cpu: &install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
74
75
76
77
78
  - run:
      name: Install Repository
      command: |
        python setup.py build develop

Jun Ru Anderson's avatar
Jun Ru Anderson committed
79
80
81
82
83
84
85
install_repo_gpu: &install_repo_gpu
  - run:
      name: Install Repository
      command: |
        export CUDA_HOME=/usr/local/cuda-10.1
        python setup.py build develop

msbaines's avatar
msbaines committed
86
87
88
89
90
91
92
93
94

run_coverage: &run_coverage
  - run:
      name: Run Unit Tests With Coverage
      command: |
        pytest --cov-report=xml --cov=./
        #Uploading test coverage for Python code
        bash <(curl -s https://codecov.io/bash) -f coverage.xml -cF Python

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
95
96
97
98
99
100
run_unittests: &run_unittests
  - run:
      name: Run Unit Tests
      command: |
        pytest --junitxml=test-results/junit.xml --verbose

101
102
103
104
run_mpi_unittests: &run_mpi_unittests
  - run:
      name: Run MPI Unit Tests
      command: |
105
        mpirun -n 4 python -m pytest -p torch_pg.pytest --only-mpi --junitxml=test-results/junit.xml --verbose tests/nn/moe
106

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
107
108
109
110
111
112
run_flake8: &run_flake8
  - run:
      name: Run Linter (flake8)
      command: |
        flake8 --show-source --statistics

Jun Ru Anderson's avatar
Jun Ru Anderson committed
113
run_pipe_benchmark: &run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
114
  - run:
Jun Ru Anderson's avatar
Jun Ru Anderson committed
115
      name: Run Pipe Benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
116
      command: |
Jun Ru Anderson's avatar
Jun Ru Anderson committed
117
        python benchmarks/pipe.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
118

119
120
121
122
run_oss_benchmark: &run_oss_benchmark
  - run:
      name: Run OSS Benchmark
      command: |
123
        python benchmarks/oss.py --check_regression --world_size 4 --reference_speed 760 --reference_memory 1120 --reference_loss 0.023
124
125
126
127
128

run_oss_gloo: &run_oss_gloo
- run:
    name: Run OSS with Gloo
    command: |
129
      python benchmarks/oss.py --gloo --optim_type oss_ddp --epochs 3
130

131
132
133
134
135
run_oss_amp: &run_oss_amp
- run:
    name: Run OSS with Torch AMP
    command: |
      python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp
136

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
# -------------------------------------------------------------------------------------
# Jobs to run
# -------------------------------------------------------------------------------------

jobs:
  cpu_tests:
    <<: *cpu

    working_directory: ~/fairscale

    steps:
      - checkout
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-cpu-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

156
      - <<: *install_dep_17
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
157
158
159
160
161
162

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-cpu-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

Jun Ru Anderson's avatar
Jun Ru Anderson committed
163
      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
164
165
166
167

      - run:
          name: Run Linter (isort)
          command: |
168
            isort . --check
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
169
170
171
172
173
174
175
176
177

      - run:
          name: Run Linter (black)
          command: |
            black --check .

      - run:
          name: Run type-checking (mypy)
          command: |
Tom Birch's avatar
Tom Birch committed
178
            mypy --ignore-missing-imports --scripts-are-modules --pretty .
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
179
180
181
182
183

      - <<: *run_flake8

      - <<: *run_unittests

184
185
      - <<: *run_mpi_unittests

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
186
187
188
      - store_test_results:
          path: test-results

189
  gpu_tests_15:
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
206
            - cache-key-gpu15-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
207

208
      - <<: *install_dep_15
209
210
211
212

      - save_cache:
          paths:
            - ~/venv
213
          key: cache-key-gpu15-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
214
215
216
217
218
219
220
221
222

      - <<: *install_repo_gpu

      - <<: *run_unittests

      - store_test_results:
          path: test-results

  gpu_tests_16:
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
239
            - cache-key-gpu16-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
240

241
      - <<: *install_dep_16
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
242
243
244
245

      - save_cache:
          paths:
            - ~/venv
246
          key: cache-key-gpu16-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
247

Jun Ru Anderson's avatar
Jun Ru Anderson committed
248
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
249
250
251

      - <<: *run_unittests

msbaines's avatar
msbaines committed
252
253
      - <<: *run_coverage

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
254
255
256
      - store_test_results:
          path: test-results

257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
  gpu_tests_17:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-gpu17-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_dep_17

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-gpu17-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo_gpu

      - <<: *run_unittests

      - store_test_results:
          path: test-results

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
290
291
292
293
294
295
296
297
298
299
  benchmarks:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

300
301
302
303
      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
304
305
306
307
308
309
310
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
311
            - cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
312

313
      - <<: *install_dep_17
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
314
315
316
317

      - save_cache:
          paths:
            - ~/venv
318
          key: cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
319

Jun Ru Anderson's avatar
Jun Ru Anderson committed
320
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
321

Jun Ru Anderson's avatar
Jun Ru Anderson committed
322
      - <<: *run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
323

324
325
      - <<: *run_oss_benchmark

326
327
      - <<: *run_oss_gloo

328
329
      - <<: *run_oss_amp

330

331

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
332
333
334
335
336
337

workflows:
  version: 2
  build:
    jobs:
      - cpu_tests
338
      - gpu_tests_15
339
      - gpu_tests_16
340
      - gpu_tests_17
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
341
      - benchmarks