config.yml 12.8 KB
Newer Older
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
1
2
3
4
5
6
7
8
9
10
11
12
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
# Adopted from
# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml

version: 2

# -------------------------------------------------------------------------------------
# Environments to run the jobs in
# -------------------------------------------------------------------------------------
13
cpu_py37: &cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
14
15
16
17
  docker:
    - image: circleci/python:3.7
  resource_class: medium

18
19
20
21
22
23
24
25
26
27
cpu_py38: &cpu_py38
  docker:
    - image: circleci/python:3.8
  resource_class: medium

cpu_py39: &cpu_py39
  docker:
    - image: circleci/python:3.9
  resource_class: medium

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
gpu: &gpu
  environment:
    CUDA_VERSION: "10.1"
  machine:
    image: ubuntu-1604-cuda-10.1:201909-23
  resource_class: gpu.large

# -------------------------------------------------------------------------------------
# Re-usable commands
# -------------------------------------------------------------------------------------
setup_venv: &setup_venv
  - run:
      name: Setup Virtual Env
      working_directory: ~/
      command: |
        python -m venv ~/venv
        echo ". ~/venv/bin/activate" >> $BASH_ENV
        . ~/venv/bin/activate
        python --version
        which python
        which pip
        pip install --upgrade pip

51
install_dep_151: &install_dep_151
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
52
  - run:
53
      name: Install Dependencies with torch 1.5.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
54
      command: |
55
        sudo apt-get install -y libopenmpi-dev
56
        pip install --progress-bar off torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
57
        pip install --progress-bar off -r requirements-test.txt
58
        pip install --progress-bar off -r requirements-benchmarks.txt
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
59
        python -c 'import torch; print("Torch version:", torch.__version__)'
60
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "5"], "wrong torch version"'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
61
62
        python -m torch.utils.collect_env

63
install_dep_160: &install_dep_160
64
  - run:
65
      name: Install Dependencies with torch 1.6.0
66
      command: |
67
        sudo apt-get install -y libopenmpi-dev
68
        pip install --progress-bar off torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
69
        pip install --progress-bar off -r requirements-test.txt
70
        pip install --progress-bar off -r requirements-benchmarks.txt
71
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
72
        python -c 'import torch; print("Torch version:", torch.__version__)'
73
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "6"], "wrong torch version"'
74
75
        python -m torch.utils.collect_env

76
install_dep_171: &install_dep_171
77
  - run:
78
      name: Install Dependencies with torch 1.7.1
79
80
      command: |
        sudo apt-get install -y libopenmpi-dev
81
        pip install --progress-bar off torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html
82
        pip install --progress-bar off -r requirements-test.txt
83
        pip install --progress-bar off -r requirements-benchmarks.txt
84
85
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
        python -c 'import torch; print("Torch version:", torch.__version__)'
86
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "7"], "wrong torch version"'
87
88
        python -m torch.utils.collect_env

89
90
91
92
93
94
95
96
97
98
99
100
101
102
install_dep_180: &install_dep_180
  - run:
      name: Install Dependencies with torch 1.8.0 nightly
      command: |
        sudo apt-get install -y libopenmpi-dev
        pip install --pre --progress-bar off torch==1.8.0.dev20210128+cu110 -f https://download.pytorch.org/whl/nightly/cu110/torch_nightly.html
        pip install --progress-bar off  git+https://github.com/min-xu-ai/torch_pg.git@c723ab4#egg=torch-pg
        pip install --progress-bar off -r requirements-test.txt
        # TODO: We don't use 180 to run benchmark yet, because torchvision is not yet available on py39.
        #pip install --progress-bar off -r requirements-benchmarks.txt
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "8"], "wrong torch version"'
        python -m torch.utils.collect_env

Jun Ru Anderson's avatar
Jun Ru Anderson committed
103
install_repo_cpu: &install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
104
105
106
  - run:
      name: Install Repository
      command: |
107
108
109
        pip install .
        # Test import.
        python -c 'import sys; sys.path = sys.path[1:]; import fairscale'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
110

Jun Ru Anderson's avatar
Jun Ru Anderson committed
111
112
113
114
115
install_repo_gpu: &install_repo_gpu
  - run:
      name: Install Repository
      command: |
        export CUDA_HOME=/usr/local/cuda-10.1
116
        pip install -e .
msbaines's avatar
msbaines committed
117

118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

run_isort: &run_isort
   - run:
       name: Run Linter (isort)
       command: |
         isort . --check

run_black: &run_black
   - run:
       name: Run Linter (black)
       command: |
         black --check .

run_mypy: &run_mypy
   - run:
       name: Run type-checking (mypy)
       command: |
         mypy --ignore-missing-imports --scripts-are-modules --pretty .

run_flake8: &run_flake8
  - run:
      name: Run Linter (flake8)
      command: |
        flake8 --show-source --statistics


msbaines's avatar
msbaines committed
144
145
146
147
run_coverage: &run_coverage
  - run:
      name: Run Unit Tests With Coverage
      command: |
148
        pytest --junitxml=test-results/junit.xml --verbose --timeout 60 --cov-report=xml --cov=./
msbaines's avatar
msbaines committed
149
150
151
        #Uploading test coverage for Python code
        bash <(curl -s https://codecov.io/bash) -f coverage.xml -cF Python

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
152
153
154
155
run_unittests: &run_unittests
  - run:
      name: Run Unit Tests
      command: |
Benjamin Lefaudeux's avatar
Benjamin Lefaudeux committed
156
        pytest --junitxml=test-results/junit.xml --verbose --timeout 60
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
157

158
159
160
161
run_mpi_unittests: &run_mpi_unittests
  - run:
      name: Run MPI Unit Tests
      command: |
162
        mpirun -n 4 python -m pytest -p torch_pg.pytest --only-mpi --junitxml=test-results/junit.xml --verbose tests/nn/moe
163

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
164

Jun Ru Anderson's avatar
Jun Ru Anderson committed
165
run_pipe_benchmark: &run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
166
  - run:
Jun Ru Anderson's avatar
Jun Ru Anderson committed
167
      name: Run Pipe Benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
168
      command: |
Jun Ru Anderson's avatar
Jun Ru Anderson committed
169
        python benchmarks/pipe.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
170

171
172
173
174
175
176
run_mp_pipe_benchmark: &run_mp_pipe_benchmark
  - run:
      name: Run Multiprocess Pipe Benchmark
      command: |
        python benchmarks/pipe.py --multiprocess

177
178
179
180
run_oss_benchmark: &run_oss_benchmark
  - run:
      name: Run OSS Benchmark
      command: |
181
182
        python benchmarks/oss.py --world_size 4 --epochs 2
        python benchmarks/oss.py --check_regression --world_size 4 --optim_type oss_sharded_ddp --reference_speed 660 --reference_memory 930 --reference_loss 0.023
183
184

run_oss_gloo: &run_oss_gloo
185
186
187
188
189
190
  - run:
      name: Run OSS with Gloo
      command: |
        python benchmarks/oss.py --gloo --optim_type oss_ddp --epochs 2
        python benchmarks/oss.py --gloo --optim_type oss_sharded_ddp --epochs 2

191
run_oss_amp: &run_oss_amp
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
   - run:
       name: Run OSS with Torch AMP
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp


run_doc_build: &run_doc_build
   - run:
       name: Testing doc build
       command: |
         cd docs
         pip install --progress-bar off -r requirements.txt
         make help
         make singlehtml | tee make.out
         ! tail make.out | grep -q warning
207

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
208
209
210
211
212
# -------------------------------------------------------------------------------------
# Jobs to run
# -------------------------------------------------------------------------------------

jobs:
213
214
  cpu_tests_py37:
    <<: *cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
215
216
217
218
219
220
221
222
223
224

    working_directory: ~/fairscale

    steps:
      - checkout
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
225
            - cache-key-cpu-py37-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
226

227
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
228
229
230
231

      - save_cache:
          paths:
            - ~/venv
232
          key: cache-key-cpu-py37-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
233

Jun Ru Anderson's avatar
Jun Ru Anderson committed
234
      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
235

236
237
238
239
240
241
242
243
244
245
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
      - <<: *run_unittests
      - <<: *run_mpi_unittests
      - <<: *run_doc_build

      - store_test_results:
          path: test-results
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
246

247
248
  cpu_tests_py38:
    <<: *cpu_py38
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
249

250
    working_directory: ~/fairscale
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
251

252
253
254
    steps:
      - checkout
      - <<: *setup_venv
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
255

256
257
258
259
260
261
262
263
264
265
266
267
268
      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-cpu-py38-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_dep_171

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-cpu-py38-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
269

270
271
272
273
274
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
      - <<: *run_unittests
275
      - <<: *run_mpi_unittests
276
      - <<: *run_doc_build
277

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
278
279
280
      - store_test_results:
          path: test-results

281
282
283
284
285
286
287
288
289
290
291
292
  cpu_tests_py39:
    <<: *cpu_py39

    working_directory: ~/fairscale

    steps:
      - checkout
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
293
            - cache-key-cpu-py39-180-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
294

295
296
297
298
299
      # py3.9 doesn't work well with torch < 1.8. See this PR:
      # https://github.com/pytorch/pytorch/pull/50998
      #
      # Therefore, we test py39 with torch 1.8.0.
      - <<: *install_dep_180
300
301
302
303

      - save_cache:
          paths:
            - ~/venv
304
          key: cache-key-cpu-py39-180-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
305
306
307
308
309
310
311

      - <<: *install_repo_cpu

      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
312
313
      - <<: *run_unittests
      - <<: *run_mpi_unittests
314
315
316
317
318
319
      - <<: *run_doc_build

      - store_test_results:
          path: test-results


320
  gpu_tests_151:
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
337
            - cache-key-gpu-151-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
338

339
      - <<: *install_dep_151
340
341
342
343

      - save_cache:
          paths:
            - ~/venv
344
          key: cache-key-gpu-151-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
345
346
347
348
349
350
351
352

      - <<: *install_repo_gpu

      - <<: *run_unittests

      - store_test_results:
          path: test-results

353
  gpu_tests_160:
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
370
            - cache-key-gpu-160-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
371

372
      - <<: *install_dep_160
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
373
374
375
376

      - save_cache:
          paths:
            - ~/venv
377
          key: cache-key-gpu-160-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
378

Jun Ru Anderson's avatar
Jun Ru Anderson committed
379
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
380
381
382
383
384
385

      - <<: *run_unittests

      - store_test_results:
          path: test-results

386
  gpu_tests_171:
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
403
            - cache-key-gpu-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
404

405
      - <<: *install_dep_171
406
407
408
409

      - save_cache:
          paths:
            - ~/venv
410
          key: cache-key-gpu-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
411
412
413

      - <<: *install_repo_gpu

414
      - <<: *run_coverage
415
416
417
418

      - store_test_results:
          path: test-results

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
419
420
421
422
423
424
425
426
427
428
  benchmarks:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

429
430
431
432
      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
433
434
435
436
437
438
439
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
440
            - cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
441

442
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
443
444
445
446

      - save_cache:
          paths:
            - ~/venv
447
          key: cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
448

Jun Ru Anderson's avatar
Jun Ru Anderson committed
449
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
450

Jun Ru Anderson's avatar
Jun Ru Anderson committed
451
      - <<: *run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
452

453
454
      - <<: *run_mp_pipe_benchmark

455
456
      - <<: *run_oss_benchmark

457
458
      - <<: *run_oss_gloo

459
460
      - <<: *run_oss_amp

461

462

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
463
464
465
466
467

workflows:
  version: 2
  build:
    jobs:
468
469
470
      - cpu_tests_py37
      - cpu_tests_py38
      - cpu_tests_py39
471
472
      - gpu_tests_151
      - gpu_tests_160
473
      - gpu_tests_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
474
      - benchmarks