config.yml 21 KB
Newer Older
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
1
2
3
4
5
6
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
# Adopted from
# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml
7
8
#
# Pro tip: download circle ci cli to validate the config locally during development.
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
9

Min Xu's avatar
Min Xu committed
10
version: 2.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
11
12
13
14

# -------------------------------------------------------------------------------------
# Environments to run the jobs in
# -------------------------------------------------------------------------------------
15
cpu_py37: &cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
16
17
18
19
  docker:
    - image: circleci/python:3.7
  resource_class: medium

20
21
22
23
24
25
26
27
28
29
cpu_py38: &cpu_py38
  docker:
    - image: circleci/python:3.8
  resource_class: medium

cpu_py39: &cpu_py39
  docker:
    - image: circleci/python:3.9
  resource_class: medium

30
31
# Here are list of GPU images:
#   https://circleci.com/docs/2.0/configuration-reference/#available-linux-gpu-images
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
32
33
34
35
36
37
38
gpu: &gpu
  environment:
    CUDA_VERSION: "10.1"
  machine:
    image: ubuntu-1604-cuda-10.1:201909-23
  resource_class: gpu.large

39
40
41
42
43
44
45
gpu_cu111: &gpu_cu111
  environment:
    CUDA_VERSION: "11.1"
  machine:
    image: ubuntu-1604-cuda-11.1:202012-01
  resource_class: gpu.large

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# -------------------------------------------------------------------------------------
# Re-usable commands
# -------------------------------------------------------------------------------------
setup_venv: &setup_venv
  - run:
      name: Setup Virtual Env
      working_directory: ~/
      command: |
        python -m venv ~/venv
        echo ". ~/venv/bin/activate" >> $BASH_ENV
        . ~/venv/bin/activate
        python --version
        which python
        which pip
        pip install --upgrade pip

62
install_dep_151: &install_dep_151
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
63
  - run:
64
      name: Install Dependencies with torch 1.5.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
65
      command: |
66
67
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
68
        sudo apt-get install -y libopenmpi-dev
69
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
70
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.5 && exit 0; fi
71
        # start installing
72
        pip install --progress-bar off torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
73
        pip install --progress-bar off -r requirements-test.txt
74
        pip install --progress-bar off -r requirements-benchmarks.txt
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
75
        python -c 'import torch; print("Torch version:", torch.__version__)'
76
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "5"], "wrong torch version"'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
77
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
78
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
79

80
install_dep_160: &install_dep_160
81
  - run:
82
      name: Install Dependencies with torch 1.6.0
83
      command: |
84
85
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
86
        sudo apt-get install -y libopenmpi-dev
87
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
88
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.6 && exit 0; fi
89
        # start installing
90
        pip install --progress-bar off torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
91
        pip install --progress-bar off -r requirements-test.txt
92
        pip install --progress-bar off -r requirements-benchmarks.txt
93
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
94
        python -c 'import torch; print("Torch version:", torch.__version__)'
95
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "6"], "wrong torch version"'
96
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
97
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
98

99
install_dep_171: &install_dep_171
100
  - run:
101
      name: Install Dependencies with torch 1.7.1
102
      command: |
103
104
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
105
        sudo apt-get install -y libopenmpi-dev
106
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
107
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.7 && exit 0; fi
108
        # start installing
109
        pip install --progress-bar off torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html
110
        pip install --progress-bar off -r requirements-test.txt
111
        pip install --progress-bar off -r requirements-benchmarks.txt
112
113
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
        python -c 'import torch; print("Torch version:", torch.__version__)'
114
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "7"], "wrong torch version"'
115
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
116
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
117

118
119
120
121
install_dep_171_cu110: &install_dep_171_cu110
  - run:
      name: Install Dependencies with torch 1.7.1+cu110
      command: |
122
123
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
124
125
126
        sudo add-apt-repository universe
        sudo apt-get update
        sudo apt-get install -y libopenmpi-dev
127
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
128
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.7 && exit 0; fi
129
        # start installing
130
131
132
133
134
135
136
        pip install --progress-bar off torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
        pip install --progress-bar off -r requirements-benchmarks.txt
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "7"], "wrong torch version"'
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
137
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
138

139
140
141
142
install_dep_180: &install_dep_180
  - run:
      name: Install Dependencies with torch 1.8.0 nightly
      command: |
143
144
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
145
        sudo apt-get install -y libopenmpi-dev
146
        # check if we have restored cache correctly, if so, just skip
Min Xu's avatar
Min Xu committed
147
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.8 && exit 0; fi
148
        # start installing
149
        pip install --progress-bar off -r requirements-test.txt
150
151
        # Since we are using nightly builds, we bypass the benchmarks req file
        # and install ourselves for testing.
152
        #pip install --progress-bar off -r requirements-benchmarks.txt
153
154
155
156
157
158
159
160
        # torchvision nightly wants torch 1.9.
        pip install --pre --progress-bar off torchtext==0.6.0 \
          torchvision==0.9.0.dev20210222+cu112 \
          -f https://download.pytorch.org/whl/nightly/cu112/torch_nightly.html
        # we only use it a bit in benchmarking, so it might be safe to use 1.8.
        pip install --pre --progress-bar off torch==1.8.0.dev20210210+cu112 \
          -f https://download.pytorch.org/whl/nightly/cu112/torch_nightly.html
        pip install --progress-bar off  git+https://github.com/min-xu-ai/torch_pg.git@c723ab4#egg=torch-pg
161
162
163
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "8"], "wrong torch version"'
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
164
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
165

Jun Ru Anderson's avatar
Jun Ru Anderson committed
166
install_repo_cpu: &install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
167
168
169
  - run:
      name: Install Repository
      command: |
170
171
172
        pip install .
        # Test import.
        python -c 'import sys; sys.path = sys.path[1:]; import fairscale'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
173

Jun Ru Anderson's avatar
Jun Ru Anderson committed
174
175
176
177
178
install_repo_gpu: &install_repo_gpu
  - run:
      name: Install Repository
      command: |
        export CUDA_HOME=/usr/local/cuda-10.1
179
        pip install -e .
msbaines's avatar
msbaines committed
180

181
182
183
184
185
186
187
install_repo_gpu_cu111: &install_repo_gpu_cu111
  - run:
      name: Install Repository
      command: |
        export CUDA_HOME=/usr/local/cuda-11.1
        pip install -e .

188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212

run_isort: &run_isort
   - run:
       name: Run Linter (isort)
       command: |
         isort . --check

run_black: &run_black
   - run:
       name: Run Linter (black)
       command: |
         black --check .

run_mypy: &run_mypy
   - run:
       name: Run type-checking (mypy)
       command: |
         mypy --ignore-missing-imports --scripts-are-modules --pretty .

run_flake8: &run_flake8
  - run:
      name: Run Linter (flake8)
      command: |
        flake8 --show-source --statistics

213
214
215
216
217
218
check_test_list: &check_test_list
  - run:
      name: Verify that unit test list files are correct
      command: |
        bash ./tests/ci_test_list_check.sh

219

Min Xu's avatar
Min Xu committed
220
221
222
223
# TODO (Min): figure out how to do coverage nightly or on-demand. Doing it
# on every commit seems like an overkill since we can easily figure out which
# code is not covered without looking at coverage results from each commit.
# Also, it is a long pole for testing time, which slows down development a lot.
msbaines's avatar
msbaines committed
224
225
226
227
run_coverage: &run_coverage
  - run:
      name: Run Unit Tests With Coverage
      command: |
228
        pytest --junitxml=test-results/junit.xml --verbose --timeout 60 --cov-report=xml --cov=./
msbaines's avatar
msbaines committed
229
230
231
        #Uploading test coverage for Python code
        bash <(curl -s https://codecov.io/bash) -f coverage.xml -cF Python

232
233
234
235
run_mpi_unittests: &run_mpi_unittests
  - run:
      name: Run MPI Unit Tests
      command: |
236
        mpirun -n 4 python -m pytest -p torch_pg.pytest --only-mpi --junitxml=test-results/junit.xml --verbose tests/nn/moe
237

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
238

Jun Ru Anderson's avatar
Jun Ru Anderson committed
239
run_pipe_benchmark: &run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
240
  - run:
Jun Ru Anderson's avatar
Jun Ru Anderson committed
241
      name: Run Pipe Benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
242
      command: |
Jun Ru Anderson's avatar
Jun Ru Anderson committed
243
        python benchmarks/pipe.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
244

245
246
247
248
run_mp_pipe_benchmark: &run_mp_pipe_benchmark
  - run:
      name: Run Multiprocess Pipe Benchmark
      command: |
249
        python benchmarks/pipe.py --multiprocess --lazy-construction
250

251
252
253
254
run_oss_benchmark: &run_oss_benchmark
  - run:
      name: Run OSS Benchmark
      command: |
255
        python benchmarks/oss.py --world_size 4 --epochs 2
256
        python benchmarks/oss.py --check_regression --world_size 4 --optim_type oss_sharded_ddp
257
258

run_oss_gloo: &run_oss_gloo
259
260
261
262
263
264
  - run:
      name: Run OSS with Gloo
      command: |
        python benchmarks/oss.py --gloo --optim_type oss_ddp --epochs 2
        python benchmarks/oss.py --gloo --optim_type oss_sharded_ddp --epochs 2

265
run_oss_amp: &run_oss_amp
266
267
268
269
270
   - run:
       name: Run OSS with Torch AMP
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp

271
272
273
274
275
276
run_oss_for_each: &run_oss_for_each
   - run:
       name: Run OSS with Torch AMP and ForEach optmizer
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp --multi_tensor_optim

277
278
279
280
281
282
283
284
285
286

run_doc_build: &run_doc_build
   - run:
       name: Testing doc build
       command: |
         cd docs
         pip install --progress-bar off -r requirements.txt
         make help
         make singlehtml | tee make.out
         ! tail make.out | grep -q warning
287

288
289
290
291
292
293
294
295
# This is an alias to run all unit tests possible on a platform.
run_unittests: &run_unittests
   - run:
       name: Run all unit tests.
       # We run all and not stopping on failure on CPU since docker time is cheaper.
       command: |
         pytest --junitxml=test-results/junit.xml --verbose --timeout 60

Min Xu's avatar
Min Xu committed
296
commands:
297
298
299
300

   # This is a command (like a function) that run tests from a given test_list_file.
   # If test_list_file is not given, this results in an error.
   run_unittests_from_list:
Min Xu's avatar
Min Xu committed
301
     parameters:
302
       test_list_file:
Min Xu's avatar
Min Xu committed
303
         type: string
304
         default: "/dev/non_exist"  # Default to error out
Min Xu's avatar
Min Xu committed
305
306
307
     steps:
       - run:
           name: Run Unit Tests
308
           # we use pytest -x so that it stops on first failure to save GPU time, which is expensive.
Min Xu's avatar
Min Xu committed
309
           command: |
310
311
             if [ ! -f <<parameters.test_list_file>> ]; then exit 1; fi
             pytest -x --junitxml=test-results/junit.xml --verbose --timeout 60 `cat <<parameters.test_list_file>>`
Min Xu's avatar
Min Xu committed
312

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
313
314
315
316
317
# -------------------------------------------------------------------------------------
# Jobs to run
# -------------------------------------------------------------------------------------

jobs:
318
319
  cpu_tests_py37:
    <<: *cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
320
321
322
323
324

    working_directory: ~/fairscale

    steps:
      - checkout
325
      - <<: *check_test_list
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
326
327
328
329
330
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
331
            - cache-key-cpu-py37-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
332

333
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
334
335
336
337

      - save_cache:
          paths:
            - ~/venv
338
          key: cache-key-cpu-py37-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
339

Jun Ru Anderson's avatar
Jun Ru Anderson committed
340
      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
341

342
343
344
345
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
346
      - <<: *run_unittests
347
348
349
350
351
      - <<: *run_mpi_unittests
      - <<: *run_doc_build

      - store_test_results:
          path: test-results
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
352

353
354
  cpu_tests_py38:
    <<: *cpu_py38
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
355

356
    working_directory: ~/fairscale
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
357

358
359
    steps:
      - checkout
360
      - <<: *check_test_list
361
      - <<: *setup_venv
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
362

363
364
365
366
367
368
369
370
371
372
373
374
      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-cpu-py38-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
      - <<: *install_dep_171

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-cpu-py38-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
375

376
377
378
379
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
380
      - <<: *run_unittests
381
      - <<: *run_mpi_unittests
382
      - <<: *run_doc_build
383

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
384
385
386
      - store_test_results:
          path: test-results

387
388
389
390
391
392
393
  cpu_tests_py39:
    <<: *cpu_py39

    working_directory: ~/fairscale

    steps:
      - checkout
394
      - <<: *check_test_list
395
396
397
398
399
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
400
            - cache-key-cpu-py39-180-3-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
401

402
403
404
405
406
      # py3.9 doesn't work well with torch < 1.8. See this PR:
      # https://github.com/pytorch/pytorch/pull/50998
      #
      # Therefore, we test py39 with torch 1.8.0.
      - <<: *install_dep_180
407
408
409
410

      - save_cache:
          paths:
            - ~/venv
411
          key: cache-key-cpu-py39-180-3-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
412
413
414
415
416
417
418

      - <<: *install_repo_cpu

      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
419
      - <<: *run_unittests
420
      - <<: *run_mpi_unittests
421
422
423
424
425
426
      - <<: *run_doc_build

      - store_test_results:
          path: test-results


427
  gpu_tests_151:
Min Xu's avatar
Min Xu committed
428
    parameters:
429
      test_list_file:
Min Xu's avatar
Min Xu committed
430
        type: string
431
        default: "/dev/non_exist"
432

433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
449
            - cache-key-gpu-151-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
450

451
      - <<: *install_dep_151
452
453
454
455

      - save_cache:
          paths:
            - ~/venv
456
          key: cache-key-gpu-151-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
457
458
459

      - <<: *install_repo_gpu

460
461
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
462
463
464
465

      - store_test_results:
          path: test-results

466
  gpu_tests_160:
Min Xu's avatar
Min Xu committed
467
    parameters:
468
      test_list_file:
Min Xu's avatar
Min Xu committed
469
        type: string
470
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
471

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
488
            - cache-key-gpu-160-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
489

490
      - <<: *install_dep_160
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
491
492
493
494

      - save_cache:
          paths:
            - ~/venv
495
          key: cache-key-gpu-160-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
496

Jun Ru Anderson's avatar
Jun Ru Anderson committed
497
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
498

499
500
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
501
502
503
504

      - store_test_results:
          path: test-results

505
  gpu_tests_171:
Min Xu's avatar
Min Xu committed
506
    parameters:
507
      test_list_file:
Min Xu's avatar
Min Xu committed
508
        type: string
509
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
510

511
    <<: *gpu_cu111
512
513
514
515
516
517
518
519

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

520
521
      # Run this to make sure we use python3 from the system.
      - run: pyenv global 3.8.6
522
523
524
525
526
527

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
528
            - cache-key-gpu-cu111-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
529

530
      - <<: *install_dep_171_cu110
531
532
533
534

      - save_cache:
          paths:
            - ~/venv
535
          key: cache-key-gpu-cu111-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
536

537
      - <<: *install_repo_gpu_cu111
538

539
540
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
541
542
543
544

      - store_test_results:
          path: test-results

545
  benchmarks_1:
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
546
547
548
549
550
551
552
553
554
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

555
556
557
558
      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
559
560
561
562
563
564
565
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
566
            - cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
567

568
569
570
571
572
      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

573
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
574
575
576
577

      - save_cache:
          paths:
            - ~/venv
578
          key: cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
579

Jun Ru Anderson's avatar
Jun Ru Anderson committed
580
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
581

Jun Ru Anderson's avatar
Jun Ru Anderson committed
582
      - <<: *run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
583

584
585
      - <<: *run_mp_pipe_benchmark

586
587
588
      - <<: *run_oss_amp

      - <<: *run_oss_for_each
589

590
591
      - <<: *run_oss_gloo

592
593
594
595
596
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

597

598

599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
  benchmarks_2:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

622
623
624
625
626
627

      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

628
629
630
631
632
633
634
635
636
637
      - <<: *install_dep_171

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo_gpu

      - <<: *run_oss_benchmark
638

639
640
641
642
643
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
644
645
646
647
648

workflows:
  version: 2
  build:
    jobs:
649
650
651
      - cpu_tests_py37
      - cpu_tests_py38
      - cpu_tests_py39
Min Xu's avatar
Min Xu committed
652
      - gpu_tests_151:
653
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
654
      - gpu_tests_160:
655
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
656
      - gpu_tests_171:
657
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
658
      - gpu_tests_151:
659
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
660
      - gpu_tests_160:
661
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
662
      - gpu_tests_171:
663
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
664
      - gpu_tests_151:
665
          test_list_file: tests/ci_test_list_3.txt
Min Xu's avatar
Min Xu committed
666
      - gpu_tests_160:
667
          test_list_file: tests/ci_test_list_3.txt
Min Xu's avatar
Min Xu committed
668
      - gpu_tests_171:
669
670
671
          test_list_file: tests/ci_test_list_3.txt
      - benchmarks_1
      - benchmarks_2