config.yml 19.6 KB
Newer Older
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
1
2
3
4
5
6
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
# Adopted from
# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml
7
8
#
# Pro tip: download circle ci cli to validate the config locally during development.
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
9

Min Xu's avatar
Min Xu committed
10
version: 2.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
11
12
13
14

# -------------------------------------------------------------------------------------
# Environments to run the jobs in
# -------------------------------------------------------------------------------------
15
cpu_py37: &cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
16
17
18
19
  docker:
    - image: circleci/python:3.7
  resource_class: medium

20
21
22
23
24
25
26
27
28
29
cpu_py38: &cpu_py38
  docker:
    - image: circleci/python:3.8
  resource_class: medium

cpu_py39: &cpu_py39
  docker:
    - image: circleci/python:3.9
  resource_class: medium

30
31
# Here are list of GPU images:
#   https://circleci.com/docs/2.0/configuration-reference/#available-linux-gpu-images
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
32
33
34
35
36
37
38
gpu: &gpu
  environment:
    CUDA_VERSION: "10.1"
  machine:
    image: ubuntu-1604-cuda-10.1:201909-23
  resource_class: gpu.large

39
40
41
42
43
44
45
gpu_cu111: &gpu_cu111
  environment:
    CUDA_VERSION: "11.1"
  machine:
    image: ubuntu-1604-cuda-11.1:202012-01
  resource_class: gpu.large

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# -------------------------------------------------------------------------------------
# Re-usable commands
# -------------------------------------------------------------------------------------
setup_venv: &setup_venv
  - run:
      name: Setup Virtual Env
      working_directory: ~/
      command: |
        python -m venv ~/venv
        echo ". ~/venv/bin/activate" >> $BASH_ENV
        . ~/venv/bin/activate
        python --version
        which python
        which pip
        pip install --upgrade pip

62
install_dep_151: &install_dep_151
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
63
  - run:
64
      name: Install Dependencies with torch 1.5.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
65
      command: |
66
        sudo apt-get install -y libopenmpi-dev
67
68
69
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
        python -c 'import torch; assert torch.__version__.split(".")[:2] != ["1", "5"]' || exit 0
        # start installing
70
        pip install --progress-bar off torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
71
        pip install --progress-bar off -r requirements-test.txt
72
        pip install --progress-bar off -r requirements-benchmarks.txt
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
73
        python -c 'import torch; print("Torch version:", torch.__version__)'
74
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "5"], "wrong torch version"'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
75
76
        python -m torch.utils.collect_env

77
install_dep_160: &install_dep_160
78
  - run:
79
      name: Install Dependencies with torch 1.6.0
80
      command: |
81
        sudo apt-get install -y libopenmpi-dev
82
83
84
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
        python -c 'import torch; assert torch.__version__.split(".")[:2] != ["1", "6"]' || exit 0
        # start installing
85
        pip install --progress-bar off torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
86
        pip install --progress-bar off -r requirements-test.txt
87
        pip install --progress-bar off -r requirements-benchmarks.txt
88
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
89
        python -c 'import torch; print("Torch version:", torch.__version__)'
90
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "6"], "wrong torch version"'
91
92
        python -m torch.utils.collect_env

93
install_dep_171: &install_dep_171
94
  - run:
95
      name: Install Dependencies with torch 1.7.1
96
97
      command: |
        sudo apt-get install -y libopenmpi-dev
98
99
100
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
        python -c 'import torch; assert torch.__version__.split(".")[:2] != ["1", "7"]' || exit 0
        # start installing
101
        pip install --progress-bar off torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html
102
        pip install --progress-bar off -r requirements-test.txt
103
        pip install --progress-bar off -r requirements-benchmarks.txt
104
105
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
        python -c 'import torch; print("Torch version:", torch.__version__)'
106
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "7"], "wrong torch version"'
107
108
        python -m torch.utils.collect_env

109
110
111
112
113
114
115
install_dep_171_cu110: &install_dep_171_cu110
  - run:
      name: Install Dependencies with torch 1.7.1+cu110
      command: |
        sudo add-apt-repository universe
        sudo apt-get update
        sudo apt-get install -y libopenmpi-dev
116
117
118
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
        python -c 'import torch; assert torch.__version__.split(".")[:2] != ["1", "7"]' || exit 0
        # start installing
119
120
121
122
123
124
125
126
        pip install --progress-bar off torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
        pip install --progress-bar off -r requirements-benchmarks.txt
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "7"], "wrong torch version"'
        python -m torch.utils.collect_env

127
128
129
130
131
install_dep_180: &install_dep_180
  - run:
      name: Install Dependencies with torch 1.8.0 nightly
      command: |
        sudo apt-get install -y libopenmpi-dev
132
133
134
        # check if we have restored cache correctly, if so, just skip
        python -c 'import torch; assert torch.__version__.split(".")[:2] != ["1", "8"]' || exit 0
        # start installing
135
        pip install --progress-bar off -r requirements-test.txt
136
137
        # Since we are using nightly builds, we bypass the benchmarks req file
        # and install ourselves for testing.
138
        #pip install --progress-bar off -r requirements-benchmarks.txt
139
140
141
142
143
144
145
146
        # torchvision nightly wants torch 1.9.
        pip install --pre --progress-bar off torchtext==0.6.0 \
          torchvision==0.9.0.dev20210222+cu112 \
          -f https://download.pytorch.org/whl/nightly/cu112/torch_nightly.html
        # we only use it a bit in benchmarking, so it might be safe to use 1.8.
        pip install --pre --progress-bar off torch==1.8.0.dev20210210+cu112 \
          -f https://download.pytorch.org/whl/nightly/cu112/torch_nightly.html
        pip install --progress-bar off  git+https://github.com/min-xu-ai/torch_pg.git@c723ab4#egg=torch-pg
147
148
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "8"], "wrong torch version"'
149
        pip list|grep torch
150
151
        python -m torch.utils.collect_env

Jun Ru Anderson's avatar
Jun Ru Anderson committed
152
install_repo_cpu: &install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
153
154
155
  - run:
      name: Install Repository
      command: |
156
157
158
        pip install .
        # Test import.
        python -c 'import sys; sys.path = sys.path[1:]; import fairscale'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
159

Jun Ru Anderson's avatar
Jun Ru Anderson committed
160
161
162
163
164
install_repo_gpu: &install_repo_gpu
  - run:
      name: Install Repository
      command: |
        export CUDA_HOME=/usr/local/cuda-10.1
165
        pip install -e .
msbaines's avatar
msbaines committed
166

167
168
169
170
171
172
173
install_repo_gpu_cu111: &install_repo_gpu_cu111
  - run:
      name: Install Repository
      command: |
        export CUDA_HOME=/usr/local/cuda-11.1
        pip install -e .

174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198

run_isort: &run_isort
   - run:
       name: Run Linter (isort)
       command: |
         isort . --check

run_black: &run_black
   - run:
       name: Run Linter (black)
       command: |
         black --check .

run_mypy: &run_mypy
   - run:
       name: Run type-checking (mypy)
       command: |
         mypy --ignore-missing-imports --scripts-are-modules --pretty .

run_flake8: &run_flake8
  - run:
      name: Run Linter (flake8)
      command: |
        flake8 --show-source --statistics

199
200
201
202
203
204
check_test_list: &check_test_list
  - run:
      name: Verify that unit test list files are correct
      command: |
        bash ./tests/ci_test_list_check.sh

205

Min Xu's avatar
Min Xu committed
206
207
208
209
# TODO (Min): figure out how to do coverage nightly or on-demand. Doing it
# on every commit seems like an overkill since we can easily figure out which
# code is not covered without looking at coverage results from each commit.
# Also, it is a long pole for testing time, which slows down development a lot.
msbaines's avatar
msbaines committed
210
211
212
213
run_coverage: &run_coverage
  - run:
      name: Run Unit Tests With Coverage
      command: |
214
        pytest --junitxml=test-results/junit.xml --verbose --timeout 60 --cov-report=xml --cov=./
msbaines's avatar
msbaines committed
215
216
217
        #Uploading test coverage for Python code
        bash <(curl -s https://codecov.io/bash) -f coverage.xml -cF Python

218
219
220
221
run_mpi_unittests: &run_mpi_unittests
  - run:
      name: Run MPI Unit Tests
      command: |
222
        mpirun -n 4 python -m pytest -p torch_pg.pytest --only-mpi --junitxml=test-results/junit.xml --verbose tests/nn/moe
223

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
224

Jun Ru Anderson's avatar
Jun Ru Anderson committed
225
run_pipe_benchmark: &run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
226
  - run:
Jun Ru Anderson's avatar
Jun Ru Anderson committed
227
      name: Run Pipe Benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
228
      command: |
Jun Ru Anderson's avatar
Jun Ru Anderson committed
229
        python benchmarks/pipe.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
230

231
232
233
234
run_mp_pipe_benchmark: &run_mp_pipe_benchmark
  - run:
      name: Run Multiprocess Pipe Benchmark
      command: |
235
        python benchmarks/pipe.py --multiprocess --lazy-construction
236

237
238
239
240
run_oss_benchmark: &run_oss_benchmark
  - run:
      name: Run OSS Benchmark
      command: |
241
        python benchmarks/oss.py --world_size 4 --epochs 2
242
        python benchmarks/oss.py --check_regression --world_size 4 --optim_type oss_sharded_ddp
243
244

run_oss_gloo: &run_oss_gloo
245
246
247
248
249
250
  - run:
      name: Run OSS with Gloo
      command: |
        python benchmarks/oss.py --gloo --optim_type oss_ddp --epochs 2
        python benchmarks/oss.py --gloo --optim_type oss_sharded_ddp --epochs 2

251
run_oss_amp: &run_oss_amp
252
253
254
255
256
   - run:
       name: Run OSS with Torch AMP
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp

257
258
259
260
261
262
run_oss_for_each: &run_oss_for_each
   - run:
       name: Run OSS with Torch AMP and ForEach optmizer
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp --multi_tensor_optim

263
264
265
266
267
268
269
270
271
272

run_doc_build: &run_doc_build
   - run:
       name: Testing doc build
       command: |
         cd docs
         pip install --progress-bar off -r requirements.txt
         make help
         make singlehtml | tee make.out
         ! tail make.out | grep -q warning
273

274
275
276
277
278
279
280
281
# This is an alias to run all unit tests possible on a platform.
run_unittests: &run_unittests
   - run:
       name: Run all unit tests.
       # We run all and not stopping on failure on CPU since docker time is cheaper.
       command: |
         pytest --junitxml=test-results/junit.xml --verbose --timeout 60

Min Xu's avatar
Min Xu committed
282
commands:
283
284
285
286

   # This is a command (like a function) that run tests from a given test_list_file.
   # If test_list_file is not given, this results in an error.
   run_unittests_from_list:
Min Xu's avatar
Min Xu committed
287
     parameters:
288
       test_list_file:
Min Xu's avatar
Min Xu committed
289
         type: string
290
         default: "/dev/non_exist"  # Default to error out
Min Xu's avatar
Min Xu committed
291
292
293
     steps:
       - run:
           name: Run Unit Tests
294
           # we use pytest -x so that it stops on first failure to save GPU time, which is expensive.
Min Xu's avatar
Min Xu committed
295
           command: |
296
297
             if [ ! -f <<parameters.test_list_file>> ]; then exit 1; fi
             pytest -x --junitxml=test-results/junit.xml --verbose --timeout 60 `cat <<parameters.test_list_file>>`
Min Xu's avatar
Min Xu committed
298

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
299
300
301
302
303
# -------------------------------------------------------------------------------------
# Jobs to run
# -------------------------------------------------------------------------------------

jobs:
304
305
  cpu_tests_py37:
    <<: *cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
306
307
308
309
310

    working_directory: ~/fairscale

    steps:
      - checkout
311
      - <<: *check_test_list
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
312
313
314
315
316
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
317
            - cache-key-cpu-py37-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
318

319
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
320
321
322
323

      - save_cache:
          paths:
            - ~/venv
324
          key: cache-key-cpu-py37-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
325

Jun Ru Anderson's avatar
Jun Ru Anderson committed
326
      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
327

328
329
330
331
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
332
      - <<: *run_unittests
333
334
335
336
337
      - <<: *run_mpi_unittests
      - <<: *run_doc_build

      - store_test_results:
          path: test-results
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
338

339
340
  cpu_tests_py38:
    <<: *cpu_py38
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
341

342
    working_directory: ~/fairscale
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
343

344
345
    steps:
      - checkout
346
      - <<: *check_test_list
347
      - <<: *setup_venv
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
348

349
350
351
352
353
354
355
356
357
358
359
360
      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-cpu-py38-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
      - <<: *install_dep_171

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-cpu-py38-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo_cpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
361

362
363
364
365
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
366
      - <<: *run_unittests
367
      - <<: *run_mpi_unittests
368
      - <<: *run_doc_build
369

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
370
371
372
      - store_test_results:
          path: test-results

373
374
375
376
377
378
379
  cpu_tests_py39:
    <<: *cpu_py39

    working_directory: ~/fairscale

    steps:
      - checkout
380
      - <<: *check_test_list
381
382
383
384
385
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
386
            - cache-key-cpu-py39-180-3-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
387

388
389
390
391
392
      # py3.9 doesn't work well with torch < 1.8. See this PR:
      # https://github.com/pytorch/pytorch/pull/50998
      #
      # Therefore, we test py39 with torch 1.8.0.
      - <<: *install_dep_180
393
394
395
396

      - save_cache:
          paths:
            - ~/venv
397
          key: cache-key-cpu-py39-180-3-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
398
399
400
401
402
403
404

      - <<: *install_repo_cpu

      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
405
      - <<: *run_unittests
406
      - <<: *run_mpi_unittests
407
408
409
410
411
412
      - <<: *run_doc_build

      - store_test_results:
          path: test-results


413
  gpu_tests_151:
Min Xu's avatar
Min Xu committed
414
    parameters:
415
      test_list_file:
Min Xu's avatar
Min Xu committed
416
        type: string
417
        default: "/dev/non_exist"
418

419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
435
            - cache-key-gpu-151-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
436

437
      - <<: *install_dep_151
438
439
440
441

      - save_cache:
          paths:
            - ~/venv
442
          key: cache-key-gpu-151-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
443
444
445

      - <<: *install_repo_gpu

446
447
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
448
449
450
451

      - store_test_results:
          path: test-results

452
  gpu_tests_160:
Min Xu's avatar
Min Xu committed
453
    parameters:
454
      test_list_file:
Min Xu's avatar
Min Xu committed
455
        type: string
456
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
457

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
474
            - cache-key-gpu-160-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
475

476
      - <<: *install_dep_160
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
477
478
479
480

      - save_cache:
          paths:
            - ~/venv
481
          key: cache-key-gpu-160-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
482

Jun Ru Anderson's avatar
Jun Ru Anderson committed
483
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
484

485
486
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
487
488
489
490

      - store_test_results:
          path: test-results

491
  gpu_tests_171:
Min Xu's avatar
Min Xu committed
492
    parameters:
493
      test_list_file:
Min Xu's avatar
Min Xu committed
494
        type: string
495
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
496

497
    <<: *gpu_cu111
498
499
500
501
502
503
504
505

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

506
507
      # Run this to make sure we use python3 from the system.
      - run: pyenv global 3.8.6
508
509
510
511
512
513

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
514
            - cache-key-gpu-cu111-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
515

516
      - <<: *install_dep_171_cu110
517
518
519
520

      - save_cache:
          paths:
            - ~/venv
521
          key: cache-key-gpu-cu111-171-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
522

523
      - <<: *install_repo_gpu_cu111
524

525
526
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
527
528
529
530

      - store_test_results:
          path: test-results

531
  benchmarks_1:
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
532
533
534
535
536
537
538
539
540
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

541
542
543
544
      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
545
546
547
548
549
550
551
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
552
            - cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
553

554
555
556
557
558
      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

559
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
560
561
562
563

      - save_cache:
          paths:
            - ~/venv
564
          key: cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
565

Jun Ru Anderson's avatar
Jun Ru Anderson committed
566
      - <<: *install_repo_gpu
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
567

Jun Ru Anderson's avatar
Jun Ru Anderson committed
568
      - <<: *run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
569

570
571
      - <<: *run_mp_pipe_benchmark

572
573
574
      - <<: *run_oss_amp

      - <<: *run_oss_for_each
575

576
577
      - <<: *run_oss_gloo

578
579
580
581
582
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

583

584

585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
  benchmarks_2:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

608
609
610
611
612
613

      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

614
615
616
617
618
619
620
621
622
623
      - <<: *install_dep_171

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-benchmarks-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo_gpu

      - <<: *run_oss_benchmark
624

625
626
627
628
629
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
630
631
632
633
634

workflows:
  version: 2
  build:
    jobs:
635
636
637
      - cpu_tests_py37
      - cpu_tests_py38
      - cpu_tests_py39
Min Xu's avatar
Min Xu committed
638
      - gpu_tests_151:
639
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
640
      - gpu_tests_160:
641
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
642
      - gpu_tests_171:
643
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
644
      - gpu_tests_151:
645
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
646
      - gpu_tests_160:
647
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
648
      - gpu_tests_171:
649
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
650
      - gpu_tests_151:
651
          test_list_file: tests/ci_test_list_3.txt
Min Xu's avatar
Min Xu committed
652
      - gpu_tests_160:
653
          test_list_file: tests/ci_test_list_3.txt
Min Xu's avatar
Min Xu committed
654
      - gpu_tests_171:
655
656
657
          test_list_file: tests/ci_test_list_3.txt
      - benchmarks_1
      - benchmarks_2