config.yml 19.4 KB
Newer Older
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
1
2
3
4
5
6
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
# Adopted from
# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml
7
8
#
# Pro tip: download circle ci cli to validate the config locally during development.
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
9

Min Xu's avatar
Min Xu committed
10
version: 2.1
11
12
orbs:
  codecov: codecov/codecov@1.0.2
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
13
14
15
# -------------------------------------------------------------------------------------
# Environments to run the jobs in
# -------------------------------------------------------------------------------------
16
cpu_py37: &cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
17
18
  docker:
    - image: circleci/python:3.7
19
  resource_class: large
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
20

21
22
23
cpu_py38: &cpu_py38
  docker:
    - image: circleci/python:3.8
24
  resource_class: large
25
26
27
28

cpu_py39: &cpu_py39
  docker:
    - image: circleci/python:3.9
29
  resource_class: large
30

31
32
# Here are list of GPU images:
#   https://circleci.com/docs/2.0/configuration-reference/#available-linux-gpu-images
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
33
34
35
gpu: &gpu
  environment:
    CUDA_VERSION: "10.1"
36
    CUDA_HOME: /usr/local/cuda-10.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
37
38
39
40
  machine:
    image: ubuntu-1604-cuda-10.1:201909-23
  resource_class: gpu.large

41
42
43
gpu_cu111: &gpu_cu111
  environment:
    CUDA_VERSION: "11.1"
44
    CUDA_HOME: /usr/local/cuda-11.1
45
46
47
48
  machine:
    image: ubuntu-1604-cuda-11.1:202012-01
  resource_class: gpu.large

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# -------------------------------------------------------------------------------------
# Re-usable commands
# -------------------------------------------------------------------------------------
setup_venv: &setup_venv
  - run:
      name: Setup Virtual Env
      working_directory: ~/
      command: |
        python -m venv ~/venv
        echo ". ~/venv/bin/activate" >> $BASH_ENV
        . ~/venv/bin/activate
        python --version
        which python
        which pip
        pip install --upgrade pip

65
install_dep_160: &install_dep_160
66
  - run:
67
      name: Install Dependencies with torch 1.6.0
68
      command: |
69
70
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
71
        sudo apt-get update -y
72
        sudo apt-get install -y libopenmpi-dev
73
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
74
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.6 && exit 0; fi
75
        # start installing
76
        pip install --progress-bar off torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
77
        pip install --progress-bar off -r requirements-test.txt
78
        pip install --progress-bar off -r requirements-benchmarks.txt
79
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
80
        python -c 'import torch; print("Torch version:", torch.__version__)'
81
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "6"], "wrong torch version"'
82
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
83
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
84

85
install_dep_171: &install_dep_171
86
  - run:
87
      name: Install Dependencies with torch 1.7.1
88
      command: |
89
90
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
91
        sudo apt-get update -y
92
        sudo apt-get install -y libopenmpi-dev
93
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
94
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.7 && exit 0; fi
95
        # start installing
96
97
98
99
100
101
102
        pip install --progress-bar off torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
        pip install --progress-bar off -r requirements-benchmarks.txt
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "7"], "wrong torch version"'
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
103
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
104

105
install_dep_181: &install_dep_181
106
  - run:
107
      name: Install Dependencies with torch 1.8.1
108
      command: |
109
110
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
111
        sudo apt-get update -y
112
        sudo apt-get install -y libopenmpi-dev
113
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
114
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.8 && exit 0; fi
115
        # start installing
116
        pip install --progress-bar off torch==1.8.1+cu101 torchvision==0.9.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html
117
        pip install --progress-bar off -r requirements-test.txt
118
        pip install --progress-bar off -r requirements-benchmarks.txt
119
120
121
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "8"], "wrong torch version"'
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
122
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
123

124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
install_dep_190: &install_dep_190
  - run:
      name: Install Dependencies with torch 1.9.0
      command: |
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
        sudo apt-get update -y
        sudo apt-get install -y libopenmpi-dev
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.8 && exit 0; fi
        # start installing
        pip install --pre --progress-bar off torch==1.9.0.dev20210330+cu101 torchvision==0.10.0.dev20210330+cu101 -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html
        pip install --progress-bar off -r requirements-test.txt
        pip install --progress-bar off -r requirements-benchmarks.txt
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "9"], "wrong torch version"'
        python -m torch.utils.collect_env
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py

143
install_repo: &install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
144
145
146
  - run:
      name: Install Repository
      command: |
147
148
149
        pip install .
        # Test import.
        python -c 'import sys; sys.path = sys.path[1:]; import fairscale'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
150

151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
run_isort: &run_isort
   - run:
       name: Run Linter (isort)
       command: |
         isort . --check

run_black: &run_black
   - run:
       name: Run Linter (black)
       command: |
         black --check .

run_mypy: &run_mypy
   - run:
       name: Run type-checking (mypy)
       command: |
         mypy --ignore-missing-imports --scripts-are-modules --pretty .

run_flake8: &run_flake8
  - run:
      name: Run Linter (flake8)
      command: |
        flake8 --show-source --statistics

175
176
177
178
179
180
check_test_list: &check_test_list
  - run:
      name: Verify that unit test list files are correct
      command: |
        bash ./tests/ci_test_list_check.sh

181

182
183
184
185
upload_coverage: &upload_coverage
  - codecov/upload:
      file: 'coverage.xml'
      token: $CODECOV_TOKEN
msbaines's avatar
msbaines committed
186

187
188
189
190
run_mpi_unittests: &run_mpi_unittests
  - run:
      name: Run MPI Unit Tests
      command: |
191
        mpirun -n 4 python -m pytest -p torch_pg.pytest --only-mpi --junitxml=test-results/junit.xml --verbose tests/nn/moe
192

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
193

Jun Ru Anderson's avatar
Jun Ru Anderson committed
194
run_pipe_benchmark: &run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
195
  - run:
Jun Ru Anderson's avatar
Jun Ru Anderson committed
196
      name: Run Pipe Benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
197
      command: |
Jun Ru Anderson's avatar
Jun Ru Anderson committed
198
        python benchmarks/pipe.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
199

200
201
202
203
run_oss_benchmark: &run_oss_benchmark
  - run:
      name: Run OSS Benchmark
      command: |
204
        python benchmarks/oss.py --world_size 4 --epochs 2
205
        python benchmarks/oss.py --check_regression --world_size 4 --optim_type oss_sharded_ddp
206
207

run_oss_gloo: &run_oss_gloo
208
209
210
211
212
213
  - run:
      name: Run OSS with Gloo
      command: |
        python benchmarks/oss.py --gloo --optim_type oss_ddp --epochs 2
        python benchmarks/oss.py --gloo --optim_type oss_sharded_ddp --epochs 2

214
run_oss_amp: &run_oss_amp
215
216
217
218
219
   - run:
       name: Run OSS with Torch AMP
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp

220
221
222
223
224
225
run_oss_for_each: &run_oss_for_each
   - run:
       name: Run OSS with Torch AMP and ForEach optmizer
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp --multi_tensor_optim

226
227
228
229
230
231
232
233
234
235

run_doc_build: &run_doc_build
   - run:
       name: Testing doc build
       command: |
         cd docs
         pip install --progress-bar off -r requirements.txt
         make help
         make singlehtml | tee make.out
         ! tail make.out | grep -q warning
236

237
238
239
240
241
242
# This is an alias to run all unit tests possible on a platform.
run_unittests: &run_unittests
   - run:
       name: Run all unit tests.
       # We run all and not stopping on failure on CPU since docker time is cheaper.
       command: |
243
         pytest --junitxml=test-results/junit.xml --verbose --timeout 60 --cov-report=xml --cov=./
244

Min Xu's avatar
Min Xu committed
245
commands:
246
247
248
249

   # This is a command (like a function) that run tests from a given test_list_file.
   # If test_list_file is not given, this results in an error.
   run_unittests_from_list:
Min Xu's avatar
Min Xu committed
250
     parameters:
251
       test_list_file:
Min Xu's avatar
Min Xu committed
252
         type: string
253
         default: "/dev/non_exist"  # Default to error out
Min Xu's avatar
Min Xu committed
254
255
256
257
     steps:
       - run:
           name: Run Unit Tests
           command: |
258
             if [ ! -f <<parameters.test_list_file>> ]; then exit 1; fi
259
             pytest --junitxml=test-results/junit.xml --verbose --timeout 60 --cov-report=xml --cov=./ `cat <<parameters.test_list_file>>`
Min Xu's avatar
Min Xu committed
260

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
261
262
263
264
265
# -------------------------------------------------------------------------------------
# Jobs to run
# -------------------------------------------------------------------------------------

jobs:
266
267
  cpu_tests_py37:
    <<: *cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
268
269
270
271
272

    working_directory: ~/fairscale

    steps:
      - checkout
273
      - <<: *check_test_list
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
274
275
276
277
278
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
279
            - cache-key-cpu-py37-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
280

281
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
282
283
284
285

      - save_cache:
          paths:
            - ~/venv
286
          key: cache-key-cpu-py37-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
287

288
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
289

290
291
292
293
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
294
      - <<: *run_unittests
295
296
297
298
299
      - <<: *run_mpi_unittests
      - <<: *run_doc_build

      - store_test_results:
          path: test-results
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
300

301
302
  cpu_tests_py38:
    <<: *cpu_py38
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
303

304
    working_directory: ~/fairscale
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
305

306
307
    steps:
      - checkout
308
      - <<: *check_test_list
309
      - <<: *setup_venv
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
310

311
312
313
      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
314
            - cache-key-cpu-py38-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
315
316
317
318
319
      - <<: *install_dep_171

      - save_cache:
          paths:
            - ~/venv
320
          key: cache-key-cpu-py38-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
321

322
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
323

324
325
326
327
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
328
      - <<: *run_unittests
329
      - <<: *run_mpi_unittests
330
      - <<: *run_doc_build
331

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
332
333
334
      - store_test_results:
          path: test-results

335
336
337
338
339
340
341
  cpu_tests_py39:
    <<: *cpu_py39

    working_directory: ~/fairscale

    steps:
      - checkout
342
      - <<: *check_test_list
343
344
345
346
347
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
348
            - cache-key-cpu-py39-181-0-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
349

350
      - <<: *install_dep_181
351
352
353
354

      - save_cache:
          paths:
            - ~/venv
355
          key: cache-key-cpu-py39-181-0-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
356

357
      - <<: *install_repo
358
359
360
361
362

      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
363
      - <<: *run_unittests
364
      # TODO(msb) - <<: *run_mpi_unittests
365
366
367
368
369
370
      - <<: *run_doc_build

      - store_test_results:
          path: test-results


371
  gpu_tests_160:
Min Xu's avatar
Min Xu committed
372
    parameters:
373
      test_list_file:
Min Xu's avatar
Min Xu committed
374
        type: string
375
        default: "/dev/non_exist"
376

377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
393
            - cache-key-gpu-160-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
394

395
      - <<: *install_dep_160
396
397
398
399

      - save_cache:
          paths:
            - ~/venv
400
          key: cache-key-gpu-160-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
401

402
      - <<: *install_repo
403

404
405
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
406
407
408

      - store_test_results:
          path: test-results
409

410
      - <<: *upload_coverage
411

412
  gpu_tests_171:
Min Xu's avatar
Min Xu committed
413
    parameters:
414
      test_list_file:
Min Xu's avatar
Min Xu committed
415
        type: string
416
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
417

418
    <<: *gpu_cu111
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
419
420
421
422
423
424
425
426

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

427
428
      # Run this to make sure we use python3 from the system.
      - run: pyenv global 3.8.6
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
429
430
431
432
433
434

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
435
            - cache-key-gpu-171-110-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
436

437
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
438
439
440
441

      - save_cache:
          paths:
            - ~/venv
442
          key: cache-key-gpu-171-110-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
443

444
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
445

446
447
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
448
449
450

      - store_test_results:
          path: test-results
451

452
      - <<: *upload_coverage
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
453

454
  gpu_tests_181:
Min Xu's avatar
Min Xu committed
455
    parameters:
456
      test_list_file:
Min Xu's avatar
Min Xu committed
457
        type: string
458
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
459

460
    <<: *gpu
461
462
463
464
465
466
467
468

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

469
      # Run this to make sure we use python3 from the system.
470
      - run: pyenv global 3.7.0
471
472
473
474
475
476

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
477
            - cache-key-gpu-181-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
478

479
      - <<: *install_dep_181
480
481
482
483

      - save_cache:
          paths:
            - ~/venv
484
          key: cache-key-gpu-181-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
485

486
      - <<: *install_repo
487

488
489
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
490
491
492

      - store_test_results:
          path: test-results
493

494
      - <<: *upload_coverage
495

496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
  gpu_tests_190:
    parameters:
      test_list_file:
        type: string
        default: "/dev/non_exist"

    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      # Run this to make sure we use python3 from the system.
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
            - cache-key-gpu-190-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_dep_190

      - save_cache:
          paths:
            - ~/venv
          key: cache-key-gpu-190-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}

      - <<: *install_repo

      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>

      - store_test_results:
          path: test-results

536
  benchmarks_1:
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
537
538
539
540
541
542
543
544
545
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

546
547
548
549
      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
550
551
552
553
554
555
556
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
557
            - cache-key-benchmarks-181-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
558

559
560
561
562
563
      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

564
      - <<: *install_dep_181
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
565
566
567
568

      - save_cache:
          paths:
            - ~/venv
569
          key: cache-key-benchmarks-181-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
570

571
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
572

Jun Ru Anderson's avatar
Jun Ru Anderson committed
573
      - <<: *run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
574

575
576
577
      - <<: *run_oss_amp

      - <<: *run_oss_for_each
578

579
580
      - <<: *run_oss_gloo

581
582
583
584
585
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
  benchmarks_2:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
607
            - cache-key-benchmarks-181-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
608

609
610
611
612
613
614

      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

615
      - <<: *install_dep_181
616
617
618
619

      - save_cache:
          paths:
            - ~/venv
620
          key: cache-key-benchmarks-181-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
621

622
      - <<: *install_repo
623
624

      - <<: *run_oss_benchmark
625

626
627
628
629
630
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
631
632
633
634
635

workflows:
  version: 2
  build:
    jobs:
636
637
638
      - cpu_tests_py37
      - cpu_tests_py38
      - cpu_tests_py39
Min Xu's avatar
Min Xu committed
639
      - gpu_tests_160:
640
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
641
      - gpu_tests_171:
642
          test_list_file: tests/ci_test_list_1.txt
643
      - gpu_tests_181:
644
          test_list_file: tests/ci_test_list_1.txt
645
646
      - gpu_tests_190:
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
647
      - gpu_tests_160:
648
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
649
      - gpu_tests_171:
650
          test_list_file: tests/ci_test_list_2.txt
651
      - gpu_tests_181:
652
          test_list_file: tests/ci_test_list_2.txt
653
654
      - gpu_tests_190:
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
655
      - gpu_tests_160:
656
          test_list_file: tests/ci_test_list_3.txt
Min Xu's avatar
Min Xu committed
657
      - gpu_tests_171:
658
          test_list_file: tests/ci_test_list_3.txt
659
      - gpu_tests_181:
660
          test_list_file: tests/ci_test_list_3.txt
661
662
      - gpu_tests_190:
          test_list_file: tests/ci_test_list_3.txt
663
664
      - benchmarks_1
      - benchmarks_2