config.yml 17.8 KB
Newer Older
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
1
2
3
4
5
6
# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
# Adopted from
# https://github.com/facebookresearch/detectron2/blob/master/.circleci/config.yml
7
8
#
# Pro tip: download circle ci cli to validate the config locally during development.
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
9

Min Xu's avatar
Min Xu committed
10
version: 2.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
11
12
13
14

# -------------------------------------------------------------------------------------
# Environments to run the jobs in
# -------------------------------------------------------------------------------------
15
cpu_py37: &cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
16
17
  docker:
    - image: circleci/python:3.7
18
  resource_class: large
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
19

20
21
22
cpu_py38: &cpu_py38
  docker:
    - image: circleci/python:3.8
23
  resource_class: large
24
25
26
27

cpu_py39: &cpu_py39
  docker:
    - image: circleci/python:3.9
28
  resource_class: large
29

30
31
# Here are list of GPU images:
#   https://circleci.com/docs/2.0/configuration-reference/#available-linux-gpu-images
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
32
33
34
gpu: &gpu
  environment:
    CUDA_VERSION: "10.1"
35
    CUDA_HOME: /usr/local/cuda-10.1
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
36
37
38
39
  machine:
    image: ubuntu-1604-cuda-10.1:201909-23
  resource_class: gpu.large

40
41
42
gpu_cu111: &gpu_cu111
  environment:
    CUDA_VERSION: "11.1"
43
    CUDA_HOME: /usr/local/cuda-11.1
44
45
46
47
  machine:
    image: ubuntu-1604-cuda-11.1:202012-01
  resource_class: gpu.large

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# -------------------------------------------------------------------------------------
# Re-usable commands
# -------------------------------------------------------------------------------------
setup_venv: &setup_venv
  - run:
      name: Setup Virtual Env
      working_directory: ~/
      command: |
        python -m venv ~/venv
        echo ". ~/venv/bin/activate" >> $BASH_ENV
        . ~/venv/bin/activate
        python --version
        which python
        which pip
        pip install --upgrade pip

64
install_dep_160: &install_dep_160
65
  - run:
66
      name: Install Dependencies with torch 1.6.0
67
      command: |
68
69
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
70
        sudo apt-get update -y
71
        sudo apt-get install -y libopenmpi-dev
72
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
73
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.6 && exit 0; fi
74
        # start installing
75
        pip install --progress-bar off torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
76
        pip install --progress-bar off -r requirements-test.txt
77
        pip install --progress-bar off -r requirements-benchmarks.txt
78
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
79
        python -c 'import torch; print("Torch version:", torch.__version__)'
80
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "6"], "wrong torch version"'
81
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
82
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
83

84
install_dep_171: &install_dep_171
85
  - run:
86
      name: Install Dependencies with torch 1.7.1
87
      command: |
88
89
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
90
        sudo apt-get update -y
91
        sudo apt-get install -y libopenmpi-dev
92
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
93
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.7 && exit 0; fi
94
        # start installing
95
96
97
98
99
100
101
        pip install --progress-bar off torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
        pip install --progress-bar off -r requirements-test.txt
        pip install --progress-bar off -r requirements-benchmarks.txt
        pip install --progress-bar off git+https://github.com/msbaines/torch_pg.git@c85c96f#egg=torch-pg
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "7"], "wrong torch version"'
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
102
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
103

104
105
install_dep_180: &install_dep_180
  - run:
106
      name: Install Dependencies with torch 1.8.0
107
      command: |
108
109
        # make sure that apt-get retries if needed
        sudo sh -c "echo 'APT::Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries"
110
        sudo apt-get update -y
111
        sudo apt-get install -y libopenmpi-dev
112
        # check if we have restored venv cache (/home/circleci/venv) correctly, if so, just skip
Min Xu's avatar
Min Xu committed
113
        if [ -f /home/circleci/venv/check_version.py ]; then python /home/circleci/venv/check_version.py torch eq 1.8 && exit 0; fi
114
        # start installing
115
        pip install --progress-bar off torch==1.8.0+cu101 torchvision==0.9.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
116
        pip install --progress-bar off -r requirements-test.txt
117
        pip install --progress-bar off -r requirements-benchmarks.txt
118
119
120
        python -c 'import torch; print("Torch version:", torch.__version__)'
        python -c 'import torch; assert torch.__version__.split(".")[:2] == ["1", "8"], "wrong torch version"'
        python -m torch.utils.collect_env
Min Xu's avatar
Min Xu committed
121
        wget -O /home/circleci/venv/check_version.py https://raw.githubusercontent.com/min-xu-ai/check_verion/main/check_version.py
122

123
install_repo: &install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
124
125
126
  - run:
      name: Install Repository
      command: |
127
128
129
        pip install .
        # Test import.
        python -c 'import sys; sys.path = sys.path[1:]; import fairscale'
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
130

131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
run_isort: &run_isort
   - run:
       name: Run Linter (isort)
       command: |
         isort . --check

run_black: &run_black
   - run:
       name: Run Linter (black)
       command: |
         black --check .

run_mypy: &run_mypy
   - run:
       name: Run type-checking (mypy)
       command: |
         mypy --ignore-missing-imports --scripts-are-modules --pretty .

run_flake8: &run_flake8
  - run:
      name: Run Linter (flake8)
      command: |
        flake8 --show-source --statistics

155
156
157
158
159
160
check_test_list: &check_test_list
  - run:
      name: Verify that unit test list files are correct
      command: |
        bash ./tests/ci_test_list_check.sh

161

Min Xu's avatar
Min Xu committed
162
163
164
165
# TODO (Min): figure out how to do coverage nightly or on-demand. Doing it
# on every commit seems like an overkill since we can easily figure out which
# code is not covered without looking at coverage results from each commit.
# Also, it is a long pole for testing time, which slows down development a lot.
msbaines's avatar
msbaines committed
166
167
168
169
run_coverage: &run_coverage
  - run:
      name: Run Unit Tests With Coverage
      command: |
170
        pytest --junitxml=test-results/junit.xml --verbose --timeout 60 --cov-report=xml --cov=./
msbaines's avatar
msbaines committed
171
172
173
        #Uploading test coverage for Python code
        bash <(curl -s https://codecov.io/bash) -f coverage.xml -cF Python

174
175
176
177
run_mpi_unittests: &run_mpi_unittests
  - run:
      name: Run MPI Unit Tests
      command: |
178
        mpirun -n 4 python -m pytest -p torch_pg.pytest --only-mpi --junitxml=test-results/junit.xml --verbose tests/nn/moe
179

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
180

Jun Ru Anderson's avatar
Jun Ru Anderson committed
181
run_pipe_benchmark: &run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
182
  - run:
Jun Ru Anderson's avatar
Jun Ru Anderson committed
183
      name: Run Pipe Benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
184
      command: |
Jun Ru Anderson's avatar
Jun Ru Anderson committed
185
        python benchmarks/pipe.py
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
186

187
188
189
190
run_mp_pipe_benchmark: &run_mp_pipe_benchmark
  - run:
      name: Run Multiprocess Pipe Benchmark
      command: |
191
        python benchmarks/pipe.py --multiprocess --lazy-construction
192

193
194
195
196
run_oss_benchmark: &run_oss_benchmark
  - run:
      name: Run OSS Benchmark
      command: |
197
        python benchmarks/oss.py --world_size 4 --epochs 2
198
        python benchmarks/oss.py --check_regression --world_size 4 --optim_type oss_sharded_ddp
199
200

run_oss_gloo: &run_oss_gloo
201
202
203
204
205
206
  - run:
      name: Run OSS with Gloo
      command: |
        python benchmarks/oss.py --gloo --optim_type oss_ddp --epochs 2
        python benchmarks/oss.py --gloo --optim_type oss_sharded_ddp --epochs 2

207
run_oss_amp: &run_oss_amp
208
209
210
211
212
   - run:
       name: Run OSS with Torch AMP
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp

213
214
215
216
217
218
run_oss_for_each: &run_oss_for_each
   - run:
       name: Run OSS with Torch AMP and ForEach optmizer
       command: |
         python benchmarks/oss.py --amp --epochs 3 --optim_type oss_sharded_ddp --multi_tensor_optim

219
220
221
222
223
224
225
226
227
228

run_doc_build: &run_doc_build
   - run:
       name: Testing doc build
       command: |
         cd docs
         pip install --progress-bar off -r requirements.txt
         make help
         make singlehtml | tee make.out
         ! tail make.out | grep -q warning
229

230
231
232
233
234
235
236
237
# This is an alias to run all unit tests possible on a platform.
run_unittests: &run_unittests
   - run:
       name: Run all unit tests.
       # We run all and not stopping on failure on CPU since docker time is cheaper.
       command: |
         pytest --junitxml=test-results/junit.xml --verbose --timeout 60

Min Xu's avatar
Min Xu committed
238
commands:
239
240
241
242

   # This is a command (like a function) that run tests from a given test_list_file.
   # If test_list_file is not given, this results in an error.
   run_unittests_from_list:
Min Xu's avatar
Min Xu committed
243
     parameters:
244
       test_list_file:
Min Xu's avatar
Min Xu committed
245
         type: string
246
         default: "/dev/non_exist"  # Default to error out
Min Xu's avatar
Min Xu committed
247
248
249
250
     steps:
       - run:
           name: Run Unit Tests
           command: |
251
             if [ ! -f <<parameters.test_list_file>> ]; then exit 1; fi
252
             pytest --junitxml=test-results/junit.xml --verbose --timeout 60 `cat <<parameters.test_list_file>>`
Min Xu's avatar
Min Xu committed
253

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
254
255
256
257
258
# -------------------------------------------------------------------------------------
# Jobs to run
# -------------------------------------------------------------------------------------

jobs:
259
260
  cpu_tests_py37:
    <<: *cpu_py37
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
261
262
263
264
265

    working_directory: ~/fairscale

    steps:
      - checkout
266
      - <<: *check_test_list
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
267
268
269
270
271
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
272
            - cache-key-cpu-py37-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
273

274
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
275
276
277
278

      - save_cache:
          paths:
            - ~/venv
279
          key: cache-key-cpu-py37-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
280

281
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
282

283
284
285
286
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
287
      - <<: *run_unittests
288
289
290
291
292
      - <<: *run_mpi_unittests
      - <<: *run_doc_build

      - store_test_results:
          path: test-results
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
293

294
295
  cpu_tests_py38:
    <<: *cpu_py38
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
296

297
    working_directory: ~/fairscale
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
298

299
300
    steps:
      - checkout
301
      - <<: *check_test_list
302
      - <<: *setup_venv
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
303

304
305
306
      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
307
            - cache-key-cpu-py38-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
308
309
310
311
312
      - <<: *install_dep_171

      - save_cache:
          paths:
            - ~/venv
313
          key: cache-key-cpu-py38-180-1-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
314

315
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
316

317
318
319
320
      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
321
      - <<: *run_unittests
322
      - <<: *run_mpi_unittests
323
      - <<: *run_doc_build
324

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
325
326
327
      - store_test_results:
          path: test-results

328
329
330
331
332
333
334
  cpu_tests_py39:
    <<: *cpu_py39

    working_directory: ~/fairscale

    steps:
      - checkout
335
      - <<: *check_test_list
336
337
338
339
340
      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
341
            - cache-key-cpu-py39-180-4-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
342

343
344
345
346
347
      # py3.9 doesn't work well with torch < 1.8. See this PR:
      # https://github.com/pytorch/pytorch/pull/50998
      #
      # Therefore, we test py39 with torch 1.8.0.
      - <<: *install_dep_180
348
349
350
351

      - save_cache:
          paths:
            - ~/venv
352
          key: cache-key-cpu-py39-180-4-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
353

354
      - <<: *install_repo
355
356
357
358
359

      - <<: *run_isort
      - <<: *run_black
      - <<: *run_mypy
      - <<: *run_flake8
360
      - <<: *run_unittests
361
      # TODO(msb) - <<: *run_mpi_unittests
362
363
364
365
366
367
      - <<: *run_doc_build

      - store_test_results:
          path: test-results


368
  gpu_tests_160:
Min Xu's avatar
Min Xu committed
369
    parameters:
370
      test_list_file:
Min Xu's avatar
Min Xu committed
371
        type: string
372
        default: "/dev/non_exist"
373

374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
390
            - cache-key-gpu-160-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
391

392
      - <<: *install_dep_160
393
394
395
396

      - save_cache:
          paths:
            - ~/venv
397
          key: cache-key-gpu-160-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
398

399
      - <<: *install_repo
400

401
402
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
403
404
405
406

      - store_test_results:
          path: test-results

407
  gpu_tests_171:
Min Xu's avatar
Min Xu committed
408
    parameters:
409
      test_list_file:
Min Xu's avatar
Min Xu committed
410
        type: string
411
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
412

413
    <<: *gpu_cu111
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
414
415
416
417
418
419
420
421

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

422
423
      # Run this to make sure we use python3 from the system.
      - run: pyenv global 3.8.6
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
424
425
426
427
428
429

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
430
            - cache-key-gpu-171-110-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
431

432
      - <<: *install_dep_171
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
433
434
435
436

      - save_cache:
          paths:
            - ~/venv
437
          key: cache-key-gpu-171-110-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
438

439
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
440

441
442
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
443
444
445
446

      - store_test_results:
          path: test-results

447
  gpu_tests_180:
Min Xu's avatar
Min Xu committed
448
    parameters:
449
      test_list_file:
Min Xu's avatar
Min Xu committed
450
        type: string
451
        default: "/dev/non_exist"
Min Xu's avatar
Min Xu committed
452

453
    <<: *gpu
454
455
456
457
458
459
460
461

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

462
      # Run this to make sure we use python3 from the system.
463
      - run: pyenv global 3.7.0
464
465
466
467
468
469

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
470
            - cache-key-gpu-180-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
471

472
      - <<: *install_dep_180
473
474
475
476

      - save_cache:
          paths:
            - ~/venv
477
          key: cache-key-gpu-180-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
478

479
      - <<: *install_repo
480

481
482
      - run_unittests_from_list:
          test_list_file: <<parameters.test_list_file>>
483
484
485
486

      - store_test_results:
          path: test-results

487
  benchmarks_1:
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
488
489
490
491
492
493
494
495
496
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

497
498
499
500
      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
501
502
503
504
505
506
507
      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
508
            - cache-key-benchmarks-180-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
509

510
511
512
513
514
      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

515
      - <<: *install_dep_180
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
516
517
518
519

      - save_cache:
          paths:
            - ~/venv
520
          key: cache-key-benchmarks-180-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
521

522
      - <<: *install_repo
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
523

Jun Ru Anderson's avatar
Jun Ru Anderson committed
524
      - <<: *run_pipe_benchmark
Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
525

526
527
      - <<: *run_mp_pipe_benchmark

528
529
530
      - <<: *run_oss_amp

      - <<: *run_oss_for_each
531

532
533
      - <<: *run_oss_gloo

534
535
536
537
538
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
  benchmarks_2:
    <<: *gpu

    working_directory: ~/fairscale

    steps:
      - checkout

      - run: nvidia-smi

      - run: pyenv uninstall -f 3.7.0

      - run: pyenv install 3.7.0

      - run: pyenv global 3.7.0

      - <<: *setup_venv

      # Cache the venv directory that contains dependencies
      - restore_cache:
          keys:
560
            - cache-key-benchmarks-180-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
561

562
563
564
565
566
567

      # Cache the MNIST directory that contains benchmark data
      - restore_cache:
          keys:
            - cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

568
      - <<: *install_dep_180
569
570
571
572

      - save_cache:
          paths:
            - ~/venv
573
          key: cache-key-benchmarks-180-101-{{ checksum "setup.py"}}-{{ checksum "requirements-test.txt"}}
574

575
      - <<: *install_repo
576
577

      - <<: *run_oss_benchmark
578

579
580
581
582
583
      - save_cache:
          paths:
            - /tmp/MNIST
          key: cache-key-benchmark-MNIST-{{ checksum "benchmarks/datasets/mnist.py"}}

Mandeep Singh Baines's avatar
Mandeep Singh Baines committed
584
585
586
587
588

workflows:
  version: 2
  build:
    jobs:
589
590
591
      - cpu_tests_py37
      - cpu_tests_py38
      - cpu_tests_py39
Min Xu's avatar
Min Xu committed
592
      - gpu_tests_160:
593
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
594
      - gpu_tests_171:
595
          test_list_file: tests/ci_test_list_1.txt
596
597
      - gpu_tests_180:
          test_list_file: tests/ci_test_list_1.txt
Min Xu's avatar
Min Xu committed
598
      - gpu_tests_160:
599
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
600
      - gpu_tests_171:
601
          test_list_file: tests/ci_test_list_2.txt
602
603
      - gpu_tests_180:
          test_list_file: tests/ci_test_list_2.txt
Min Xu's avatar
Min Xu committed
604
      - gpu_tests_160:
605
          test_list_file: tests/ci_test_list_3.txt
Min Xu's avatar
Min Xu committed
606
      - gpu_tests_171:
607
          test_list_file: tests/ci_test_list_3.txt
608
609
      - gpu_tests_180:
          test_list_file: tests/ci_test_list_3.txt
610
611
      - benchmarks_1
      - benchmarks_2