config.yml.in 25.6 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
5
6
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8
#     - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
Edward Z. Yang's avatar
Edward Z. Yang committed
7
#       Job names are 'name:' key.
8

9
10
11
12
13
14
executors:
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe
peterjc123's avatar
peterjc123 committed
15

16
17
18
  windows-gpu:
    machine:
      resource_class: windows.gpu.nvidia.medium
19
      image: windows-server-2019-nvidia:stable
20
21
      shell: bash.exe

22
commands:
23
24
25
26
27
28
  generate_cache_key:
    description: "Generates a cache key file that changes daily"
    steps:
      - run:
          name: Generate cache key
          command: echo "$(date +"%Y-%m-%d")" > .cachekey
29
30
31
32
33
34
35
36
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
37
            if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then
38
39
40
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
Eli Uriegas's avatar
Eli Uriegas committed
41
42
43
44
45
46
47
  load_conda_channel_flags:
    description: "Determines whether we need extra conda channels"
    steps:
      - run:
          name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV
          command: |
              CONDA_CHANNEL_FLAGS=""
48
              # formerly used to add conda-forge flags for Python 3.9, reserving the mechanism for future python upgrades
49
50
51
52
53
54
55
  windows_install_cuda:
    description: "Install desired CUDA version on Windows runners"
    steps:
      - run:
          name: Install CUDA
          command: |
              packaging/windows/internal/cuda_install.bat
56

57
58
binary_common: &binary_common
  parameters:
Edward Z. Yang's avatar
Edward Z. Yang committed
59
    # Edit these defaults to do a release
60
61
62
63
64
65
66
67
68
69
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
70
      description: "Python version to build against (e.g., 3.8)"
71
      type: string
Nikita Shulga's avatar
Nikita Shulga committed
72
73
74
75
76
77
78
79
80
81
82
83
    cuda_version:
      description: "CUDA version to build against (e.g., cpu, cu101)"
      type: string
      default: "cpu"
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
      default: "pytorch/manylinux-cuda102"
    conda_docker_image:
      description: "Conda only: what docker image to use"
      type: string
      default: "pytorch/conda-builder:cuda102"
84
  environment: &environment
85
86
87
    PYTHON_VERSION: << parameters.python_version >>
    BUILD_VERSION: << parameters.build_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
Nikita Shulga's avatar
Nikita Shulga committed
88
    CU_VERSION: << parameters.cuda_version >>
89

Mingbo Wan's avatar
Mingbo Wan committed
90
91
92
smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
93
    - image: pytorch/torchaudio_unittest_base:smoke_test-20220425
Caroline Chen's avatar
Caroline Chen committed
94
  resource_class: large
Mingbo Wan's avatar
Mingbo Wan committed
95

96
97
98
jobs:
  circleci_consistency:
    docker:
moto's avatar
moto committed
99
      - image: cimg/python:3.8
100
101
102
103
    steps:
      - checkout
      - run:
          command: |
104
            pip install --user --progress-bar off jinja2 pyyaml
105
106
107
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

108
109
110
111
112
113
114
115
116
117
118
119
120
  lint_python_and_config:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          name: Install pre-commit
          command: pip install --user --progress-bar off pre-commit
      - run:
          name: Install pre-commit hooks
          command: pre-commit install-hooks
      - run:
          name: Lint Python code and config files
121
          command: pre-commit run --all-files
122
123
      - run:
          name: Required lint modifications
124
          when: always
moto's avatar
moto committed
125
          command: git --no-pager diff --color=always
126

127
128
  download_third_parties_nix:
    docker:
moto's avatar
moto committed
129
      - image: "pytorch/torchaudio_unittest_base:manylinux"
130
131
132
    resource_class: small
    steps:
      - checkout
133
      - generate_cache_key
134
135
136
      - restore_cache:
          {% raw %}
          keys:
moto's avatar
moto committed
137
            - tp-nix-v2-{{ checksum ".cachekey" }}
138
139
          {% endraw %}
      - run:
moto's avatar
moto committed
140
          command: |
141
142
              mkdir -p third_party/archives/
              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/*/CMakeLists.txt)
143
144
      - save_cache:
          {% raw %}
moto's avatar
moto committed
145
          key: tp-nix-v2-{{ checksum ".cachekey" }}
146
147
          {% endraw %}
          paths:
148
            - third_party/archives
149
150
151
      - persist_to_workspace:
          root: third_party
          paths:
152
            - archives
153

154
155
156
  binary_linux_wheel:
    <<: *binary_common
    docker:
Nikita Shulga's avatar
Nikita Shulga committed
157
      - image: << parameters.wheel_docker_image >>
158
159
160
    resource_class: 2xlarge+
    steps:
      - checkout
161
162
      - attach_workspace:
          at: third_party
163
164
165
166
      - run:
          command: |
            ./tools/bootstrap_ffmpeg.sh
            packaging/build_wheel.sh
167
168
          environment:
            BUILD_FFMPEG: true
169
170
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
171
172
173
174
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
175
176
177
178

  binary_linux_conda:
    <<: *binary_common
    docker:
Nikita Shulga's avatar
Nikita Shulga committed
179
      - image: "<< parameters.conda_docker_image >>"
180
181
182
    resource_class: 2xlarge+
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
183
      - load_conda_channel_flags
184
185
      - attach_workspace:
          at: third_party
186
187
188
189
      - run:
          command: packaging/build_conda.sh
          environment:
            BUILD_FFMPEG: true
190
191
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
192
      - persist_to_workspace:
193
          root: /opt/conda
Edward Z. Yang's avatar
Edward Z. Yang committed
194
          paths:
195
            - "conda-bld/*"
196
197
198
199

  binary_macos_wheel:
    <<: *binary_common
    macos:
200
      xcode: "12.0"
201
202
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
203
      - load_conda_channel_flags
204
205
      - attach_workspace:
          at: third_party
206
207
208
209
210
211
212
213
214
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
215
216
          environment:
            BUILD_FFMPEG: true
217
218
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
219
220
221
222
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
223
224
225
226

  binary_macos_conda:
    <<: *binary_common
    macos:
227
      xcode: "12.0"
228
229
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
230
      - load_conda_channel_flags
231
232
      - attach_workspace:
          at: third_party
233
234
235
236
237
238
239
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
240
241
          environment:
            BUILD_FFMPEG: true
242
243
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
244
      - persist_to_workspace:
245
          root: /Users/distiller/miniconda3
Edward Z. Yang's avatar
Edward Z. Yang committed
246
          paths:
247
            - "conda-bld/*"
248

peterjc123's avatar
peterjc123 committed
249
250
251
252
253
254
  binary_windows_wheel:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
255
      - load_conda_channel_flags
256
      - windows_install_cuda
peterjc123's avatar
peterjc123 committed
257
      - run:
258
          name: Build wheel packages
peterjc123's avatar
peterjc123 committed
259
          command: |
260
            set -ex
peterjc123's avatar
peterjc123 committed
261
262
263
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            bash packaging/build_wheel.sh
264
265
          environment:
            BUILD_FFMPEG: true
peterjc123's avatar
peterjc123 committed
266
267
268
269
270
271
272
273
274
275
276
277
278
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_windows_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
279
      - load_conda_channel_flags
280
      - windows_install_cuda
peterjc123's avatar
peterjc123 committed
281
      - run:
282
          name: Build conda packages
283
          no_output_timeout: 20m
peterjc123's avatar
peterjc123 committed
284
          command: |
285
            set -ex
peterjc123's avatar
peterjc123 committed
286
287
288
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
289
290
291
292
            # cudatoolkit >= 11 isn't available for windows in the nvidia channel
            if [[ "${CU_VERSION}" =~ cu11.* ]]; then
              export CONDA_CHANNEL_FLAGS="-c conda-forge"
            fi
peterjc123's avatar
peterjc123 committed
293
            bash packaging/build_conda.sh
294
295
          environment:
            BUILD_FFMPEG: true
peterjc123's avatar
peterjc123 committed
296
297
298
      - store_artifacts:
          path: C:/tools/miniconda3/conda-bld/win-64
      - persist_to_workspace:
299
          root: C:/tools/miniconda3
peterjc123's avatar
peterjc123 committed
300
          paths:
301
            - "conda-bld/*"
peterjc123's avatar
peterjc123 committed
302

Edward Z. Yang's avatar
Edward Z. Yang committed
303
304
305
306
307
308
309
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
310
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
311
312
313
314
315
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
316
            anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
Edward Z. Yang's avatar
Edward Z. Yang committed
317
318
319

  # Requires org-member context
  binary_wheel_upload:
320
321
322
323
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
324
    docker:
moto's avatar
moto committed
325
      - image: cimg/python:3.8
Edward Z. Yang's avatar
Edward Z. Yang committed
326
327
328
329
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
330
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
331
332
333
334
335
336
337
338
339
340
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
            for pkg in ~/workspace/*.whl; do
341
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
Edward Z. Yang's avatar
Edward Z. Yang committed
342
343
            done

Mingbo Wan's avatar
Mingbo Wan committed
344
345
346
  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
Mingbo Wan's avatar
Mingbo Wan committed
347
348
      - attach_workspace:
          at: ~/workspace
349
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
350
      - load_conda_channel_flags
Mingbo Wan's avatar
Mingbo Wan committed
351
      - run:
Mingbo Wan's avatar
Mingbo Wan committed
352
          name: install binaries
Mingbo Wan's avatar
Mingbo Wan committed
353
354
355
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
356
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
357
            conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
moto's avatar
moto committed
358
      - checkout
Mingbo Wan's avatar
Mingbo Wan committed
359
360
361
362
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
moto's avatar
moto committed
363
            ./test/smoke_test/run_smoke_test.sh
Mingbo Wan's avatar
Mingbo Wan committed
364

Caroline Chen's avatar
Caroline Chen committed
365
366
367
368
369
370
371
372
373
374
375
376
377
378
  smoke_test_linux_conda_gpu:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - load_conda_channel_flags
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge
            conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
moto's avatar
moto committed
379
      - checkout
Caroline Chen's avatar
Caroline Chen committed
380
381
382
383
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
moto's avatar
moto committed
384
            ./test/smoke_test/run_smoke_test.sh
Caroline Chen's avatar
Caroline Chen committed
385

Mingbo Wan's avatar
Mingbo Wan committed
386
387
388
  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
Mingbo Wan's avatar
Mingbo Wan committed
389
390
      - attach_workspace:
          at: ~/workspace
391
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
392
      - load_conda_channel_flags
Mingbo Wan's avatar
Mingbo Wan committed
393
      - run:
Mingbo Wan's avatar
Mingbo Wan committed
394
          name: install binaries
Mingbo Wan's avatar
Mingbo Wan committed
395
396
397
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Caroline Chen's avatar
Caroline Chen committed
398
            pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
moto's avatar
moto committed
399
      - checkout
Mingbo Wan's avatar
Mingbo Wan committed
400
401
402
403
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
moto's avatar
moto committed
404
            ./test/smoke_test/run_smoke_test.sh
Mingbo Wan's avatar
Mingbo Wan committed
405

406
407
408
409
410
411
412
  smoke_test_windows_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
413
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
414
      - load_conda_channel_flags
415
416
417
418
419
420
421
422
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
423
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
424
            conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
moto's avatar
moto committed
425
      - checkout
426
427
428
429
430
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
moto's avatar
moto committed
431
            ./test/smoke_test/run_smoke_test.sh
432

433
434
435
436
437
  smoke_test_windows_conda_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    steps:
Andrey Talman's avatar
Andrey Talman committed
438
      - checkout
439
440
441
442
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - load_conda_channel_flags
Andrey Talman's avatar
Andrey Talman committed
443
444
445
446
      - windows_install_cuda
      - run:
          name: Update CUDA driver
          command: packaging/windows/internal/driver_update.bat
447
448
      - run:
          name: install binaries
Andrey Talman's avatar
Andrey Talman committed
449
          no_output_timeout: 30m
450
451
452
453
454
455
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
Andrey Talman's avatar
Andrey Talman committed
456
457
458
            # Include numpy and cudatoolkit in the install conda-forge chanell is used for cudatoolkit
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch numpy cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge
            # Install from torchaudio file
459
460
461
462
463
464
            conda install -v -y $(ls ~/workspace/conda-bld/win-64/torchaudio*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
Andrey Talman's avatar
Andrey Talman committed
465
466
            # Install sound backend
            pip install PySoundFile
moto's avatar
moto committed
467
            ./test/smoke_test/run_smoke_test.sh
468

469
470
471
472
473
474
475
  smoke_test_windows_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
476
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
477
      - load_conda_channel_flags
478
479
480
481
482
483
484
485
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
486
            pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html"
moto's avatar
moto committed
487
      - checkout
488
489
490
491
492
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
moto's avatar
moto committed
493
            ./test/smoke_test/run_smoke_test.sh
494

moto's avatar
moto committed
495
  unittest_linux_cpu:
moto's avatar
moto committed
496
497
    <<: *binary_common
    docker:
498
      - image: pytorch/torchaudio_unittest_base:manylinux-20210121
moto's avatar
moto committed
499
500
501
    resource_class: 2xlarge+
    steps:
      - checkout
502
503
      - attach_workspace:
          at: third_party
504
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
505
      - load_conda_channel_flags
506
507
508
509
510
511
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Install torchaudio
          command: .circleci/unittest/linux/scripts/install.sh
512
          environment:
513
              BUILD_MAD: true
514
              BUILD_FFMPEG: true
515
516
517
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
moto's avatar
moto committed
518
519
          environment:
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
520
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310: true
521
522
      - store_test_results:
          path: test-results
523
524
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
525
526
527
  unittest_linux_gpu:
    <<: *binary_common
    machine:
528
      image: ubuntu-2004-cuda-11.4:202110-01
529
    resource_class: gpu.nvidia.medium
moto's avatar
moto committed
530
    environment:
531
      <<: *environment
moto's avatar
moto committed
532
      image_name: pytorch/torchaudio_unittest_base:manylinux-cuda10.2-cudnn8-20210623
moto's avatar
moto committed
533
534
    steps:
      - checkout
535
536
      - attach_workspace:
          at: third_party
537
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
538
      - load_conda_channel_flags
539
540
541
      - run:
          name: Pull Docker image
          command: docker pull --quiet "${image_name}"
moto's avatar
moto committed
542
543
      - run:
          name: Setup
544
          command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
moto's avatar
moto committed
545
546
      - run:
          name: Install torchaudio
547
          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -e BUILD_FFMPEG=1 -e BUILD_MAD=1 -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
moto's avatar
moto committed
548
549
      - run:
          name: Run tests
550
551
552
553
          environment:
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310: true
          command: |
            docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "CI=${CI}" -e TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310 "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
moto's avatar
moto committed
554
555
      - store_test_results:
          path: test-results
556
557
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
558
559

  unittest_windows_cpu:
560
561
562
    <<: *binary_common
    executor:
      name: windows-cpu
563
564
    steps:
      - checkout
565
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
566
      - load_conda_channel_flags
567
568
569
570
571
572
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - run:
          name: Install torchaudio
          command: .circleci/unittest/windows/scripts/install.sh
573
574
          environment:
              BUILD_FFMPEG: true
575
576
577
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
moto's avatar
moto committed
578
579
580
581
582
583
584
585
586
587
588
          environment:
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX: true
589
590
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece: true
591
592
      - store_test_results:
          path: test-results
593
594
      - store_artifacts:
          path: test/htmlcov
595
596
597
598
599
600

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
601
      <<: *environment
602
      CUDA_VERSION: "11.3"
603
604
    steps:
      - checkout
605
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
606
      - load_conda_channel_flags
moto's avatar
moto committed
607
608
      - run:
          name: Setup
609
          command: .circleci/unittest/windows/scripts/setup_env.sh
610
611
612
613
614
615
      - run:
          name: Install CUDA
          command: packaging/windows/internal/cuda_install.bat
      - run:
          name: Update CUDA driver
          command: packaging/windows/internal/driver_update.bat
moto's avatar
moto committed
616
      - run:
617
          name: Install torchaudio
618
          command: .circleci/unittest/windows/scripts/install.sh
619
620
          environment:
              BUILD_FFMPEG: true
moto's avatar
moto committed
621
      - run:
622
          name: Run tests
623
          command: .circleci/unittest/windows/scripts/run_test.sh
moto's avatar
moto committed
624
625
626
627
628
629
630
631
632
633
          environment:
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_SOX: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_KALDI: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX: true
634
635
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece: true
moto's avatar
moto committed
636
637
      - store_test_results:
          path: test-results
638
639
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
640

moto's avatar
moto committed
641
642
643
  unittest_macos_cpu:
    <<: *binary_common
    macos:
644
      xcode: "12.0"
moto's avatar
moto committed
645
646
647
    resource_class: large
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
648
      - load_conda_channel_flags
moto's avatar
moto committed
649
650
      - attach_workspace:
          at: third_party
651
      - designate_upload_channel
moto's avatar
moto committed
652
653
654
655
656
657
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Install torchaudio
          command: .circleci/unittest/linux/scripts/install.sh
658
659
          environment:
              BUILD_FFMPEG: true
660
              BUILD_MAD: true
moto's avatar
moto committed
661
662
663
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
moto's avatar
moto committed
664
665
666
667
668
669
670
671
          environment:
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CUDA: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_QUANTIZATION: true
672
673
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310: true
              TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_sentencepiece: true
moto's avatar
moto committed
674
675
      - store_test_results:
          path: test-results
676
677
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
678

679
680
681
682
683
684
685
  stylecheck:
    <<: *binary_common
    docker:
      - image: "pytorch/torchaudio_unittest_base:manylinux"
    resource_class: medium
    steps:
      - checkout
686
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
687
      - load_conda_channel_flags
688
689
690
691
692
693
694
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Run style check
          command: .circleci/unittest/linux/scripts/run_style_checks.sh

695
696
697
  build_docs:
    <<: *binary_common
    docker:
698
      - image: "python:<< parameters.python_version >>"
699
700
701
702
703
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
704
      - load_conda_channel_flags
705
      - run:
706
707
708
709
          name: Install packages
          command: |
            apt-get -qq update && apt-get -qq install -y ffmpeg libsndfile-dev
            .circleci/build_docs/install_wheels.sh
710
711
      - run:
          name: Build docs
712
713
714
715
716
          command: |
            cd docs
            make 'SPHINXOPTS=-W' html
          environment:
            BUILD_GALLERY: 1
717
      - persist_to_workspace:
718
          root: ./
719
720
          paths:
            - "*"
721
722
723
      - store_artifacts:
          path: ./docs/build/html
          destination: docs
724
725
726
727
728
729
730
731
732

  upload_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
733
734
735
736
737
738
739
740
741
742
      - run:
          name: Generate netrc
          command: |
            # set credentials for https pushing
            # requires the org-member context
            cat > ~/.netrc \<<DONE
              machine github.com
              login pytorchbot
              password ${GITHUB_PYTORCHBOT_TOKEN}
            DONE
743
744
745
      - run:
          name: Upload docs
          command: |
746
747
            # Don't use "checkout" step since it uses ssh, which cannot git push
            # https://circleci.com/docs/2.0/configuration-reference/#checkout
748
            set -ex
749
750
            # turn v1.12.0rc3 into 1.12.0
            tag=$(echo $CIRCLE_TAG | sed -e 's/v*\([0-9.]*\).*/\1/')
751
            target=${tag:-main}
Matti Picus's avatar
Matti Picus committed
752
            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target
753

754
755
756
  docstring_parameters_sync:
    <<: *binary_common
    docker:
moto's avatar
moto committed
757
      - image: cimg/python:3.8
758
759
760
761
762
763
764
765
    steps:
      - checkout
      - run:
          name: Check parameters docstring sync
          command: |
            pip install --user pydocstyle
            pydocstyle torchaudio

Edward Z. Yang's avatar
Edward Z. Yang committed
766
workflows:
767
768
769
  lint:
    jobs:
      - lint_python_and_config
Edward Z. Yang's avatar
Edward Z. Yang committed
770
771
772
  build:
    jobs:
      - circleci_consistency
moto's avatar
moto committed
773
774
775
776
      {{ build_workflows() }}
  unittest:
    jobs:
      {{ unittest_workflows() }}
Edward Z. Yang's avatar
Edward Z. Yang committed
777
778
  nightly:
    jobs:
moto's avatar
moto committed
779
780
781
782
      - circleci_consistency:
          filters:
            branches:
              only: nightly
moto's avatar
moto committed
783
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}