config.yml.in 21.8 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
5
6
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8
#     - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
Edward Z. Yang's avatar
Edward Z. Yang committed
7
#       Job names are 'name:' key.
8

9
10
11
12
13
14
executors:
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe
peterjc123's avatar
peterjc123 committed
15

16
17
18
  windows-gpu:
    machine:
      resource_class: windows.gpu.nvidia.medium
19
      image: windows-server-2019-nvidia:stable
20
21
      shell: bash.exe

22
commands:
23
24
25
26
27
28
  generate_cache_key:
    description: "Generates a cache key file that changes daily"
    steps:
      - run:
          name: Generate cache key
          command: echo "$(date +"%Y-%m-%d")" > .cachekey
29
30
31
32
33
34
35
36
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
37
            if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then
38
39
40
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
moto's avatar
moto committed
41
42
  install_build_tools_macos:
    description: "installs tools required to build torchaudio"
moto's avatar
moto committed
43
44
    steps:
      - run:
45
          name: Install build tools
moto's avatar
moto committed
46
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget
moto's avatar
moto committed
47
          # Disable brew auto update which is very slow
Eli Uriegas's avatar
Eli Uriegas committed
48
49
50
51
52
53
54
  load_conda_channel_flags:
    description: "Determines whether we need extra conda channels"
    steps:
      - run:
          name: Adding CONDA_CHANNEL_FLAGS to BASH_ENV
          command: |
              CONDA_CHANNEL_FLAGS=""
55
              # formerly used to add conda-forge flags for Python 3.9, reserving the mechanism for future python upgrades
56
57
58
59
60
61
62
  windows_install_cuda:
    description: "Install desired CUDA version on Windows runners"
    steps:
      - run:
          name: Install CUDA
          command: |
              packaging/windows/internal/cuda_install.bat
63

64
65
binary_common: &binary_common
  parameters:
Edward Z. Yang's avatar
Edward Z. Yang committed
66
    # Edit these defaults to do a release
67
68
69
70
71
72
73
74
75
76
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
77
      description: "Python version to build against (e.g., 3.8)"
78
      type: string
Nikita Shulga's avatar
Nikita Shulga committed
79
80
81
82
83
84
85
86
87
88
89
90
    cuda_version:
      description: "CUDA version to build against (e.g., cpu, cu101)"
      type: string
      default: "cpu"
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
      default: "pytorch/manylinux-cuda102"
    conda_docker_image:
      description: "Conda only: what docker image to use"
      type: string
      default: "pytorch/conda-builder:cuda102"
91
  environment: &environment
92
93
94
    PYTHON_VERSION: << parameters.python_version >>
    BUILD_VERSION: << parameters.build_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
Nikita Shulga's avatar
Nikita Shulga committed
95
    CU_VERSION: << parameters.cuda_version >>
96

Mingbo Wan's avatar
Mingbo Wan committed
97
98
99
smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
Caroline Chen's avatar
Caroline Chen committed
100
101
    - image: pytorch/torchaudio_unittest_base:smoke_test-20211019
  resource_class: large
Mingbo Wan's avatar
Mingbo Wan committed
102

103
104
105
jobs:
  circleci_consistency:
    docker:
moto's avatar
moto committed
106
      - image: cimg/python:3.8
107
108
109
110
    steps:
      - checkout
      - run:
          command: |
111
            pip install --user --progress-bar off jinja2 pyyaml
112
113
114
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

115
116
  download_third_parties_nix:
    docker:
moto's avatar
moto committed
117
      - image: "pytorch/torchaudio_unittest_base:manylinux"
118
119
120
    resource_class: small
    steps:
      - checkout
121
      - generate_cache_key
122
123
124
      - restore_cache:
          {% raw %}
          keys:
moto's avatar
moto committed
125
            - tp-nix-v2-{{ checksum ".cachekey" }}
126
127
          {% endraw %}
      - run:
moto's avatar
moto committed
128
          command: |
129
130
              mkdir -p third_party/sox/archives/
              wget --no-clobber --directory-prefix=third_party/sox/archives/ $(awk '/URL /{print $2}' third_party/sox/CMakeLists.txt)
131
132
      - save_cache:
          {% raw %}
moto's avatar
moto committed
133
          key: tp-nix-v2-{{ checksum ".cachekey" }}
134
135
          {% endraw %}
          paths:
136
            - third_party/sox/archives
137
138
139
      - persist_to_workspace:
          root: third_party
          paths:
140
            - sox/archives
141

142
143
144
  binary_linux_wheel:
    <<: *binary_common
    docker:
Nikita Shulga's avatar
Nikita Shulga committed
145
      - image: << parameters.wheel_docker_image >>
146
147
148
    resource_class: 2xlarge+
    steps:
      - checkout
149
150
      - attach_workspace:
          at: third_party
151
152
153
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
154
155
156
157
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
158
159
160
161

  binary_linux_conda:
    <<: *binary_common
    docker:
Nikita Shulga's avatar
Nikita Shulga committed
162
      - image: "<< parameters.conda_docker_image >>"
163
164
165
    resource_class: 2xlarge+
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
166
      - load_conda_channel_flags
167
168
      - attach_workspace:
          at: third_party
169
170
171
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
172
      - persist_to_workspace:
173
          root: /opt/conda
Edward Z. Yang's avatar
Edward Z. Yang committed
174
          paths:
175
            - "conda-bld/*"
176
177
178
179

  binary_macos_wheel:
    <<: *binary_common
    macos:
180
      xcode: "12.0"
181
182
    steps:
      - checkout
moto's avatar
moto committed
183
      - install_build_tools_macos
Eli Uriegas's avatar
Eli Uriegas committed
184
      - load_conda_channel_flags
185
186
      - attach_workspace:
          at: third_party
187
188
189
190
191
192
193
194
195
196
197
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
198
199
200
201
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
202
203
204
205

  binary_macos_conda:
    <<: *binary_common
    macos:
206
      xcode: "12.0"
207
208
    steps:
      - checkout
moto's avatar
moto committed
209
      - install_build_tools_macos
Eli Uriegas's avatar
Eli Uriegas committed
210
      - load_conda_channel_flags
211
212
      - attach_workspace:
          at: third_party
213
214
215
216
217
218
219
220
221
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
222
      - persist_to_workspace:
223
          root: /Users/distiller/miniconda3
Edward Z. Yang's avatar
Edward Z. Yang committed
224
          paths:
225
            - "conda-bld/*"
226

peterjc123's avatar
peterjc123 committed
227
228
229
230
231
232
  binary_windows_wheel:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
233
      - load_conda_channel_flags
234
      - windows_install_cuda
peterjc123's avatar
peterjc123 committed
235
      - run:
236
          name: Build wheel packages
peterjc123's avatar
peterjc123 committed
237
          command: |
238
            set -ex
peterjc123's avatar
peterjc123 committed
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            bash packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_windows_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
255
      - load_conda_channel_flags
256
      - windows_install_cuda
peterjc123's avatar
peterjc123 committed
257
      - run:
258
          name: Build conda packages
259
          no_output_timeout: 20m
peterjc123's avatar
peterjc123 committed
260
          command: |
261
            set -ex
peterjc123's avatar
peterjc123 committed
262
263
264
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
265
266
267
268
            # cudatoolkit >= 11 isn't available for windows in the nvidia channel
            if [[ "${CU_VERSION}" =~ cu11.* ]]; then
              export CONDA_CHANNEL_FLAGS="-c conda-forge"
            fi
peterjc123's avatar
peterjc123 committed
269
270
271
272
            bash packaging/build_conda.sh
      - store_artifacts:
          path: C:/tools/miniconda3/conda-bld/win-64
      - persist_to_workspace:
273
          root: C:/tools/miniconda3
peterjc123's avatar
peterjc123 committed
274
          paths:
275
            - "conda-bld/*"
peterjc123's avatar
peterjc123 committed
276

Edward Z. Yang's avatar
Edward Z. Yang committed
277
278
279
280
281
282
283
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
284
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
285
286
287
288
289
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
290
            anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
Edward Z. Yang's avatar
Edward Z. Yang committed
291
292
293

  # Requires org-member context
  binary_wheel_upload:
294
295
296
297
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
Edward Z. Yang's avatar
Edward Z. Yang committed
298
    docker:
moto's avatar
moto committed
299
      - image: cimg/python:3.8
Edward Z. Yang's avatar
Edward Z. Yang committed
300
301
302
303
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
304
      - designate_upload_channel
Edward Z. Yang's avatar
Edward Z. Yang committed
305
306
307
308
309
310
311
312
313
314
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
            for pkg in ~/workspace/*.whl; do
315
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
Edward Z. Yang's avatar
Edward Z. Yang committed
316
317
            done

Mingbo Wan's avatar
Mingbo Wan committed
318
319
320
  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
Mingbo Wan's avatar
Mingbo Wan committed
321
322
      - attach_workspace:
          at: ~/workspace
323
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
324
      - load_conda_channel_flags
Mingbo Wan's avatar
Mingbo Wan committed
325
      - run:
Mingbo Wan's avatar
Mingbo Wan committed
326
          name: install binaries
Mingbo Wan's avatar
Mingbo Wan committed
327
328
329
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
330
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
331
            conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
Mingbo Wan's avatar
Mingbo Wan committed
332
333
334
335
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Mingbo Wan's avatar
Mingbo Wan committed
336
337
            python -c "import torchaudio"

Caroline Chen's avatar
Caroline Chen committed
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
  smoke_test_linux_conda_gpu:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - load_conda_channel_flags
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cudatoolkit=${CU_VERSION:2:2}.${CU_VERSION:4} -c conda-forge
            conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchaudio"

Mingbo Wan's avatar
Mingbo Wan committed
358
359
360
  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
Mingbo Wan's avatar
Mingbo Wan committed
361
362
      - attach_workspace:
          at: ~/workspace
363
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
364
      - load_conda_channel_flags
Mingbo Wan's avatar
Mingbo Wan committed
365
      - run:
Mingbo Wan's avatar
Mingbo Wan committed
366
          name: install binaries
Mingbo Wan's avatar
Mingbo Wan committed
367
368
369
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Caroline Chen's avatar
Caroline Chen committed
370
            pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
Mingbo Wan's avatar
Mingbo Wan committed
371
372
373
374
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Mingbo Wan's avatar
Mingbo Wan committed
375
376
            python -c "import torchaudio"

377
378
379
380
381
382
383
  smoke_test_windows_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
384
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
385
      - load_conda_channel_flags
386
387
388
389
390
391
392
393
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
394
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
            conda install -v -y $(ls ~/workspace/torchaudio*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchaudio"

  smoke_test_windows_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
410
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
411
      - load_conda_channel_flags
412
413
414
415
416
417
418
419
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
420
            pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html"
421
422
423
424
425
426
427
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchaudio"

428
  smoke_test_docker_image_build:
Mingbo Wan's avatar
Mingbo Wan committed
429
430
431
432
433
434
435
436
437
438
439
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchaudio/smoke_test
    steps:
      - checkout
      - run:
          name: build_docker image
          no_output_timeout: "1h"
          command: |
440
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
Mingbo Wan's avatar
Mingbo Wan committed
441
442
443
444
445
446
447
448
449
450
      - run:
          name: upload docker image
          no_output_timeout: "1h"
          command: |
            set +x
            export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
            export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
            eval $(aws ecr get-login --region us-east-1 --no-include-email)
            set -x
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
451
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:latest
Mingbo Wan's avatar
Mingbo Wan committed
452
            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
453
            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:latest
Edward Z. Yang's avatar
Edward Z. Yang committed
454

moto's avatar
moto committed
455
  unittest_linux_cpu:
moto's avatar
moto committed
456
457
    <<: *binary_common
    docker:
458
      - image: pytorch/torchaudio_unittest_base:manylinux-20210121
moto's avatar
moto committed
459
460
461
    resource_class: 2xlarge+
    steps:
      - checkout
462
463
      - attach_workspace:
          at: third_party
464
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
465
      - load_conda_channel_flags
466
467
468
469
470
471
472
473
474
475
476
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Install torchaudio
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - store_test_results:
          path: test-results
477
478
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
479
480
481
482
483
484
  unittest_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
485
      <<: *environment
moto's avatar
moto committed
486
      image_name: pytorch/torchaudio_unittest_base:manylinux-cuda10.2-cudnn8-20210623
moto's avatar
moto committed
487
488
    steps:
      - checkout
489
490
      - attach_workspace:
          at: third_party
491
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
492
      - load_conda_channel_flags
493
494
495
      - run:
          name: Pull Docker image
          command: docker pull --quiet "${image_name}"
moto's avatar
moto committed
496
497
      - run:
          name: Setup
498
          command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
moto's avatar
moto committed
499
500
      - run:
          name: Install torchaudio
Eli Uriegas's avatar
Eli Uriegas committed
501
          command: docker run -t --gpus all -e UPLOAD_CHANNEL -e CONDA_CHANNEL_FLAGS -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
moto's avatar
moto committed
502
503
      - run:
          name: Run tests
504
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e "TORCHAUDIO_TEST_FORCE_CUDA=1" "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
moto's avatar
moto committed
505
506
      - store_test_results:
          path: test-results
507
508
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
509
510

  unittest_windows_cpu:
511
512
513
    <<: *binary_common
    executor:
      name: windows-cpu
514
515
    steps:
      - checkout
516
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
517
      - load_conda_channel_flags
518
519
520
521
522
523
524
525
526
527
528
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - run:
          name: Install torchaudio
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - store_test_results:
          path: test-results
529
530
      - store_artifacts:
          path: test/htmlcov
531
532
533
534
535
536

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
537
      <<: *environment
538
      CUDA_VERSION: "11.3"
539
      TORCHAUDIO_TEST_FORCE_CUDA: 1
540
541
    steps:
      - checkout
542
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
543
      - load_conda_channel_flags
moto's avatar
moto committed
544
545
      - run:
          name: Setup
546
          command: .circleci/unittest/windows/scripts/setup_env.sh
547
548
549
550
551
552
      - run:
          name: Install CUDA
          command: packaging/windows/internal/cuda_install.bat
      - run:
          name: Update CUDA driver
          command: packaging/windows/internal/driver_update.bat
moto's avatar
moto committed
553
      - run:
554
          name: Install torchaudio
555
          command: .circleci/unittest/windows/scripts/install.sh
moto's avatar
moto committed
556
      - run:
557
          name: Run tests
558
          command: .circleci/unittest/windows/scripts/run_test.sh
moto's avatar
moto committed
559
560
      - store_test_results:
          path: test-results
561
562
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
563

moto's avatar
moto committed
564
565
566
  unittest_macos_cpu:
    <<: *binary_common
    macos:
567
      xcode: "12.0"
moto's avatar
moto committed
568
569
570
571
    resource_class: large
    steps:
      - checkout
      - install_build_tools_macos
Eli Uriegas's avatar
Eli Uriegas committed
572
      - load_conda_channel_flags
moto's avatar
moto committed
573
574
      - attach_workspace:
          at: third_party
575
      - designate_upload_channel
moto's avatar
moto committed
576
577
578
579
580
581
582
583
584
585
586
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Install torchaudio
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - store_test_results:
          path: test-results
587
588
      - store_artifacts:
          path: test/htmlcov
moto's avatar
moto committed
589

590
591
592
593
594
595
596
  stylecheck:
    <<: *binary_common
    docker:
      - image: "pytorch/torchaudio_unittest_base:manylinux"
    resource_class: medium
    steps:
      - checkout
597
      - designate_upload_channel
Eli Uriegas's avatar
Eli Uriegas committed
598
      - load_conda_channel_flags
599
600
601
602
603
604
605
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Run style check
          command: .circleci/unittest/linux/scripts/run_style_checks.sh

606
607
608
609
610
611
612
613
614
  build_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
Eli Uriegas's avatar
Eli Uriegas committed
615
      - load_conda_channel_flags
616
617
618
619
620
621
622
      - run:
          name: Install pytorch-audio
          command: .circleci/build_docs/install_wheels.sh
      - run:
          name: Build docs
          command: .circleci/build_docs/build_docs.sh
      - persist_to_workspace:
623
          root: ./
624
625
          paths:
            - "*"
626
627
628
      - store_artifacts:
          path: ./docs/build/html
          destination: docs
629
630
631
632
633
634
635
636
637

  upload_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
638
639
640
641
642
643
644
645
646
647
      - run:
          name: Generate netrc
          command: |
            # set credentials for https pushing
            # requires the org-member context
            cat > ~/.netrc \<<DONE
              machine github.com
              login pytorchbot
              password ${GITHUB_PYTORCHBOT_TOKEN}
            DONE
648
649
650
      - run:
          name: Upload docs
          command: |
651
652
            # Don't use "checkout" step since it uses ssh, which cannot git push
            # https://circleci.com/docs/2.0/configuration-reference/#checkout
653
654
            set -ex
            tag=${CIRCLE_TAG:1:5}
655
            target=${tag:-main}
Matti Picus's avatar
Matti Picus committed
656
            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target
657

658
659
660
  docstring_parameters_sync:
    <<: *binary_common
    docker:
moto's avatar
moto committed
661
      - image: cimg/python:3.8
662
663
664
665
666
667
668
669
    steps:
      - checkout
      - run:
          name: Check parameters docstring sync
          command: |
            pip install --user pydocstyle
            pydocstyle torchaudio

Edward Z. Yang's avatar
Edward Z. Yang committed
670
671
672
673
workflows:
  build:
    jobs:
      - circleci_consistency
moto's avatar
moto committed
674
675
676
677
      {{ build_workflows() }}
  unittest:
    jobs:
      {{ unittest_workflows() }}
Edward Z. Yang's avatar
Edward Z. Yang committed
678
679
  nightly:
    jobs:
moto's avatar
moto committed
680
681
682
683
      - circleci_consistency:
          filters:
            branches:
              only: nightly
moto's avatar
moto committed
684
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
Mingbo Wan's avatar
Mingbo Wan committed
685
686
687
688
689
690
691
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
692
                - main
Mingbo Wan's avatar
Mingbo Wan committed
693
    jobs:
694
      - smoke_test_docker_image_build