config.yml.in

version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8
#     - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
#       Job names are 'name:' key.

executors:
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
    machine:
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
      shell: bash.exe

commands:
  generate_cache_key:
    description: "Generates a cache key file that changes daily"
    steps:
      - run:
          name: Generate cache key
          command: echo "$(date +"%Y-%m-%d")" > .cachekey
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
  install_build_tools_macos:
    description: "installs tools required to build torchaudio"
    steps:
      - run:
          name: Install cmake and pkg-config
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake pkg-config wget
          # Disable brew auto update which is very slow

binary_common: &binary_common
  parameters:
    # Edit these defaults to do a release
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.8)"
      type: string
  environment: &environment
    PYTHON_VERSION: << parameters.python_version >>
    BUILD_VERSION: << parameters.build_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    CU_VERSION: cpu

smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:56c846a5-acaa-41a7-92f5-46ec66186c61
      aws_auth:
        aws_access_key_id: ${ECR_AWS_ACCESS_KEY}
        aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY}

jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.8
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off jinja2 pyyaml
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

  download_third_parties_nix:
    docker:
      - image: "pytorch/torchaudio_unittest_base:manylinux"
    resource_class: small
    steps:
      - checkout
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - tp-nix-v2-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          command: |
              mkdir -p third_party/archives/
              wget --no-clobber --directory-prefix=third_party/archives/ $(awk '/URL /{print $2}' third_party/CMakeLists.txt)
      - save_cache:
          {% raw %}
          key: tp-nix-v2-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - third_party/archives
      - persist_to_workspace:
          root: third_party
          paths:
            - archives

  binary_linux_wheel:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - checkout
      - attach_workspace:
          at: third_party
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_linux_conda:
    <<: *binary_common
    docker:
      - image: "pytorch/conda-cuda"
    resource_class: 2xlarge+
    steps:
      - checkout
      - attach_workspace:
          at: third_party
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
      - persist_to_workspace:
          root: /opt/conda
          paths:
            - "conda-bld/*"

  binary_macos_wheel:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    steps:
      - checkout
      - install_build_tools_macos
      - attach_workspace:
          at: third_party
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_macos_conda:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    steps:
      - checkout
      - install_build_tools_macos
      - attach_workspace:
          at: third_party
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
      - persist_to_workspace:
          root: /Users/distiller/miniconda3
          paths:
            - "conda-bld/*"

  binary_windows_wheel:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
      - run:
          name: build
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            bash packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_windows_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
      - run:
          name: build
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
            bash packaging/build_conda.sh
      - store_artifacts:
          path: C:/tools/miniconda3/conda-bld/win-64
      - persist_to_workspace:
          root: C:/tools/miniconda3
          paths:
            - "conda-bld/*"

  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
            anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force

  # Requires org-member context
  binary_wheel_upload:
    docker:
      - image: circleci/python:3.8
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
      - designate_upload_channel
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
            for pkg in ~/workspace/*.whl; do
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read
            done

  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch
            conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchaudio"

  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html"
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchaudio"

  smoke_test_windows_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch
            conda install -v -y $(ls ~/workspace/torchaudio*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchaudio"

  smoke_test_windows_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html"
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchaudio"

  smoke_test_docker_image_build:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchaudio/smoke_test
    steps:
      - checkout
      - run:
          name: build_docker image
          no_output_timeout: "1h"
          command: |
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
      - run:
          name: upload docker image
          no_output_timeout: "1h"
          command: |
            set +x
            export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
            export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
            eval $(aws ecr get-login --region us-east-1 --no-include-email)
            set -x
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:latest
            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:latest

  unittest_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/torchaudio_unittest_base:manylinux"
    resource_class: 2xlarge+
    steps:
      - checkout
      - attach_workspace:
          at: third_party
      - designate_upload_channel
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - conda
            - env
            - third_party/build
            - third_party/install
            - third_party/src
      - run:
          name: Install torchaudio
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
      <<: *environment
      image_name: "pytorch/torchaudio_unittest_base:manylinux-cuda10.1"
    steps:
      - checkout
      - attach_workspace:
          at: third_party
      - designate_upload_channel
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          name: Pull Docker image
          command: docker pull --quiet "${image_name}"
      - run:
          name: Setup
          command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - conda
            - env
            - third_party/build
            - third_party/install
            - third_party/src
      - run:
          name: Install torchaudio
          command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post Process
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
      - designate_upload_channel
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/setup_env.sh" }}-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/setup_env.sh" }}-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchaudio
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
      <<: *environment
      CUDA_VERSION: "10.1"
    steps:
      - checkout
      - designate_upload_channel
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/setup_env.sh" }}-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/setup_env.sh" }}-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchaudio
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_macos_cpu:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    resource_class: large
    steps:
      - checkout
      - install_build_tools_macos
      - attach_workspace:
          at: third_party
      - designate_upload_channel
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum "third_party/CMakeLists.txt" }}-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - conda
            - env
            - third_party/build
            - third_party/install
            - third_party/src
      - run:
          name: Install torchaudio
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  stylecheck:
    <<: *binary_common
    docker:
      - image: "pytorch/torchaudio_unittest_base:manylinux"
    resource_class: medium
    steps:
      - checkout
      - designate_upload_channel
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/setup_env.sh" }}-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - conda
            - env
            - third_party/build
            - third_party/install
            - third_party/src
      - run:
          name: Run style check
          command: .circleci/unittest/linux/scripts/run_style_checks.sh

  torchscript_bc_test:
    docker:
      - image: "pytorch/torchaudio_unittest_base:manylinux"
    resource_class: large
    steps:
      - checkout
      - generate_cache_key
      - restore_cache:
          {% raw %}
          keys:
            - torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
          {% endraw %}
      - run:
          name: Generate Objects
          command: |
             .circleci/torchscript_bc_test/setup_release_envs.sh
             .circleci/torchscript_bc_test/generate_objects.sh
      - save_cache:
          {% raw %}
          key: torchscript-bc-test-release-envs-v1-{{ arch }}-{{ checksum ".cachekey" }}
          {% endraw %}
          paths:
            - conda
            - envs
      - store_artifacts:
          path: test/torchscript_bc_test/assets
      - persist_to_workspace:
          root: .
          paths:
            - test/torchscript_bc_test/assets
      - run:
          name: Run BC check test
          command: |
            .circleci/torchscript_bc_test/setup_master_envs.sh
            .circleci/torchscript_bc_test/validate_objects.sh
  build_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
      - run:
          name: Install pytorch-audio
          command: .circleci/build_docs/install_wheels.sh
      - run:
          name: Build docs
          command: .circleci/build_docs/build_docs.sh
      - persist_to_workspace:
          root: ./
          paths:
            - "*"

  upload_docs:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda100"
    resource_class: 2xlarge+
    steps:
      - attach_workspace:
          at: ~/workspace
      - run:
          name: Generate netrc
          command: |
            # set credentials for https pushing
            # requires the org-member context
            cat > ~/.netrc \<<DONE
              machine github.com
              login pytorchbot
              password ${GITHUB_PYTORCHBOT_TOKEN}
            DONE
      - run:
          name: Upload docs
          command: |
            # Don't use "checkout" step since it uses ssh, which cannot git push
            # https://circleci.com/docs/2.0/configuration-reference/#checkout
            set -ex
            echo $PWD
            ls ~/workspace
            ls ~/workspace/doc
            tag=${CIRCLE_TAG:1:5}
            target=${tag:-master}
            .circleci/build_docs/commit_docs.sh ~/workspace $target

workflows:
  build:
    jobs:
      - circleci_consistency
      {{ build_workflows() }}
  unittest:
    jobs:
      - torchscript_bc_test:
          requires:
          - download_third_parties_nix
      {{ unittest_workflows() }}
  nightly:
    jobs:
      - circleci_consistency:
          filters:
            branches:
              only: nightly
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
                - master
    jobs:
      - smoke_test_docker_image_build