version: 2.1 # How to test the Linux jobs: # - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ # - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8 # - Replace binary_linux_wheel_py3.8 with the name of the job you want to test. # Job names are 'name:' key. executors: windows-cpu: machine: resource_class: windows.xlarge image: windows-server-2019-vs2019:stable shell: bash.exe windows-gpu: machine: resource_class: windows.gpu.nvidia.medium image: windows-server-2019-nvidia:stable shell: bash.exe commands: generate_cache_key: description: "Generates a cache key file that changes daily" steps: - run: name: Generate cache key command: echo "$(date +"%Y-%m-%d")" > .cachekey designate_upload_channel: description: "inserts the correct upload channel into ${BASH_ENV}" steps: - run: name: adding UPLOAD_CHANNEL to BASH_ENV command: | our_upload_channel=nightly # On tags upload to test instead if [[ -n "${CIRCLE_TAG}" ]] || [[ ${CIRCLE_BRANCH} =~ release/* ]]; then our_upload_channel=test fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} install_build_tools_macos: description: "installs tools required to build torchaudio" steps: - run: name: Install pkg-config command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config wget # Disable brew auto update which is very slow binary_common: &binary_common parameters: # Edit these defaults to do a release build_version: description: "version number of release binary; by default, build a nightly" type: string default: "" pytorch_version: description: "PyTorch version to build against; by default, use a nightly" type: string default: "" # Don't edit these python_version: description: "Python version to build against (e.g., 3.8)" type: string environment: &environment PYTHON_VERSION: << parameters.python_version >> BUILD_VERSION: << parameters.build_version >> PYTORCH_VERSION: << parameters.pytorch_version >> CU_VERSION: cpu smoke_test_common: &smoke_test_common <<: *binary_common docker: - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:56c846a5-acaa-41a7-92f5-46ec66186c61 aws_auth: aws_access_key_id: ${ECR_AWS_ACCESS_KEY} aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY} jobs: circleci_consistency: docker: - image: circleci/python:3.8 steps: - checkout - run: command: | pip install --user --progress-bar off jinja2 pyyaml python .circleci/regenerate.py git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) download_third_parties_nix: docker: - image: "pytorch/torchaudio_unittest_base:manylinux" resource_class: small steps: - checkout - generate_cache_key - restore_cache: keys: - tp-nix-v2-{{ checksum ".cachekey" }} - run: command: | mkdir -p third_party/sox/archives/ wget --no-clobber --directory-prefix=third_party/sox/archives/ $(awk '/URL /{print $2}' third_party/sox/CMakeLists.txt) - save_cache: key: tp-nix-v2-{{ checksum ".cachekey" }} paths: - third_party/sox/archives - persist_to_workspace: root: third_party paths: - sox/archives binary_linux_wheel: <<: *binary_common docker: - image: "pytorch/manylinux-cuda100" resource_class: 2xlarge+ steps: - checkout - attach_workspace: at: third_party - run: packaging/build_wheel.sh - store_artifacts: path: dist - persist_to_workspace: root: dist paths: - "*" binary_linux_conda: <<: *binary_common docker: - image: "pytorch/conda-cuda" resource_class: 2xlarge+ steps: - checkout - attach_workspace: at: third_party - run: packaging/build_conda.sh - store_artifacts: path: /opt/conda/conda-bld/linux-64 - persist_to_workspace: root: /opt/conda paths: - "conda-bld/*" binary_macos_wheel: <<: *binary_common macos: xcode: "9.4.1" steps: - checkout - install_build_tools_macos - attach_workspace: at: third_party - run: # Cannot easily deduplicate this as source'ing activate # will set environment variables which we need to propagate # to build_wheel.sh command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate packaging/build_wheel.sh - store_artifacts: path: dist - persist_to_workspace: root: dist paths: - "*" binary_macos_conda: <<: *binary_common macos: xcode: "9.4.1" steps: - checkout - install_build_tools_macos - attach_workspace: at: third_party - run: command: | curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh sh conda.sh -b source $HOME/miniconda3/bin/activate conda install -yq conda-build packaging/build_conda.sh - store_artifacts: path: /Users/distiller/miniconda3/conda-bld/osx-64 - persist_to_workspace: root: /Users/distiller/miniconda3 paths: - "conda-bld/*" binary_windows_wheel: <<: *binary_common executor: name: windows-cpu steps: - checkout - run: name: build command: | eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base bash packaging/build_wheel.sh - store_artifacts: path: dist - persist_to_workspace: root: dist paths: - "*" binary_windows_conda: <<: *binary_common executor: name: windows-cpu steps: - checkout - run: name: build command: | eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate base conda install -yq conda-build "conda-package-handling!=1.5.0" bash packaging/build_conda.sh - store_artifacts: path: C:/tools/miniconda3/conda-bld/win-64 - persist_to_workspace: root: C:/tools/miniconda3 paths: - "conda-bld/*" # Requires org-member context binary_conda_upload: docker: - image: continuumio/miniconda steps: - attach_workspace: at: ~/workspace - designate_upload_channel - run: command: | # Prevent credential from leaking conda install -yq anaconda-client set -x anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force # Requires org-member context binary_wheel_upload: docker: - image: circleci/python:3.8 steps: - attach_workspace: at: ~/workspace - checkout - designate_upload_channel - run: command: | pip install --user awscli export PATH="$HOME/.local/bin:$PATH" # Prevent credential from leaking set +x export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" set -x for pkg in ~/workspace/*.whl; do aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read done smoke_test_linux_conda: <<: *smoke_test_common steps: - attach_workspace: at: ~/workspace - designate_upload_channel - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly conda install -v -y -c file://$HOME/workspace/conda-bld torchaudio - run: name: smoke test command: | source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} python -c "import torchaudio" smoke_test_linux_pip: <<: *smoke_test_common steps: - attach_workspace: at: ~/workspace - designate_upload_channel - run: name: install binaries command: | set -x source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} python -c "import torchaudio" smoke_test_windows_conda: <<: *binary_common executor: name: windows-cpu steps: - attach_workspace: at: ~/workspace - designate_upload_channel - run: name: install binaries command: | set -x eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} conda install -v -y -c pytorch-${UPLOAD_CHANNEL} pytorch cpuonly conda install -v -y $(ls ~/workspace/torchaudio*.tar.bz2) - run: name: smoke test command: | eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate python${PYTHON_VERSION} python -c "import torchaudio" smoke_test_windows_pip: <<: *binary_common executor: name: windows-cpu steps: - attach_workspace: at: ~/workspace - designate_upload_channel - run: name: install binaries command: | set -x eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda env remove -n python${PYTHON_VERSION} || true conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} conda activate python${PYTHON_VERSION} pip install $(ls ~/workspace/torchaudio*.whl) -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - run: name: smoke test command: | eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" conda activate python${PYTHON_VERSION} python -c "import torchaudio" smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01 resource_class: large environment: image_name: torchaudio/smoke_test steps: - checkout - run: name: build_docker image no_output_timeout: "1h" command: | cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - run: name: upload docker image no_output_timeout: "1h" command: | set +x export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY} export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY} eval $(aws ecr get-login --region us-east-1 --no-include-email) set -x docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:latest docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:latest unittest_linux_cpu: <<: *binary_common docker: - image: pytorch/torchaudio_unittest_base:manylinux-20210121 resource_class: 2xlarge+ steps: - checkout - attach_workspace: at: third_party - designate_upload_channel - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh - run: name: Run tests command: .circleci/unittest/linux/scripts/run_test.sh - run: name: Post process command: .circleci/unittest/linux/scripts/post_process.sh - store_test_results: path: test-results unittest_linux_gpu: <<: *binary_common machine: image: ubuntu-1604-cuda-10.1:201909-23 resource_class: gpu.small environment: <<: *environment image_name: pytorch/torchaudio_unittest_base:manylinux-cuda10.1-cudnn7-20210121 steps: - checkout - attach_workspace: at: third_party - designate_upload_channel - run: name: Pull Docker image command: docker pull --quiet "${image_name}" - run: name: Setup command: docker run -t --gpus all -e PYTHON_VERSION -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - run: name: Install torchaudio command: docker run -t --gpus all -e UPLOAD_CHANNEL -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh - run: name: Post Process command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh - store_test_results: path: test-results unittest_windows_cpu: <<: *binary_common executor: name: windows-cpu steps: - checkout - designate_upload_channel - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh - run: name: Install torchaudio command: .circleci/unittest/windows/scripts/install.sh - run: name: Run tests command: .circleci/unittest/windows/scripts/run_test.sh - run: name: Post process command: .circleci/unittest/windows/scripts/post_process.sh - store_test_results: path: test-results unittest_windows_gpu: <<: *binary_common executor: name: windows-gpu environment: <<: *environment CUDA_VERSION: "10.1" steps: - checkout - designate_upload_channel - run: name: Setup command: .circleci/unittest/windows/scripts/setup_env.sh - run: name: Install torchaudio command: .circleci/unittest/windows/scripts/install.sh - run: name: Run tests command: .circleci/unittest/windows/scripts/run_test.sh - run: name: Post process command: .circleci/unittest/windows/scripts/post_process.sh - store_test_results: path: test-results unittest_macos_cpu: <<: *binary_common macos: xcode: "9.4.1" resource_class: large steps: - checkout - install_build_tools_macos - attach_workspace: at: third_party - designate_upload_channel - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - run: name: Install torchaudio command: .circleci/unittest/linux/scripts/install.sh - run: name: Run tests command: .circleci/unittest/linux/scripts/run_test.sh - run: name: Post process command: .circleci/unittest/linux/scripts/post_process.sh - store_test_results: path: test-results stylecheck: <<: *binary_common docker: - image: "pytorch/torchaudio_unittest_base:manylinux" resource_class: medium steps: - checkout - designate_upload_channel - run: name: Setup command: .circleci/unittest/linux/scripts/setup_env.sh - run: name: Run style check command: .circleci/unittest/linux/scripts/run_style_checks.sh build_docs: <<: *binary_common docker: - image: "pytorch/manylinux-cuda100" resource_class: 2xlarge+ steps: - attach_workspace: at: ~/workspace - checkout - run: name: Install pytorch-audio command: .circleci/build_docs/install_wheels.sh - run: name: Build docs command: .circleci/build_docs/build_docs.sh - persist_to_workspace: root: ./ paths: - "*" upload_docs: <<: *binary_common docker: - image: "pytorch/manylinux-cuda100" resource_class: 2xlarge+ steps: - attach_workspace: at: ~/workspace - run: name: Generate netrc command: | # set credentials for https pushing # requires the org-member context cat > ~/.netrc \<