config.yml.in

version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
#       Job names are 'name:' key.

executors:
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
    machine:
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
      shell: bash.exe

commands:
  checkout_merge:
    description: "checkout merge branch"
    steps:
      - checkout
#     - run:
#         name: Checkout merge branch
#         command: |
#           set -ex
#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
#           if [[ "$BRANCH" != "master" ]]; then
#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
#             git checkout "merged/$CIRCLE_BRANCH"
#           fi
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}

binary_common: &binary_common
  parameters:
    # Edit these defaults to do a release`
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
      type: string
      default: "cpu"
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
      default: "pytorch/manylinux-cuda101"
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
    CU_VERSION: << parameters.cu_version >>

smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: torchvision/smoke_test:latest

jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off jinja2 pyyaml
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

  python_lint:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off flake8 typing
            flake8 --config=setup.cfg .

  python_type_check:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            sudo apt-get update -y
            sudo apt install -y libturbojpeg-dev
            pip install --user --progress-bar off numpy mypy
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            pip install --user --progress-bar off --editable .
            mypy --config-file mypy.ini

  clang_format:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format
            chmod +x clang-format
            sudo mv clang-format /opt/clang-format
            ./travis-scripts/run-clang-format/run-clang-format.py -r torchvision/csrc --clang-format-executable /opt/clang-format

  torchhub_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off numpy
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
            python test/test_hub.py

  torch_onnx_test:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
            pip install --user --progress-bar off numpy
            pip install --user --progress-bar off --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
            # need to install torchvision dependencies due to transitive imports
            pip install --user --progress-bar off --editable .
            pip install --user onnx
            pip install --user onnxruntime
            python test/test_onnx.py

  binary_linux_wheel:
    <<: *binary_common
    docker:
      - image: << parameters.wheel_docker_image >>
    resource_class: 2xlarge+
    steps:
      - checkout_merge
      - designate_upload_channel
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_linux_conda:
    <<: *binary_common
    docker:
      - image: "pytorch/conda-cuda"
    resource_class: 2xlarge+
    steps:
      - checkout_merge
      - designate_upload_channel
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  binary_win_conda:
    <<: *binary_common
    executor: windows-cpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Build conda packages
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
            packaging/build_conda.sh
            rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
      - store_artifacts:
          path: C:/tools/miniconda3/conda-bld/win-64
      - persist_to_workspace:
          root: C:/tools/miniconda3/conda-bld/win-64
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  binary_win_wheel:
    <<: *binary_common
    executor: windows-cpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Build wheel packages
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  binary_macos_wheel:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_macos_conda:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force

  # Requires org-member context
  binary_wheel_upload:
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
    docker:
      - image: circleci/python:3.7
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - checkout
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
            for pkg in ~/workspace/*.whl; do
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
            done

  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_docker_image_build:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchvision/smoke_test
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Build and push Docker image
          no_output_timeout: "1h"
          command: |
            set +x
            echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin
            set -x
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest
            docker push ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push ${image_name}:latest

  smoke_test_win_conda:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            conda install Pillow
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchvision*.tar.bz2)
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  smoke_test_win_pip:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          name: install binaries
          command: |
            set -x
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda env remove -n python${PYTHON_VERSION} || true
            conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION}
            conda activate python${PYTHON_VERSION}
            pip install $(ls ~/workspace/torchvision*.whl) --pre -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate python${PYTHON_VERSION}
            python -c "import torchvision"

  unittest_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
      image_name: "pytorch/manylinux-cuda101"
      PYTHON_VERSION: << parameters.python_version >>
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - run:
          name: Setup
          command: docker run -e PYTHON_VERSION -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post Process
          command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
      CUDA_VERSION: "10.1"
      PYTHON_VERSION: << parameters.python_version >>
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_macos_cpu:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    resource_class: large
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Install wget
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
          # Disable brew auto update which is very slow
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v3-macos-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/linux/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/linux/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/linux/scripts/post_process.sh
      - store_test_results:
          path: test-results

  cmake_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cuda102"
    resource_class: 2xlarge+
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Setup conda
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run: packaging/build_cmake.sh

  cmake_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-1604-cuda-10.1:201909-23
    resource_class: gpu.small
    environment:
      PYTHON_VERSION: << parameters.python_version >>
      PYTORCH_VERSION: << parameters.pytorch_version >>
      UNICODE_ABI: << parameters.unicode_abi >>
      CU_VERSION: << parameters.cu_version >>
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Setup conda
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Build torchvision C++ distribution and test
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh

  cmake_macos_cpu:
    <<: *binary_common
    macos:
      xcode: "9.4.1"
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build cmake
            packaging/build_cmake.sh

  cmake_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/build_cmake.sh

  cmake_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_cmake.sh

workflows:
  build:
{%- if True %}
    jobs:
      - circleci_consistency
      {{ build_workflows(windows_latest_only=True) }}
      - python_lint
      - python_type_check
      - clang_format
      - torchhub_test
      - torch_onnx_test

  unittest:
    jobs:
      {{ unittest_workflows() }}

  cmake:
    jobs:
      {{ cmake_workflows() }}

  nightly:
{%- endif %}
    jobs:
      - circleci_consistency
      - python_lint
      - python_type_check
      - clang_format
      - torchhub_test
      - torch_onnx_test
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
                - master
    jobs:
      - smoke_test_docker_image_build:
          context: org-member