config.yml.in

version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_win_wheel_py3.8
#     - Replace binary_win_wheel_py3.8 with the name of the job you want to test.
#       Job names are 'name:' key.

executors:
  windows-cpu:
    machine:
      resource_class: windows.xlarge
      image: windows-server-2019-vs2019:stable
      shell: bash.exe

  windows-gpu:
    machine:
      resource_class: windows.gpu.nvidia.medium
      image: windows-server-2019-nvidia:stable
      shell: bash.exe

commands:
  checkout_merge:
    description: "checkout merge branch"
    steps:
      - checkout
#     - run:
#         name: Checkout merge branch
#         command: |
#           set -ex
#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
#           if [[ "$BRANCH" != "main" ]]; then
#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
#             git checkout "merged/$CIRCLE_BRANCH"
#           fi
  designate_upload_channel:
    description: "inserts the correct upload channel into ${BASH_ENV}"
    steps:
      - run:
          name: adding UPLOAD_CHANNEL to BASH_ENV
          command: |
            our_upload_channel=nightly
            # On tags upload to test instead
            if [[ -n "${CIRCLE_TAG}" ]]; then
              our_upload_channel=test
            fi
            echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}

  brew_update:
    description: "Update Homebrew and install base formulae"
    steps:
      - run:
          name: Update Homebrew
          no_output_timeout: "10m"
          command: |
            set -ex

            # Update repositories manually.
            # Running `brew update` produces a comparison between the
            # current checkout and the updated checkout, which takes a
            # very long time because the existing checkout is 2y old.
            for path in $(find /usr/local/Homebrew -type d -name .git)
            do
            cd $path/..
            git fetch --depth=1 origin
            git reset --hard origin/master
            done

            export HOMEBREW_NO_AUTO_UPDATE=1

            # Install expect and moreutils so that we can call `unbuffer` and `ts`.
            # moreutils installs a `parallel` executable by default, which conflicts
            # with the executable from the GNU `parallel`, so we must unlink GNU
            # `parallel` first, and relink it afterwards.
            brew install coreutils
            brew unlink parallel
            brew install moreutils
            brew link parallel --overwrite
            brew install expect

  brew_install:
    description: "Install Homebrew formulae"
    parameters:
      formulae:
        type: string
        default: ""
    steps:
      - run:
          name: Install << parameters.formulae >>
          no_output_timeout: "10m"
          command: |
            set -ex
            export HOMEBREW_NO_AUTO_UPDATE=1
            brew install << parameters.formulae >>

  run_brew_for_ios_build:
    steps:
      - brew_update
      - brew_install:
          formulae: libtool

  pip_install:
    parameters:
      args:
        type: string
      descr:
        type: string
        default: ""
      user:
        type: boolean
        default: true
    steps:
      - run:
          name: >
            <<^ parameters.descr >> pip install << parameters.args >> <</ parameters.descr >>
            <<# parameters.descr >> << parameters.descr >>            <</ parameters.descr >>
          command: >
            pip install
            <<# parameters.user >> --user <</ parameters.user >>
            --progress-bar=off
            << parameters.args >>

binary_common: &binary_common
  parameters:
    # Edit these defaults to do a release
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.8)"
      type: string
    cu_version:
      description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
      type: string
      default: "cpu"
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
    wheel_docker_image:
      description: "Wheel only: what docker image to use"
      type: string
      default: ""
    conda_docker_image:
      description: "Conda only: what docker image to use"
      type: string
      default: "pytorch/conda-builder:cpu"
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
    CU_VERSION: << parameters.cu_version >>
    MACOSX_DEPLOYMENT_TARGET: 10.9

torchvision_ios_params: &torchvision_ios_params
  parameters:
    build_environment:
      type: string
      default: ""
    ios_arch:
      type: string
      default: ""
    ios_platform:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>
    IOS_ARCH: << parameters.ios_arch >>
    IOS_PLATFORM: << parameters.ios_platform >>

torchvision_android_params: &torchvision_android_params
  parameters:
    build_environment:
      type: string
      default: ""
  environment:
    BUILD_ENVIRONMENT: << parameters.build_environment >>

smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: torchvision/smoke_test:latest

jobs:
  circleci_consistency:
    docker:
      - image: cimg/python:3.8
    steps:
      - checkout
      - pip_install:
          args: jinja2 pyyaml
      - run:
          name: Check CircleCI config consistency
          command: |
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

  binary_win_conda:
    <<: *binary_common
    executor: windows-cpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Build conda packages
          no_output_timeout: 30m
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda install -yq conda-build "conda-package-handling!=1.5.0"
            packaging/build_conda.sh
            rm /C/tools/miniconda3/conda-bld/win-64/vs${VC_YEAR}*.tar.bz2
      - store_artifacts:
          path: C:/tools/miniconda3/conda-bld/win-64
      - persist_to_workspace:
          root: C:/tools/miniconda3/conda-bld/win-64
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  binary_win_wheel:
    <<: *binary_common
    executor: windows-cpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Build wheel packages
          no_output_timeout: 30m
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  binary_macos_wheel:
    <<: *binary_common
    macos:
      xcode: "14.0"
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
      - persist_to_workspace:
          root: dist
          paths:
            - "*"

  binary_ios_build:
    <<: *torchvision_ios_params
    macos:
      xcode: "14.0"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_build.sh"
          cat "$script"
          source "$script"
    - persist_to_workspace:
        root: /Users/distiller/workspace/
        paths: ios

  binary_ios_upload:
    <<: *torchvision_ios_params
    macos:
      xcode: "14.0"
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run_brew_for_ios_build
    - run:
        name: Upload
        no_output_timeout: "1h"
        command: |
          script="/Users/distiller/project/.circleci/unittest/ios/scripts/binary_ios_upload.sh"
          cat "$script"
          source "$script"

  binary_android_build:
    <<: *torchvision_android_params
    docker:
      - image: cimg/android:2021.08-ndk
    resource_class: xlarge
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run:
        name: Build
        no_output_timeout: "1h"
        command: |
          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_build.sh"
          cat "$script"
          source "$script"
    - store_artifacts:
        path: ~/workspace/artifacts

  binary_android_upload:
    <<: *torchvision_android_params
    docker:
      - image: cimg/android:2021.08-ndk
    resource_class: xlarge
    steps:
    - attach_workspace:
        at: ~/workspace
    - checkout
    - run:
        name: Upload
        no_output_timeout: "1h"
        command: |
          script="/home/circleci/project/.circleci/unittest/android/scripts/binary_android_upload.sh"
          cat "$script"
          source "$script"

  binary_macos_conda:
    <<: *binary_common
    macos:
      xcode: "14.0"
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
      - store_test_results:
          path: build_results/

  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set -x
            anaconda  -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force

  # Requires org-member context
  binary_wheel_upload:
    parameters:
      subfolder:
        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
        type: string
    docker:
      - image: cimg/python:3.8
    steps:
      - attach_workspace:
          at: ~/workspace
      - designate_upload_channel
      - checkout
      - pip_install:
          args: awscli
      - run:
          command: |
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
            for pkg in ~/workspace/*.whl; do
              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
            done

  smoke_test_docker_image_build:
    machine:
      image: ubuntu-2004:202104-01
    resource_class: large
    environment:
      image_name: torchvision/smoke_test
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Build and push Docker image
          no_output_timeout: "1h"
          command: |
            set +x
            echo "${DOCKER_HUB_TOKEN}" | docker login --username "${DOCKER_HUB_USERNAME}" --password-stdin
            set -x
            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} ${image_name}:latest
            docker push ${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push ${image_name}:latest

  unittest_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  unittest_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    environment:
      CUDA_VERSION: "11.7"
      PYTHON_VERSION: << parameters.python_version >>
    steps:
      - checkout
      - designate_upload_channel
      - run:
          name: Generate cache key
          # This will refresh cache on Sundays, nightly build should generate new cache.
          command: echo "$(date +"%Y-%U")" > .circleci-weekly
      - restore_cache:
          {% raw %}
          keys:
            - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
      - run:
          name: Setup
          command: .circleci/unittest/windows/scripts/setup_env.sh
      - save_cache:
          {% raw %}
          key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
          {% endraw %}
          paths:
            - conda
            - env
      - run:
          name: Install CUDA
          command: packaging/windows/internal/cuda_install.bat
      - run:
          name: Update CUDA driver
          command: packaging/windows/internal/driver_update.bat
      - run:
          name: Install torchvision
          command: .circleci/unittest/windows/scripts/install.sh
      - run:
          name: Run tests
          command: .circleci/unittest/windows/scripts/run_test.sh
      - run:
          name: Post process
          command: .circleci/unittest/windows/scripts/post_process.sh
      - store_test_results:
          path: test-results

  cmake_linux_cpu:
    <<: *binary_common
    docker:
      - image: "pytorch/manylinux-cpu"
    resource_class: 2xlarge+
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Setup conda
          command: .circleci/unittest/linux/scripts/setup_env.sh
      - run: packaging/build_cmake.sh

  cmake_linux_gpu:
    <<: *binary_common
    machine:
      image: ubuntu-2004-cuda-11.4:202110-01
    resource_class: gpu.nvidia.small
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Setup conda
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
      - run:
          name: Build torchvision C++ distribution and test
          no_output_timeout: 30m
          command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh

  cmake_macos_cpu:
    <<: *binary_common
    macos:
      xcode: "14.0"
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build cmake python=<< parameters.python_version >>
            packaging/build_cmake.sh

  cmake_windows_cpu:
    <<: *binary_common
    executor:
      name: windows-cpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate base
            conda create -yn python39 python=3.9
            conda activate python39
            packaging/build_cmake.sh

  cmake_windows_gpu:
    <<: *binary_common
    executor:
      name: windows-gpu
    steps:
      - checkout_merge
      - designate_upload_channel
      - run:
          name: Update CUDA driver
          command: packaging/windows/internal/driver_update.bat
      - run:
          command: |
            set -ex
            source packaging/windows/internal/vc_install_helper.sh
            packaging/windows/internal/cuda_install.bat
            eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
            conda activate
            conda update -y conda
            conda create -yn python39 python=3.9
            conda activate python39
            packaging/build_cmake.sh


workflows:
  lint:
    jobs:
      - circleci_consistency

  build:
    jobs:
      {{ build_workflows(windows_latest_only=True) }}
      {{ ios_workflows() }}
      {{ android_workflows() }}

  unittest:
    jobs:
      {{ unittest_workflows() }}

  cmake:
    jobs:
      {{ cmake_workflows() }}

  nightly:
    jobs:
      {{ ios_workflows(nightly=True) }}
      {{ android_workflows(nightly=True) }}
      {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
                - main
    jobs:
      - smoke_test_docker_image_build:
          context: org-member