Skip to content

GitLab

Unverified Commit 9d9e716f authored by guyang3532's avatar guyang3532 Committed by GitHub
Browse files

Add unittest job to CircleCI (#2328)

parent e50c2e36
Hide whitespace changes
Inline Side-by-side
Showing with 690 additions and 280 deletions
.circleci/config.yml
View file @ 9d9e716f
......@@ -155,106 +155,7 @@ jobs:
- store_test_results:
path: build_results/
binary_linux_conda_cuda:
<<: *binary_common
machine:
image: ubuntu-1604:201903-01
resource_class: gpu.medium
steps:
- checkout_merge
- run:
name: Setup environment
command: |
set -ex
curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
sudo apt-get update
sudo apt-get install \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
sudo apt-get update
export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial"
sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io=1.2.6-3
# Add the package repositories
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
export NVIDIA_CONTAINER_VERSION="1.0.3-1"
sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION}
sudo systemctl restart docker
DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
nvidia-smi
- run:
name: Pull docker image
command: |
set -ex
export DOCKER_IMAGE=pytorch/conda-cuda
echo Pulling docker image $DOCKER_IMAGE
docker pull $DOCKER_IMAGE >/dev/null
- run:
name: Build and run tests
command: |
set -ex
cd ${HOME}/project/
export DOCKER_IMAGE=pytorch/conda-cuda
export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e UNICODE_ABI -e CU_VERSION"
docker run --gpus all --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh
binary_win_conda:
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- run:
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
- store_test_results:
path: build_results/
binary_win_conda_cuda:
<<: *binary_common
executor: windows-gpu
steps:
- checkout_merge
- run:
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
binary_win_conda_release:
<<: *binary_common
executor: windows-cpu
steps:
......@@ -279,7 +180,7 @@ jobs:
- store_test_results:
path: build_results/
binary_win_wheel_release:
binary_win_wheel:
<<: *binary_common
executor: windows-cpu
steps:
......@@ -385,6 +286,159 @@ jobs:
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
unittest_linux_cpu:
<<: *binary_common
docker:
- image: "pytorch/manylinux-cuda102"
resource_class: 2xlarge+
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
- run:
name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
paths:
- conda
- env
- run:
name: Install torchvision
command: .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: .circleci/unittest/linux/scripts/run_test.sh
- run:
name: Post process
command: .circleci/unittest/linux/scripts/post_process.sh
- store_test_results:
path: test-results
unittest_linux_gpu:
<<: *binary_common
machine:
image: ubuntu-1604-cuda-10.1:201909-23
resource_class: gpu.small
environment:
image_name: "pytorch/manylinux-cuda101"
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
- run:
name: Setup
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
paths:
- conda
- env
- run:
name: Install torchvision
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
- run:
name: Post Process
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
- store_test_results:
path: test-results
unittest_windows_cpu:
<<: *binary_common
executor:
name: windows-cpu
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
keys:
- env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
- run:
name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache:
key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
paths:
- conda
- env
- run:
name: Install torchvision
command: .circleci/unittest/windows/scripts/install.sh
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
- run:
name: Post process
command: .circleci/unittest/windows/scripts/post_process.sh
- store_test_results:
path: test-results
unittest_windows_gpu:
<<: *binary_common
executor:
name: windows-gpu
environment:
CUDA_VERSION: "10.1"
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
keys:
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
- run:
name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache:
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
paths:
- conda
- env
- run:
name: Install torchvision
command: .circleci/unittest/windows/scripts/install.sh
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
- run:
name: Post process
command: .circleci/unittest/windows/scripts/post_process.sh
- store_test_results:
path: test-results
workflows:
build:
......@@ -465,7 +519,7 @@ workflows:
name: binary_macos_wheel_py3.8_cpu
python_version: '3.8'
wheel_docker_image: pytorch/manylinux-cuda102
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cpu
filters:
branches:
......@@ -474,7 +528,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.6_cpu
python_version: '3.6'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu92
filters:
branches:
......@@ -483,7 +537,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.6_cu92
python_version: '3.6'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu101
filters:
branches:
......@@ -492,7 +546,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.6_cu101
python_version: '3.6'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu102
filters:
branches:
......@@ -501,7 +555,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.6_cu102
python_version: '3.6'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cpu
filters:
branches:
......@@ -510,7 +564,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.7_cpu
python_version: '3.7'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu92
filters:
branches:
......@@ -519,7 +573,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.7_cu92
python_version: '3.7'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu101
filters:
branches:
......@@ -528,7 +582,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.7_cu101
python_version: '3.7'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu102
filters:
branches:
......@@ -537,11 +591,11 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.7_cu102
python_version: '3.7'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cpu
name: binary_win_wheel_py3.8_cpu
python_version: '3.8'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu92
filters:
branches:
......@@ -550,7 +604,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.8_cu92
python_version: '3.8'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu101
filters:
branches:
......@@ -559,7 +613,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_wheel_py3.8_cu101
python_version: '3.8'
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu102
name: binary_win_wheel_py3.8_cu102
python_version: '3.8'
......@@ -638,7 +692,7 @@ workflows:
name: binary_macos_conda_py3.8_cpu
python_version: '3.8'
wheel_docker_image: pytorch/manylinux-cuda102
- binary_win_conda_release:
- binary_win_conda:
cu_version: cpu
filters:
branches:
......@@ -647,7 +701,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.6_cpu
python_version: '3.6'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu92
filters:
branches:
......@@ -656,7 +710,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.6_cu92
python_version: '3.6'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu101
filters:
branches:
......@@ -665,7 +719,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.6_cu101
python_version: '3.6'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu102
filters:
branches:
......@@ -674,7 +728,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.6_cu102
python_version: '3.6'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cpu
filters:
branches:
......@@ -683,7 +737,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.7_cpu
python_version: '3.7'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu92
filters:
branches:
......@@ -692,7 +746,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.7_cu92
python_version: '3.7'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu101
filters:
branches:
......@@ -701,7 +755,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.7_cu101
python_version: '3.7'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu102
filters:
branches:
......@@ -710,11 +764,11 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.7_cu102
python_version: '3.7'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cpu
name: binary_win_conda_py3.8_cpu
python_version: '3.8'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu92
filters:
branches:
......@@ -723,7 +777,7 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.8_cu92
python_version: '3.8'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu101
filters:
branches:
......@@ -732,26 +786,84 @@ workflows:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: binary_win_conda_py3.8_cu101
python_version: '3.8'
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu102
name: binary_win_conda_py3.8_cu102
python_version: '3.8'
- binary_linux_conda_cuda:
name: torchvision_linux_py3.8_cu102_cuda
python_version: "3.8"
cu_version: "cu102"
- binary_win_conda:
name: torchvision_win_py3.6_cpu
python_version: "3.6"
cu_version: "cpu"
- binary_win_conda_cuda:
name: torchvision_win_py3.6_cu101
python_version: "3.6"
cu_version: "cu101"
- python_lint
- python_type_check
- clang_format
unittest:
jobs:
- unittest_linux_cpu:
cu_version: cpu
name: unittest_linux_cpu_py3.6
python_version: '3.6'
- unittest_linux_cpu:
cu_version: cpu
name: unittest_linux_cpu_py3.7
python_version: '3.7'
- unittest_linux_cpu:
cu_version: cpu
name: unittest_linux_cpu_py3.8
python_version: '3.8'
- unittest_linux_gpu:
cu_version: cu101
filters:
branches:
only:
- master
- nightly
name: unittest_linux_gpu_py3.6
python_version: '3.6'
- unittest_linux_gpu:
cu_version: cu101
filters:
branches:
only:
- master
- nightly
name: unittest_linux_gpu_py3.7
python_version: '3.7'
- unittest_linux_gpu:
cu_version: cu101
name: unittest_linux_gpu_py3.8
python_version: '3.8'
- unittest_windows_cpu:
cu_version: cpu
name: unittest_windows_cpu_py3.6
python_version: '3.6'
- unittest_windows_cpu:
cu_version: cpu
name: unittest_windows_cpu_py3.7
python_version: '3.7'
- unittest_windows_cpu:
cu_version: cpu
name: unittest_windows_cpu_py3.8
python_version: '3.8'
- unittest_windows_gpu:
cu_version: cu101
filters:
branches:
only:
- master
- nightly
name: unittest_windows_gpu_py3.6
python_version: '3.6'
- unittest_windows_gpu:
cu_version: cu101
filters:
branches:
only:
- master
- nightly
name: unittest_windows_gpu_py3.7
python_version: '3.7'
- unittest_windows_gpu:
cu_version: cu101
name: unittest_windows_gpu_py3.8
python_version: '3.8'
nightly:
jobs:
- circleci_consistency
......@@ -1073,7 +1185,7 @@ workflows:
requires:
- nightly_binary_macos_wheel_py3.8_cpu
subfolder: ''
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cpu
filters:
branches:
......@@ -1093,7 +1205,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.6_cpu
subfolder: cpu/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu92
filters:
branches:
......@@ -1113,7 +1225,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.6_cu92
subfolder: cu92/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu101
filters:
branches:
......@@ -1133,7 +1245,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.6_cu101
subfolder: cu101/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu102
filters:
branches:
......@@ -1153,7 +1265,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.6_cu102
subfolder: cu102/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cpu
filters:
branches:
......@@ -1173,7 +1285,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.7_cpu
subfolder: cpu/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu92
filters:
branches:
......@@ -1193,7 +1305,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.7_cu92
subfolder: cu92/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu101
filters:
branches:
......@@ -1213,7 +1325,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.7_cu101
subfolder: cu101/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu102
filters:
branches:
......@@ -1233,7 +1345,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.7_cu102
subfolder: cu102/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cpu
filters:
branches:
......@@ -1253,7 +1365,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.8_cpu
subfolder: cpu/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu92
filters:
branches:
......@@ -1273,7 +1385,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.8_cu92
subfolder: cu92/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu101
filters:
branches:
......@@ -1293,7 +1405,7 @@ workflows:
requires:
- nightly_binary_win_wheel_py3.8_cu101
subfolder: cu101/
- binary_win_wheel_release:
- binary_win_wheel:
cu_version: cu102
filters:
branches:
......@@ -1613,7 +1725,7 @@ workflows:
name: nightly_binary_macos_conda_py3.8_cpu_upload
requires:
- nightly_binary_macos_conda_py3.8_cpu
- binary_win_conda_release:
- binary_win_conda:
cu_version: cpu
filters:
branches:
......@@ -1632,7 +1744,7 @@ workflows:
name: nightly_binary_win_conda_py3.6_cpu_upload
requires:
- nightly_binary_win_conda_py3.6_cpu
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu92
filters:
branches:
......@@ -1651,7 +1763,7 @@ workflows:
name: nightly_binary_win_conda_py3.6_cu92_upload
requires:
- nightly_binary_win_conda_py3.6_cu92
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu101
filters:
branches:
......@@ -1670,7 +1782,7 @@ workflows:
name: nightly_binary_win_conda_py3.6_cu101_upload
requires:
- nightly_binary_win_conda_py3.6_cu101
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu102
filters:
branches:
......@@ -1689,7 +1801,7 @@ workflows:
name: nightly_binary_win_conda_py3.6_cu102_upload
requires:
- nightly_binary_win_conda_py3.6_cu102
- binary_win_conda_release:
- binary_win_conda:
cu_version: cpu
filters:
branches:
......@@ -1708,7 +1820,7 @@ workflows:
name: nightly_binary_win_conda_py3.7_cpu_upload
requires:
- nightly_binary_win_conda_py3.7_cpu
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu92
filters:
branches:
......@@ -1727,7 +1839,7 @@ workflows:
name: nightly_binary_win_conda_py3.7_cu92_upload
requires:
- nightly_binary_win_conda_py3.7_cu92
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu101
filters:
branches:
......@@ -1746,7 +1858,7 @@ workflows:
name: nightly_binary_win_conda_py3.7_cu101_upload
requires:
- nightly_binary_win_conda_py3.7_cu101
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu102
filters:
branches:
......@@ -1765,7 +1877,7 @@ workflows:
name: nightly_binary_win_conda_py3.7_cu102_upload
requires:
- nightly_binary_win_conda_py3.7_cu102
- binary_win_conda_release:
- binary_win_conda:
cu_version: cpu
filters:
branches:
......@@ -1784,7 +1896,7 @@ workflows:
name: nightly_binary_win_conda_py3.8_cpu_upload
requires:
- nightly_binary_win_conda_py3.8_cpu
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu92
filters:
branches:
......@@ -1803,7 +1915,7 @@ workflows:
name: nightly_binary_win_conda_py3.8_cu92_upload
requires:
- nightly_binary_win_conda_py3.8_cu92
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu101
filters:
branches:
......@@ -1822,7 +1934,7 @@ workflows:
name: nightly_binary_win_conda_py3.8_cu101_upload
requires:
- nightly_binary_win_conda_py3.8_cu101
- binary_win_conda_release:
- binary_win_conda:
cu_version: cu102
filters:
branches:
......
.circleci/config.yml.in
View file @ 9d9e716f
......@@ -155,106 +155,7 @@ jobs:
- store_test_results:
path: build_results/
binary_linux_conda_cuda:
<<: *binary_common
machine:
image: ubuntu-1604:201903-01
resource_class: gpu.medium
steps:
- checkout_merge
- run:
name: Setup environment
command: |
set -ex
curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
curl -L https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
sudo apt-get update
sudo apt-get install \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
sudo apt-get update
export DOCKER_VERSION="5:19.03.2~3-0~ubuntu-xenial"
sudo apt-get install docker-ce=${DOCKER_VERSION} docker-ce-cli=${DOCKER_VERSION} containerd.io=1.2.6-3
# Add the package repositories
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
export NVIDIA_CONTAINER_VERSION="1.0.3-1"
sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit=${NVIDIA_CONTAINER_VERSION}
sudo systemctl restart docker
DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
nvidia-smi
- run:
name: Pull docker image
command: |
set -ex
export DOCKER_IMAGE=pytorch/conda-cuda
echo Pulling docker image $DOCKER_IMAGE
docker pull $DOCKER_IMAGE >/dev/null
- run:
name: Build and run tests
command: |
set -ex
cd ${HOME}/project/
export DOCKER_IMAGE=pytorch/conda-cuda
export VARS_TO_PASS="-e PYTHON_VERSION -e BUILD_VERSION -e PYTORCH_VERSION -e UNICODE_ABI -e CU_VERSION"
docker run --gpus all --ipc=host -v $(pwd):/remote -w /remote ${VARS_TO_PASS} ${DOCKER_IMAGE} ./packaging/build_conda.sh
binary_win_conda:
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- run:
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
- store_test_results:
path: build_results/
binary_win_conda_cuda:
<<: *binary_common
executor: windows-gpu
steps:
- checkout_merge
- run:
command: |
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
binary_win_conda_release:
<<: *binary_common
executor: windows-cpu
steps:
......@@ -279,7 +180,7 @@ jobs:
- store_test_results:
path: build_results/
binary_win_wheel_release:
binary_win_wheel:
<<: *binary_common
executor: windows-cpu
steps:
......@@ -385,29 +286,173 @@ jobs:
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
unittest_linux_cpu:
<<: *binary_common
docker:
- image: "pytorch/manylinux-cuda102"
resource_class: 2xlarge+
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
{% raw %}
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
- run:
name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
{% raw %}
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
paths:
- conda
- env
- run:
name: Install torchvision
command: .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: .circleci/unittest/linux/scripts/run_test.sh
- run:
name: Post process
command: .circleci/unittest/linux/scripts/post_process.sh
- store_test_results:
path: test-results
unittest_linux_gpu:
<<: *binary_common
machine:
image: ubuntu-1604-cuda-10.1:201909-23
resource_class: gpu.small
environment:
image_name: "pytorch/manylinux-cuda101"
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
{% raw %}
keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
- run:
name: Setup
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- save_cache:
{% raw %}
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
paths:
- conda
- env
- run:
name: Install torchvision
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
- run:
name: Post Process
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh
- store_test_results:
path: test-results
unittest_windows_cpu:
<<: *binary_common
executor:
name: windows-cpu
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
{% raw %}
keys:
- env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
- run:
name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache:
{% raw %}
key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
paths:
- conda
- env
- run:
name: Install torchvision
command: .circleci/unittest/windows/scripts/install.sh
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
- run:
name: Post process
command: .circleci/unittest/windows/scripts/post_process.sh
- store_test_results:
path: test-results
unittest_windows_gpu:
<<: *binary_common
executor:
name: windows-gpu
environment:
CUDA_VERSION: "10.1"
steps:
- checkout
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache:
{% raw %}
keys:
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
- run:
name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache:
{% raw %}
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }}
{% endraw %}
paths:
- conda
- env
- run:
name: Install torchvision
command: .circleci/unittest/windows/scripts/install.sh
- run:
name: Run tests
command: .circleci/unittest/windows/scripts/run_test.sh
- run:
name: Post process
command: .circleci/unittest/windows/scripts/post_process.sh
- store_test_results:
path: test-results
workflows:
build:
{%- if True %}
jobs:
- circleci_consistency
{{ workflows(windows_latest_only=True) }}
- binary_linux_conda_cuda:
name: torchvision_linux_py3.8_cu102_cuda
python_version: "3.8"
cu_version: "cu102"
- binary_win_conda:
name: torchvision_win_py3.6_cpu
python_version: "3.6"
cu_version: "cpu"
- binary_win_conda_cuda:
name: torchvision_win_py3.6_cu101
python_version: "3.6"
cu_version: "cu101"
{{ build_workflows(windows_latest_only=True) }}
- python_lint
- python_type_check
- clang_format
unittest:
jobs:
{{ unittest_workflows() }}
nightly:
{%- endif %}
jobs:
......@@ -415,4 +460,4 @@ workflows:
- python_lint
- python_type_check
- clang_format
{{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
{{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
.circleci/regenerate.py
View file @ 9d9e716f
......@@ -19,11 +19,14 @@ import yaml
import os.path
def workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]
def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
w = []
for btype in ["wheel", "conda"]:
for os_type in ["linux", "macos", "win"]:
python_versions = ["3.6", "3.7", "3.8"]
python_versions = PYTHON_VERSIONS
cu_versions = (["cpu", "cu92", "cu101", "cu102"] if os_type == "linux" or os_type == "win" else ["cpu"])
for python_version in python_versions:
for cu_version in cu_versions:
......@@ -97,10 +100,14 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version,
}
}
w = f"binary_{os_type}_{btype}_release" if os_type == "win" else f"binary_{os_type}_{btype}"
w = f"binary_{os_type}_{btype}"
return {w: d}
def gen_filter_branch_tree(*branches):
return {"branches": {"only": [b for b in branches]}}
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
d = {
"name": f"{base_workflow_name}_upload",
......@@ -131,6 +138,28 @@ def indent(indentation, data_list):
yaml.dump(data_list, default_flow_style=False).splitlines())
def unittest_workflows(indentation=6):
jobs = []
for os_type in ["linux", "windows"]:
for device_type in ["cpu", "gpu"]:
for i, python_version in enumerate(PYTHON_VERSIONS):
job = {
"name": f"unittest_{os_type}_{device_type}_py{python_version}",
"python_version": python_version,
}
if device_type == 'gpu':
if python_version != "3.8":
job['filters'] = gen_filter_branch_tree('master', 'nightly')
job['cu_version'] = 'cu101'
else:
job['cu_version'] = 'cpu'
jobs.append({f"unittest_{os_type}_{device_type}": job})
return indent(indentation, jobs)
if __name__ == "__main__":
d = os.path.dirname(__file__)
env = jinja2.Environment(
......@@ -140,4 +169,7 @@ if __name__ == "__main__":
)
with open(os.path.join(d, 'config.yml'), 'w') as f:
f.write(env.get_template('config.yml.in').render(workflows=workflows))
f.write(env.get_template('config.yml.in').render(
build_workflows=build_workflows,
unittest_workflows=unittest_workflows,
))
.circleci/unittest/linux/scripts/environment.yml 0 → 100644
View file @ 9d9e716f
channels:
- defaults
dependencies:
- numpy
- pytest
- pytest-cov
- codecov
- pip
- ca-certificates
- pip:
- future
- pillow>=4.1.1
- scipy
- av
\ No newline at end of file
.circleci/unittest/linux/scripts/install.sh 0 → 100755
View file @ 9d9e716f
#!/usr/bin/env bash
unset PYTORCH_VERSION
# For unittest, nightly PyTorch is used as the following section,
# so no need to set PYTORCH_VERSION.
# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
set -e
eval "$(./conda/bin/conda shell.bash hook)"
conda activate ./env
if [ -z "${CUDA_VERSION:-}" ] ; then
cudatoolkit="cpuonly"
else
version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
cudatoolkit="cudatoolkit=${version}"
fi
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
printf "* Installing torchvision\n"
python setup.py develop
\ No newline at end of file
.circleci/unittest/linux/scripts/post_process.sh 0 → 100755
View file @ 9d9e716f
#!/usr/bin/env bash
set -e
eval "$(./conda/bin/conda shell.bash hook)"
conda activate ./env
codecov
\ No newline at end of file
.circleci/unittest/linux/scripts/run_test.sh 0 → 100755
View file @ 9d9e716f
#!/usr/bin/env bash
set -e
eval "$(./conda/bin/conda shell.bash hook)"
conda activate ./env
python -m torch.utils.collect_env
pytest --cov=torchvision --junitxml=test-results/junit.xml -v --durations 20 test
\ No newline at end of file
.circleci/unittest/linux/scripts/setup_env.sh 0 → 100755
View file @ 9d9e716f
#!/usr/bin/env bash
# This script is for setting up environment in which unit test is ran.
# To speed up the CI time, the resulting environment is cached.
#
# Do not install PyTorch and torchvision here, otherwise they also get cached.
set -e
this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
root_dir="$(git rev-parse --show-toplevel)"
conda_dir="${root_dir}/conda"
env_dir="${root_dir}/env"
cd "${root_dir}"
# 1. Install conda at ./conda
if [ ! -d "${conda_dir}" ]; then
printf "* Installing conda\n"
wget -O miniconda.sh http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
bash ./miniconda.sh -b -f -p "${conda_dir}"
fi
eval "$(${conda_dir}/bin/conda shell.bash hook)"
# 2. Create test environment at ./env
if [ ! -d "${env_dir}" ]; then
printf "* Creating a test environment\n"
conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
fi
conda activate "${env_dir}"
# 3. Install Conda dependencies
printf "* Installing dependencies (except PyTorch)\n"
conda env update --file "${this_dir}/environment.yml" --prune
.circleci/unittest/windows/scripts/environment.yml 0 → 100644
View file @ 9d9e716f
channels:
- defaults
dependencies:
- numpy
- pytest
- pytest-cov
- codecov
- pip
- ca-certificates
- pip:
- future
- pillow>=4.1.1
- scipy==1.4.1
- av
\ No newline at end of file
.circleci/unittest/windows/scripts/install.sh 0 → 100644
View file @ 9d9e716f
#!/usr/bin/env bash
unset PYTORCH_VERSION
# For unittest, nightly PyTorch is used as the following section,
# so no need to set PYTORCH_VERSION.
# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config.
set -e
this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
conda activate ./env
if [ -z "${CUDA_VERSION:-}" ] ; then
cudatoolkit="cpuonly"
else
version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
cudatoolkit="cudatoolkit=${version}"
fi
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
printf "* Installing torchvision\n"
"$this_dir/vc_env_helper.bat" python setup.py develop
\ No newline at end of file
.circleci/unittest/windows/scripts/install_conda.bat 0 → 100644
View file @ 9d9e716f
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%
\ No newline at end of file
.circleci/unittest/windows/scripts/post_process.sh 0 → 100644
View file @ 9d9e716f
#!/usr/bin/env bash
set -e
eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
conda activate ./env
codecov
.circleci/unittest/windows/scripts/run_test.sh 0 → 100644
View file @ 9d9e716f
#!/usr/bin/env bash
set -e
eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
conda activate ./env
python -m torch.utils.collect_env
pytest --cov=torchvision --junitxml=test-results/junit.xml -v --durations 20 test
\ No newline at end of file
.circleci/unittest/windows/scripts/setup_env.sh 0 → 100644
View file @ 9d9e716f
#!/usr/bin/env bash
# This script is for setting up environment in which unit test is ran.
# To speed up the CI time, the resulting environment is cached.
#
# Do not install PyTorch and torchvision here, otherwise they also get cached.
set -e
this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
root_dir="$(git rev-parse --show-toplevel)"
conda_dir="${root_dir}/conda"
env_dir="${root_dir}/env"
cd "${root_dir}"
# 1. Install conda at ./conda
if [ ! -d "${conda_dir}" ]; then
printf "* Installing conda\n"
export tmp_conda="$(echo $conda_dir | tr '/' '\\')"
export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe"
curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O
"$this_dir/install_conda.bat"
unset tmp_conda
unset miniconda_exe
fi
eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')"
# 2. Create test environment at ./env
if [ ! -d "${env_dir}" ]; then
printf "* Creating a test environment\n"
conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION"
fi
conda activate "${env_dir}"
# 3. Install Conda dependencies
printf "* Installing dependencies (except PyTorch)\n"
conda env update --file "${this_dir}/environment.yml" --prune
\ No newline at end of file
.circleci/unittest/windows/scripts/vc_env_helper.bat 0 → 100644
View file @ 9d9e716f
@echo on
set VC_VERSION_LOWER=16
set VC_VERSION_UPPER=17
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15INSTALLDIR=%%i"
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto vswhere
)
)
:vswhere
if "%VSDEVCMD_ARGS%" == "" (
call "%VS15VCVARSALL%" x64 || exit /b 1
) else (
call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1
)
@echo on
set DISTUTILS_USE_SDK=1
set args=%1
shift
:start
if [%1] == [] goto done
set args=%args% %1
shift
goto start
:done
if "%args%" == "" (
echo Usage: vc_env_helper.bat [command] [args]
echo e.g. vc_env_helper.bat cl /c test.cpp
)
%args% || exit /b 1
packaging/torchvision/meta.yaml
View file @ 9d9e716f
......@@ -47,8 +47,6 @@ test:
- av
- ca-certificates
{{ environ.get('CONDA_TYPING_CONSTRAINT') }}
commands:
pytest . --verbose --junitxml={{ environ.get("CONDA_PYTORCH_BUILD_RESULTS_DIRECTORY", "build/test_results.xml" )}}
about:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment