Skip to content

GitLab

Unverified Commit 6713f034 authored by moto's avatar moto Committed by GitHub
Browse files

Update unittest and packaging to use UPLOAD_CHANNEL (#2846) (#2848) 


Co-authored-by: default avatarFrancisco Massa <fvsmassa@gmail.com>
parent 34708e4c
Hide whitespace changes
Inline Side-by-side
Showing with 53 additions and 12 deletions
.circleci/config.yml
View file @ 6713f034
......@@ -138,6 +138,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout_merge
- designate_upload_channel
- run: packaging/build_wheel.sh
- store_artifacts:
path: dist
......@@ -153,6 +154,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout_merge
- designate_upload_channel
- run: packaging/build_conda.sh
- store_artifacts:
path: /opt/conda/conda-bld/linux-64
......@@ -168,6 +170,7 @@ jobs:
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build conda packages
command: |
......@@ -193,6 +196,7 @@ jobs:
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build wheel packages
command: |
......@@ -215,6 +219,7 @@ jobs:
xcode: "9.4.1"
steps:
- checkout_merge
- designate_upload_channel
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
......@@ -237,6 +242,7 @@ jobs:
xcode: "9.4.1"
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
......@@ -299,6 +305,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -317,6 +324,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -337,6 +345,7 @@ jobs:
image_name: torchvision/smoke_test
steps:
- checkout
- designate_upload_channel
- run:
name: Build and push Docker image
no_output_timeout: "1h"
......@@ -356,6 +365,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -381,6 +391,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -404,6 +415,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -444,6 +456,7 @@ jobs:
image_name: "pytorch/manylinux-cuda101"
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -465,7 +478,7 @@ jobs:
- env
- run:
name: Install torchvision
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
......@@ -481,6 +494,7 @@ jobs:
name: windows-cpu
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -520,6 +534,7 @@ jobs:
CUDA_VERSION: "10.1"
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -558,6 +573,7 @@ jobs:
resource_class: large
steps:
- checkout
- designate_upload_channel
- run:
name: Install wget
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
......@@ -600,6 +616,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Setup conda
command: .circleci/unittest/linux/scripts/setup_env.sh
......@@ -617,12 +634,13 @@ jobs:
CU_VERSION: << parameters.cu_version >>
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Setup conda
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
- run:
name: Build torchvision C++ distribution and test
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
cmake_macos_cpu:
<<: *binary_common
......@@ -630,6 +648,7 @@ jobs:
xcode: "9.0"
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
......@@ -644,6 +663,7 @@ jobs:
name: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
set -ex
......@@ -656,6 +676,7 @@ jobs:
name: windows-gpu
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
set -ex
......
.circleci/config.yml.in
View file @ 6713f034
......@@ -138,6 +138,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout_merge
- designate_upload_channel
- run: packaging/build_wheel.sh
- store_artifacts:
path: dist
......@@ -153,6 +154,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout_merge
- designate_upload_channel
- run: packaging/build_conda.sh
- store_artifacts:
path: /opt/conda/conda-bld/linux-64
......@@ -168,6 +170,7 @@ jobs:
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build conda packages
command: |
......@@ -193,6 +196,7 @@ jobs:
executor: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Build wheel packages
command: |
......@@ -215,6 +219,7 @@ jobs:
xcode: "9.4.1"
steps:
- checkout_merge
- designate_upload_channel
- run:
# Cannot easily deduplicate this as source'ing activate
# will set environment variables which we need to propagate
......@@ -237,6 +242,7 @@ jobs:
xcode: "9.4.1"
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
......@@ -299,6 +305,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -317,6 +324,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -337,6 +345,7 @@ jobs:
image_name: torchvision/smoke_test
steps:
- checkout
- designate_upload_channel
- run:
name: Build and push Docker image
no_output_timeout: "1h"
......@@ -356,6 +365,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -381,6 +391,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
name: install binaries
command: |
......@@ -404,6 +415,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -444,6 +456,7 @@ jobs:
image_name: "pytorch/manylinux-cuda101"
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -465,7 +478,7 @@ jobs:
- env
- run:
name: Install torchvision
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD -e UPLOAD_CHANNEL "${image_name}" .circleci/unittest/linux/scripts/install.sh
- run:
name: Run tests
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh
......@@ -481,6 +494,7 @@ jobs:
name: windows-cpu
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -520,6 +534,7 @@ jobs:
CUDA_VERSION: "10.1"
steps:
- checkout
- designate_upload_channel
- run:
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
......@@ -558,6 +573,7 @@ jobs:
resource_class: large
steps:
- checkout
- designate_upload_channel
- run:
name: Install wget
command: HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
......@@ -600,6 +616,7 @@ jobs:
resource_class: 2xlarge+
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Setup conda
command: .circleci/unittest/linux/scripts/setup_env.sh
......@@ -617,12 +634,13 @@ jobs:
CU_VERSION: << parameters.cu_version >>
steps:
- checkout_merge
- designate_upload_channel
- run:
name: Setup conda
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> .circleci/unittest/linux/scripts/setup_env.sh
- run:
name: Build torchvision C++ distribution and test
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
command: docker run -e CU_VERSION -e PYTHON_VERSION -e UNICODE_ABI -e PYTORCH_VERSION -e UPLOAD_CHANNEL -t --gpus all -v $PWD:$PWD -w $PWD << parameters.wheel_docker_image >> packaging/build_cmake.sh
cmake_macos_cpu:
<<: *binary_common
......@@ -630,6 +648,7 @@ jobs:
xcode: "9.0"
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
......@@ -644,6 +663,7 @@ jobs:
name: windows-cpu
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
set -ex
......@@ -656,6 +676,7 @@ jobs:
name: windows-gpu
steps:
- checkout_merge
- designate_upload_channel
- run:
command: |
set -ex
......
.circleci/unittest/linux/scripts/install.sh
View file @ 6713f034
......@@ -23,7 +23,7 @@ else
cudatoolkit="cudatoolkit=${version}"
fi
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
printf "* Installing torchvision\n"
python setup.py develop
\ No newline at end of file
python setup.py develop
.circleci/unittest/windows/scripts/install.sh
View file @ 6713f034
......@@ -25,7 +25,7 @@ else
cudatoolkit="cudatoolkit=${version}"
fi
printf "Installing PyTorch with %s\n" "${cudatoolkit}"
conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
conda install -y -c "pytorch-${UPLOAD_CHANNEL}" pytorch "${cudatoolkit}"
printf "* Installing torchvision\n"
"$this_dir/vc_env_helper.bat" python setup.py develop
\ No newline at end of file
"$this_dir/vc_env_helper.bat" python setup.py develop
packaging/build_cmake.sh
View file @ 6713f034
......@@ -23,7 +23,7 @@ fi
setup_visual_studio_constraint
setup_junit_results_folder
conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE -c pytorch-nightly
conda install -yq pytorch=$PYTORCH_VERSION $CONDA_CUDATOOLKIT_CONSTRAINT $CONDA_CPUONLY_FEATURE -c "pytorch-${UPLOAD_CHANNEL}"
TORCH_PATH=$(dirname $(python -c "import torch; print(torch.__file__)"))
if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
......
packaging/pkg_helpers.bash
View file @ 6713f034
......@@ -235,8 +235,7 @@ setup_pip_pytorch_version() {
else
pip_install "torch==$PYTORCH_VERSION$PYTORCH_VERSION_SUFFIX" \
-f "https://download.pytorch.org/whl/${CU_VERSION}/torch_stable.html" \
-f "https://download.pytorch.org/whl/test/${CU_VERSION}/torch_test.html" \
-f "https://download.pytorch.org/whl/nightly/${CU_VERSION}/torch_nightly.html"
-f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${CU_VERSION}/torch_${UPLOAD_CHANNEL}.html"
fi
}
......@@ -261,7 +260,7 @@ setup_conda_pytorch_constraint() {
exit 1
fi
else
export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly -c pytorch-test"
export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-${UPLOAD_CHANNEL}"
fi
if [[ "$CU_VERSION" == cpu ]]; then
export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION${PYTORCH_VERSION_SUFFIX}"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment