Skip to content

GitLab

Unverified Commit 40ad5572 authored by Eli Uriegas's avatar Eli Uriegas Committed by GitHub
Browse files

ci: Automate uploading to correct upload channel (#2204)

parent 4e138bd5
Show whitespace changes
Inline Side-by-side
Showing with 156 additions and 7 deletions
.circleci/config.yml
View file @ 40ad5572
......@@ -30,6 +30,18 @@ commands:
# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
# git checkout "merged/$CIRCLE_BRANCH"
# fi
designate_upload_channel:
description: "inserts the correct upload channel into ${BASH_ENV}"
steps:
- run:
name: adding UPLOAD_CHANNEL to BASH_ENV
command: |
our_upload_channel=nightly
# On tags upload to test instead
if [[ -n "${CIRCLE_TAG}" ]]; then
our_upload_channel=test
fi
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
binary_common: &binary_common
parameters:
......@@ -59,7 +71,6 @@ binary_common: &binary_common
default: "pytorch/manylinux-cuda101"
environment:
PYTHON_VERSION: << parameters.python_version >>
BUILD_VERSION: << parameters.build_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: << parameters.cu_version >>
......@@ -358,12 +369,13 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set -x
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
......@@ -376,6 +388,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- checkout
- run:
command: |
......@@ -387,7 +400,7 @@ jobs:
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/<< parameters.subfolder >>" --acl public-read
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
......@@ -736,6 +749,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.6_cpu_upload
requires:
- nightly_binary_linux_wheel_py3.6_cpu
......@@ -753,6 +768,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.6_cu92_upload
requires:
- nightly_binary_linux_wheel_py3.6_cu92
......@@ -770,6 +787,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.6_cu101_upload
requires:
- nightly_binary_linux_wheel_py3.6_cu101
......@@ -787,6 +806,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.6_cu102_upload
requires:
- nightly_binary_linux_wheel_py3.6_cu102
......@@ -804,6 +825,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.7_cpu_upload
requires:
- nightly_binary_linux_wheel_py3.7_cpu
......@@ -821,6 +844,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.7_cu92_upload
requires:
- nightly_binary_linux_wheel_py3.7_cu92
......@@ -838,6 +863,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.7_cu101_upload
requires:
- nightly_binary_linux_wheel_py3.7_cu101
......@@ -855,6 +882,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.7_cu102_upload
requires:
- nightly_binary_linux_wheel_py3.7_cu102
......@@ -872,6 +901,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.8_cpu_upload
requires:
- nightly_binary_linux_wheel_py3.8_cpu
......@@ -889,6 +920,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.8_cu92_upload
requires:
- nightly_binary_linux_wheel_py3.8_cu92
......@@ -906,6 +939,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.8_cu101_upload
requires:
- nightly_binary_linux_wheel_py3.8_cu101
......@@ -923,6 +958,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_wheel_py3.8_cu102_upload
requires:
- nightly_binary_linux_wheel_py3.8_cu102
......@@ -940,6 +977,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_macos_wheel_py3.6_cpu_upload
requires:
- nightly_binary_macos_wheel_py3.6_cpu
......@@ -957,6 +996,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_macos_wheel_py3.7_cpu_upload
requires:
- nightly_binary_macos_wheel_py3.7_cpu
......@@ -974,6 +1015,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_macos_wheel_py3.8_cpu_upload
requires:
- nightly_binary_macos_wheel_py3.8_cpu
......@@ -990,6 +1033,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.6_cpu_upload
requires:
- nightly_binary_win_wheel_py3.6_cpu
......@@ -1006,6 +1051,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.6_cu92_upload
requires:
- nightly_binary_win_wheel_py3.6_cu92
......@@ -1022,6 +1069,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.6_cu101_upload
requires:
- nightly_binary_win_wheel_py3.6_cu101
......@@ -1038,6 +1087,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.6_cu102_upload
requires:
- nightly_binary_win_wheel_py3.6_cu102
......@@ -1054,6 +1105,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.7_cpu_upload
requires:
- nightly_binary_win_wheel_py3.7_cpu
......@@ -1070,6 +1123,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.7_cu92_upload
requires:
- nightly_binary_win_wheel_py3.7_cu92
......@@ -1086,6 +1141,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.7_cu101_upload
requires:
- nightly_binary_win_wheel_py3.7_cu101
......@@ -1102,6 +1159,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.7_cu102_upload
requires:
- nightly_binary_win_wheel_py3.7_cu102
......@@ -1118,6 +1177,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.8_cpu_upload
requires:
- nightly_binary_win_wheel_py3.8_cpu
......@@ -1134,6 +1195,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.8_cu92_upload
requires:
- nightly_binary_win_wheel_py3.8_cu92
......@@ -1150,6 +1213,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.8_cu101_upload
requires:
- nightly_binary_win_wheel_py3.8_cu101
......@@ -1166,6 +1231,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_wheel_py3.8_cu102_upload
requires:
- nightly_binary_win_wheel_py3.8_cu102
......@@ -1183,6 +1250,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.6_cpu_upload
requires:
- nightly_binary_linux_conda_py3.6_cpu
......@@ -1199,6 +1268,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.6_cu92_upload
requires:
- nightly_binary_linux_conda_py3.6_cu92
......@@ -1215,6 +1286,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.6_cu101_upload
requires:
- nightly_binary_linux_conda_py3.6_cu101
......@@ -1231,6 +1304,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.6_cu102_upload
requires:
- nightly_binary_linux_conda_py3.6_cu102
......@@ -1247,6 +1322,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.7_cpu_upload
requires:
- nightly_binary_linux_conda_py3.7_cpu
......@@ -1263,6 +1340,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.7_cu92_upload
requires:
- nightly_binary_linux_conda_py3.7_cu92
......@@ -1279,6 +1358,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.7_cu101_upload
requires:
- nightly_binary_linux_conda_py3.7_cu101
......@@ -1295,6 +1376,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.7_cu102_upload
requires:
- nightly_binary_linux_conda_py3.7_cu102
......@@ -1311,6 +1394,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.8_cpu_upload
requires:
- nightly_binary_linux_conda_py3.8_cpu
......@@ -1327,6 +1412,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.8_cu92_upload
requires:
- nightly_binary_linux_conda_py3.8_cu92
......@@ -1343,6 +1430,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.8_cu101_upload
requires:
- nightly_binary_linux_conda_py3.8_cu101
......@@ -1359,6 +1448,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_linux_conda_py3.8_cu102_upload
requires:
- nightly_binary_linux_conda_py3.8_cu102
......@@ -1375,6 +1466,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_macos_conda_py3.6_cpu_upload
requires:
- nightly_binary_macos_conda_py3.6_cpu
......@@ -1391,6 +1484,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_macos_conda_py3.7_cpu_upload
requires:
- nightly_binary_macos_conda_py3.7_cpu
......@@ -1407,6 +1502,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_macos_conda_py3.8_cpu_upload
requires:
- nightly_binary_macos_conda_py3.8_cpu
......@@ -1422,6 +1519,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.6_cpu_upload
requires:
- nightly_binary_win_conda_py3.6_cpu
......@@ -1437,6 +1536,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.6_cu92_upload
requires:
- nightly_binary_win_conda_py3.6_cu92
......@@ -1452,6 +1553,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.6_cu101_upload
requires:
- nightly_binary_win_conda_py3.6_cu101
......@@ -1467,6 +1570,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.6_cu102_upload
requires:
- nightly_binary_win_conda_py3.6_cu102
......@@ -1482,6 +1587,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.7_cpu_upload
requires:
- nightly_binary_win_conda_py3.7_cpu
......@@ -1497,6 +1604,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.7_cu92_upload
requires:
- nightly_binary_win_conda_py3.7_cu92
......@@ -1512,6 +1621,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.7_cu101_upload
requires:
- nightly_binary_win_conda_py3.7_cu101
......@@ -1527,6 +1638,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.7_cu102_upload
requires:
- nightly_binary_win_conda_py3.7_cu102
......@@ -1542,6 +1655,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.8_cpu_upload
requires:
- nightly_binary_win_conda_py3.8_cpu
......@@ -1557,6 +1672,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.8_cu92_upload
requires:
- nightly_binary_win_conda_py3.8_cu92
......@@ -1572,6 +1689,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.8_cu101_upload
requires:
- nightly_binary_win_conda_py3.8_cu101
......@@ -1587,6 +1706,8 @@ workflows:
filters:
branches:
only: nightly
tags:
only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/
name: nightly_binary_win_conda_py3.8_cu102_upload
requires:
- nightly_binary_win_conda_py3.8_cu102
\ No newline at end of file
.circleci/config.yml.in
View file @ 40ad5572
......@@ -30,6 +30,18 @@ commands:
# git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
# git checkout "merged/$CIRCLE_BRANCH"
# fi
designate_upload_channel:
description: "inserts the correct upload channel into ${BASH_ENV}"
steps:
- run:
name: adding UPLOAD_CHANNEL to BASH_ENV
command: |
our_upload_channel=nightly
# On tags upload to test instead
if [[ -n "${CIRCLE_TAG}" ]]; then
our_upload_channel=test
fi
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
binary_common: &binary_common
parameters:
......@@ -59,7 +71,6 @@ binary_common: &binary_common
default: "pytorch/manylinux-cuda101"
environment:
PYTHON_VERSION: << parameters.python_version >>
BUILD_VERSION: << parameters.build_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: << parameters.cu_version >>
......@@ -358,12 +369,13 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- run:
command: |
# Prevent credential from leaking
conda install -yq anaconda-client
set -x
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force
# Requires org-member context
binary_wheel_upload:
......@@ -376,6 +388,7 @@ jobs:
steps:
- attach_workspace:
at: ~/workspace
- designate_upload_channel
- checkout
- run:
command: |
......@@ -387,7 +400,7 @@ jobs:
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/<< parameters.subfolder >>" --acl public-read
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
......
.circleci/regenerate.py
View file @ 40ad5572
......@@ -103,7 +103,16 @@ def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *,
d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"
if filter_branch is not None:
d["filters"] = {"branches": {"only": filter_branch}}
d["filters"] = {
"branches": {
"only": filter_branch
},
"tags": {
# Using a raw string here to avoid having to escape
# anything
"only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
}
}
return {f"binary_{btype}_upload": d}
......
packaging/pkg_helpers.bash
View file @ 40ad5572
......@@ -117,6 +117,12 @@ setup_build_version() {
else
export BUILD_VERSION="$BUILD_VERSION$VERSION_SUFFIX"
fi
# Set build version based on tag if on tag
if [[ -n "${CIRCLE_TAG}" ]]; then
# Strip tag
export BUILD_VERSION="$(echo "${CIRCLE_TAG}" | sed -e 's/^v//' -e 's/-.*$//')"
fi
}
# Set some useful variables for OS X, if applicable
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment