Skip to content

GitLab

Unverified Commit 98435e59 authored by Nikita Shulga's avatar Nikita Shulga Committed by GitHub
Browse files

Upload wheels to respective folders (#1751) 

Cherry-pick of
https://github.com/pytorch/vision/commit/d2460a75de237cfef8e5c3415f7bb0ad8467c0e5
into this repo

Fixes https://github.com/pytorch/audio/issues/1750
parent a164477d
Hide whitespace changes
Inline Side-by-side
Showing with 52 additions and 4 deletions
.circleci/config.yml
View file @ 98435e59
......@@ -293,6 +293,10 @@ jobs:
# Requires org-member context
binary_wheel_upload:
parameters:
subfolder:
description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
type: string
docker:
- image: circleci/python:3.8
steps:
......@@ -310,7 +314,7 @@ jobs:
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
smoke_test_linux_conda:
......@@ -1243,6 +1247,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.6_cpu_upload
requires:
- nightly_binary_linux_wheel_py3.6_cpu
subfolder: cpu/
- smoke_test_linux_pip:
cuda_version: cpu
filters:
......@@ -1279,6 +1284,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.6_cu102_upload
requires:
- nightly_binary_linux_wheel_py3.6_cu102
subfolder: cu102/
- smoke_test_linux_pip:
cuda_version: cu102
filters:
......@@ -1315,6 +1321,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.6_cu111_upload
requires:
- nightly_binary_linux_wheel_py3.6_cu111
subfolder: cu111/
- smoke_test_linux_pip:
cuda_version: cu111
filters:
......@@ -1351,6 +1358,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.6_cu113_upload
requires:
- nightly_binary_linux_wheel_py3.6_cu113
subfolder: cu113/
- smoke_test_linux_pip:
cuda_version: cu113
filters:
......@@ -1387,6 +1395,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.6_rocm4.1_upload
requires:
- nightly_binary_linux_wheel_py3.6_rocm4.1
subfolder: rocm4.1/
- smoke_test_linux_pip:
cuda_version: rocm4.1
filters:
......@@ -1422,6 +1431,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.7_cpu_upload
requires:
- nightly_binary_linux_wheel_py3.7_cpu
subfolder: cpu/
- smoke_test_linux_pip:
cuda_version: cpu
filters:
......@@ -1458,6 +1468,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.7_cu102_upload
requires:
- nightly_binary_linux_wheel_py3.7_cu102
subfolder: cu102/
- smoke_test_linux_pip:
cuda_version: cu102
filters:
......@@ -1494,6 +1505,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.7_cu111_upload
requires:
- nightly_binary_linux_wheel_py3.7_cu111
subfolder: cu111/
- smoke_test_linux_pip:
cuda_version: cu111
filters:
......@@ -1530,6 +1542,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.7_cu113_upload
requires:
- nightly_binary_linux_wheel_py3.7_cu113
subfolder: cu113/
- smoke_test_linux_pip:
cuda_version: cu113
filters:
......@@ -1566,6 +1579,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.7_rocm4.1_upload
requires:
- nightly_binary_linux_wheel_py3.7_rocm4.1
subfolder: rocm4.1/
- smoke_test_linux_pip:
cuda_version: rocm4.1
filters:
......@@ -1601,6 +1615,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.8_cpu_upload
requires:
- nightly_binary_linux_wheel_py3.8_cpu
subfolder: cpu/
- smoke_test_linux_pip:
cuda_version: cpu
filters:
......@@ -1637,6 +1652,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.8_cu102_upload
requires:
- nightly_binary_linux_wheel_py3.8_cu102
subfolder: cu102/
- smoke_test_linux_pip:
cuda_version: cu102
filters:
......@@ -1673,6 +1689,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.8_cu111_upload
requires:
- nightly_binary_linux_wheel_py3.8_cu111
subfolder: cu111/
- smoke_test_linux_pip:
cuda_version: cu111
filters:
......@@ -1709,6 +1726,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.8_cu113_upload
requires:
- nightly_binary_linux_wheel_py3.8_cu113
subfolder: cu113/
- smoke_test_linux_pip:
cuda_version: cu113
filters:
......@@ -1745,6 +1763,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.8_rocm4.1_upload
requires:
- nightly_binary_linux_wheel_py3.8_rocm4.1
subfolder: rocm4.1/
- smoke_test_linux_pip:
cuda_version: rocm4.1
filters:
......@@ -1780,6 +1799,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.9_cpu_upload
requires:
- nightly_binary_linux_wheel_py3.9_cpu
subfolder: cpu/
- smoke_test_linux_pip:
cuda_version: cpu
filters:
......@@ -1816,6 +1836,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.9_cu102_upload
requires:
- nightly_binary_linux_wheel_py3.9_cu102
subfolder: cu102/
- smoke_test_linux_pip:
cuda_version: cu102
filters:
......@@ -1852,6 +1873,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.9_cu111_upload
requires:
- nightly_binary_linux_wheel_py3.9_cu111
subfolder: cu111/
- smoke_test_linux_pip:
cuda_version: cu111
filters:
......@@ -1888,6 +1910,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.9_cu113_upload
requires:
- nightly_binary_linux_wheel_py3.9_cu113
subfolder: cu113/
- smoke_test_linux_pip:
cuda_version: cu113
filters:
......@@ -1924,6 +1947,7 @@ workflows:
name: nightly_binary_linux_wheel_py3.9_rocm4.1_upload
requires:
- nightly_binary_linux_wheel_py3.9_rocm4.1
subfolder: rocm4.1/
- smoke_test_linux_pip:
cuda_version: rocm4.1
filters:
......@@ -1959,6 +1983,7 @@ workflows:
name: nightly_binary_macos_wheel_py3.6_cpu_upload
requires:
- nightly_binary_macos_wheel_py3.6_cpu
subfolder: ''
- binary_macos_wheel:
cuda_version: cpu
filters:
......@@ -1982,6 +2007,7 @@ workflows:
name: nightly_binary_macos_wheel_py3.7_cpu_upload
requires:
- nightly_binary_macos_wheel_py3.7_cpu
subfolder: ''
- binary_macos_wheel:
cuda_version: cpu
filters:
......@@ -2005,6 +2031,7 @@ workflows:
name: nightly_binary_macos_wheel_py3.8_cpu_upload
requires:
- nightly_binary_macos_wheel_py3.8_cpu
subfolder: ''
- binary_macos_wheel:
cuda_version: cpu
filters:
......@@ -2028,6 +2055,7 @@ workflows:
name: nightly_binary_macos_wheel_py3.9_cpu_upload
requires:
- nightly_binary_macos_wheel_py3.9_cpu
subfolder: ''
- binary_windows_wheel:
cuda_version: cpu
filters:
......@@ -2049,6 +2077,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.6_cpu_upload
requires:
- nightly_binary_windows_wheel_py3.6_cpu
subfolder: cpu/
- smoke_test_windows_pip:
cuda_version: cpu
filters:
......@@ -2083,6 +2112,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.6_cu102_upload
requires:
- nightly_binary_windows_wheel_py3.6_cu102
subfolder: cu102/
- smoke_test_windows_pip:
cuda_version: cu102
filters:
......@@ -2117,6 +2147,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.6_cu111_upload
requires:
- nightly_binary_windows_wheel_py3.6_cu111
subfolder: cu111/
- smoke_test_windows_pip:
cuda_version: cu111
filters:
......@@ -2150,6 +2181,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.7_cpu_upload
requires:
- nightly_binary_windows_wheel_py3.7_cpu
subfolder: cpu/
- smoke_test_windows_pip:
cuda_version: cpu
filters:
......@@ -2184,6 +2216,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.7_cu102_upload
requires:
- nightly_binary_windows_wheel_py3.7_cu102
subfolder: cu102/
- smoke_test_windows_pip:
cuda_version: cu102
filters:
......@@ -2218,6 +2251,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.7_cu111_upload
requires:
- nightly_binary_windows_wheel_py3.7_cu111
subfolder: cu111/
- smoke_test_windows_pip:
cuda_version: cu111
filters:
......@@ -2251,6 +2285,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.8_cpu_upload
requires:
- nightly_binary_windows_wheel_py3.8_cpu
subfolder: cpu/
- smoke_test_windows_pip:
cuda_version: cpu
filters:
......@@ -2285,6 +2320,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.8_cu102_upload
requires:
- nightly_binary_windows_wheel_py3.8_cu102
subfolder: cu102/
- smoke_test_windows_pip:
cuda_version: cu102
filters:
......@@ -2319,6 +2355,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.8_cu111_upload
requires:
- nightly_binary_windows_wheel_py3.8_cu111
subfolder: cu111/
- smoke_test_windows_pip:
cuda_version: cu111
filters:
......@@ -2352,6 +2389,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.9_cpu_upload
requires:
- nightly_binary_windows_wheel_py3.9_cpu
subfolder: cpu/
- smoke_test_windows_pip:
cuda_version: cpu
filters:
......@@ -2386,6 +2424,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.9_cu102_upload
requires:
- nightly_binary_windows_wheel_py3.9_cu102
subfolder: cu102/
- smoke_test_windows_pip:
cuda_version: cu102
filters:
......@@ -2420,6 +2459,7 @@ workflows:
name: nightly_binary_windows_wheel_py3.9_cu111_upload
requires:
- nightly_binary_windows_wheel_py3.9_cu111
subfolder: cu111/
- smoke_test_windows_pip:
cuda_version: cu111
filters:
......
.circleci/config.yml.in
View file @ 98435e59
......@@ -293,6 +293,10 @@ jobs:
# Requires org-member context
binary_wheel_upload:
parameters:
subfolder:
description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
type: string
docker:
- image: circleci/python:3.8
steps:
......@@ -310,7 +314,7 @@ jobs:
export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
set -x
for pkg in ~/workspace/*.whl; do
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read
aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
done
smoke_test_linux_conda:
......
.circleci/regenerate.py
View file @ 98435e59
......@@ -75,7 +75,7 @@ def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branc
if upload:
w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype))
w.append(generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version))
if filter_branch == 'nightly' and os_type != 'macos':
pydistro = 'pip' if btype == 'wheel' else 'conda'
......@@ -157,13 +157,17 @@ def gen_filter_branch_tree(*branches):
}
def generate_upload_workflow(base_workflow_name, filter_branch, btype):
def generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version):
d = {
"name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name),
"context": "org-member",
"requires": [base_workflow_name],
}
if btype == 'wheel':
d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"
if filter_branch:
d["filters"] = gen_filter_branch_tree(filter_branch)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment