Upload wheels to respective folders (#1751)

Cherry-pick of
https://github.com/pytorch/vision/commit/d2460a75de237cfef8e5c3415f7bb0ad8467c0e5
into this repo

Fixes https://github.com/pytorch/audio/issues/1750

.circleci/config.yml

@@ -293,6 +293,10 @@ jobs:
 
   # Requires org-member context
   binary_wheel_upload:
+    parameters:
+      subfolder:
+        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
+        type: string
     docker:
       - image: circleci/python:3.8
     steps:
@@ -310,7 +314,7 @@ jobs:
             export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
             set -x
             for pkg in ~/workspace/*.whl; do
-              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read
+              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
             done
 
   smoke_test_linux_conda:
@@ -1243,6 +1247,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.6_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_cpu
+          subfolder: cpu/
       - smoke_test_linux_pip:
           cuda_version: cpu
           filters:
@@ -1279,6 +1284,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.6_cu102_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_cu102
+          subfolder: cu102/
       - smoke_test_linux_pip:
           cuda_version: cu102
           filters:
@@ -1315,6 +1321,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.6_cu111_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_cu111
+          subfolder: cu111/
       - smoke_test_linux_pip:
           cuda_version: cu111
           filters:
@@ -1351,6 +1358,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.6_cu113_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_cu113
+          subfolder: cu113/
       - smoke_test_linux_pip:
           cuda_version: cu113
           filters:
@@ -1387,6 +1395,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.6_rocm4.1_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_rocm4.1
+          subfolder: rocm4.1/
       - smoke_test_linux_pip:
           cuda_version: rocm4.1
           filters:
@@ -1422,6 +1431,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.7_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_cpu
+          subfolder: cpu/
       - smoke_test_linux_pip:
           cuda_version: cpu
           filters:
@@ -1458,6 +1468,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.7_cu102_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_cu102
+          subfolder: cu102/
       - smoke_test_linux_pip:
           cuda_version: cu102
           filters:
@@ -1494,6 +1505,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.7_cu111_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_cu111
+          subfolder: cu111/
       - smoke_test_linux_pip:
           cuda_version: cu111
           filters:
@@ -1530,6 +1542,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.7_cu113_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_cu113
+          subfolder: cu113/
       - smoke_test_linux_pip:
           cuda_version: cu113
           filters:
@@ -1566,6 +1579,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.7_rocm4.1_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_rocm4.1
+          subfolder: rocm4.1/
       - smoke_test_linux_pip:
           cuda_version: rocm4.1
           filters:
@@ -1601,6 +1615,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.8_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py3.8_cpu
+          subfolder: cpu/
       - smoke_test_linux_pip:
           cuda_version: cpu
           filters:
@@ -1637,6 +1652,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.8_cu102_upload
           requires:
           - nightly_binary_linux_wheel_py3.8_cu102
+          subfolder: cu102/
       - smoke_test_linux_pip:
           cuda_version: cu102
           filters:
@@ -1673,6 +1689,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.8_cu111_upload
           requires:
           - nightly_binary_linux_wheel_py3.8_cu111
+          subfolder: cu111/
       - smoke_test_linux_pip:
           cuda_version: cu111
           filters:
@@ -1709,6 +1726,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.8_cu113_upload
           requires:
           - nightly_binary_linux_wheel_py3.8_cu113
+          subfolder: cu113/
       - smoke_test_linux_pip:
           cuda_version: cu113
           filters:
@@ -1745,6 +1763,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.8_rocm4.1_upload
           requires:
           - nightly_binary_linux_wheel_py3.8_rocm4.1
+          subfolder: rocm4.1/
       - smoke_test_linux_pip:
           cuda_version: rocm4.1
           filters:
@@ -1780,6 +1799,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.9_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py3.9_cpu
+          subfolder: cpu/
       - smoke_test_linux_pip:
           cuda_version: cpu
           filters:
@@ -1816,6 +1836,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.9_cu102_upload
           requires:
           - nightly_binary_linux_wheel_py3.9_cu102
+          subfolder: cu102/
       - smoke_test_linux_pip:
           cuda_version: cu102
           filters:
@@ -1852,6 +1873,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.9_cu111_upload
           requires:
           - nightly_binary_linux_wheel_py3.9_cu111
+          subfolder: cu111/
       - smoke_test_linux_pip:
           cuda_version: cu111
           filters:
@@ -1888,6 +1910,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.9_cu113_upload
           requires:
           - nightly_binary_linux_wheel_py3.9_cu113
+          subfolder: cu113/
       - smoke_test_linux_pip:
           cuda_version: cu113
           filters:
@@ -1924,6 +1947,7 @@ workflows:
           name: nightly_binary_linux_wheel_py3.9_rocm4.1_upload
           requires:
           - nightly_binary_linux_wheel_py3.9_rocm4.1
+          subfolder: rocm4.1/
       - smoke_test_linux_pip:
           cuda_version: rocm4.1
           filters:
@@ -1959,6 +1983,7 @@ workflows:
           name: nightly_binary_macos_wheel_py3.6_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py3.6_cpu
+          subfolder: ''
       - binary_macos_wheel:
           cuda_version: cpu
           filters:
@@ -1982,6 +2007,7 @@ workflows:
           name: nightly_binary_macos_wheel_py3.7_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py3.7_cpu
+          subfolder: ''
       - binary_macos_wheel:
           cuda_version: cpu
           filters:
@@ -2005,6 +2031,7 @@ workflows:
           name: nightly_binary_macos_wheel_py3.8_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py3.8_cpu
+          subfolder: ''
       - binary_macos_wheel:
           cuda_version: cpu
           filters:
@@ -2028,6 +2055,7 @@ workflows:
           name: nightly_binary_macos_wheel_py3.9_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py3.9_cpu
+          subfolder: ''
       - binary_windows_wheel:
           cuda_version: cpu
           filters:
@@ -2049,6 +2077,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.6_cpu_upload
           requires:
           - nightly_binary_windows_wheel_py3.6_cpu
+          subfolder: cpu/
       - smoke_test_windows_pip:
           cuda_version: cpu
           filters:
@@ -2083,6 +2112,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.6_cu102_upload
           requires:
           - nightly_binary_windows_wheel_py3.6_cu102
+          subfolder: cu102/
       - smoke_test_windows_pip:
           cuda_version: cu102
           filters:
@@ -2117,6 +2147,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.6_cu111_upload
           requires:
           - nightly_binary_windows_wheel_py3.6_cu111
+          subfolder: cu111/
       - smoke_test_windows_pip:
           cuda_version: cu111
           filters:
@@ -2150,6 +2181,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.7_cpu_upload
           requires:
           - nightly_binary_windows_wheel_py3.7_cpu
+          subfolder: cpu/
       - smoke_test_windows_pip:
           cuda_version: cpu
           filters:
@@ -2184,6 +2216,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.7_cu102_upload
           requires:
           - nightly_binary_windows_wheel_py3.7_cu102
+          subfolder: cu102/
       - smoke_test_windows_pip:
           cuda_version: cu102
           filters:
@@ -2218,6 +2251,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.7_cu111_upload
           requires:
           - nightly_binary_windows_wheel_py3.7_cu111
+          subfolder: cu111/
       - smoke_test_windows_pip:
           cuda_version: cu111
           filters:
@@ -2251,6 +2285,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.8_cpu_upload
           requires:
           - nightly_binary_windows_wheel_py3.8_cpu
+          subfolder: cpu/
       - smoke_test_windows_pip:
           cuda_version: cpu
           filters:
@@ -2285,6 +2320,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.8_cu102_upload
           requires:
           - nightly_binary_windows_wheel_py3.8_cu102
+          subfolder: cu102/
       - smoke_test_windows_pip:
           cuda_version: cu102
           filters:
@@ -2319,6 +2355,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.8_cu111_upload
           requires:
           - nightly_binary_windows_wheel_py3.8_cu111
+          subfolder: cu111/
       - smoke_test_windows_pip:
           cuda_version: cu111
           filters:
@@ -2352,6 +2389,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.9_cpu_upload
           requires:
           - nightly_binary_windows_wheel_py3.9_cpu
+          subfolder: cpu/
       - smoke_test_windows_pip:
           cuda_version: cpu
           filters:
@@ -2386,6 +2424,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.9_cu102_upload
           requires:
           - nightly_binary_windows_wheel_py3.9_cu102
+          subfolder: cu102/
       - smoke_test_windows_pip:
           cuda_version: cu102
           filters:
@@ -2420,6 +2459,7 @@ workflows:
           name: nightly_binary_windows_wheel_py3.9_cu111_upload
           requires:
           - nightly_binary_windows_wheel_py3.9_cu111
+          subfolder: cu111/
       - smoke_test_windows_pip:
           cuda_version: cu111
           filters:

.circleci/config.yml.in

@@ -293,6 +293,10 @@ jobs:
 
   # Requires org-member context
   binary_wheel_upload:
+    parameters:
+      subfolder:
+        description: "What whl subfolder to upload to, e.g., blank or cu100/ (trailing slash is important)"
+        type: string
     docker:
       - image: circleci/python:3.8
     steps:
@@ -310,7 +314,7 @@ jobs:
             export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
             set -x
             for pkg in ~/workspace/*.whl; do
-              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read
+              aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read
             done
 
   smoke_test_linux_conda:

.circleci/regenerate.py

@@ -75,7 +75,7 @@ def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branc
 
     if upload:
 
-        w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype))
+        w.append(generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version))
 
         if filter_branch == 'nightly' and os_type != 'macos':
             pydistro = 'pip' if btype == 'wheel' else 'conda'
@@ -157,13 +157,17 @@ def gen_filter_branch_tree(*branches):
     }
 
 
-def generate_upload_workflow(base_workflow_name, filter_branch, btype):
+def generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version):
     d = {
         "name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name),
         "context": "org-member",
         "requires": [base_workflow_name],
     }
 
+    if btype == 'wheel':
+        d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"
+
+
     if filter_branch:
         d["filters"] = gen_filter_branch_tree(filter_branch)