Switch to push-based nightlies. (#1441)

* Switch to push-based nightlies.

See https://github.com/pytorch/pytorch/issues/26817

Signed-off-by: Edward Z. Yang <ezyang@fb.com>

* disable merge
Signed-off-by: Edward Z. Yang <ezyang@fb.com>

* fixup
Signed-off-by: Edward Z. Yang <ezyang@fb.com>

.circleci/config.yml

@@ -14,15 +14,15 @@ commands:
     description: "checkout merge branch"
     steps:
       - checkout
-      - run:
-          name: Checkout merge branch
-          command: |
-            set -ex
-            BRANCH=$(git rev-parse --abbrev-ref HEAD)
-            if [[ "$BRANCH" != "master" ]]; then
-              git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
-              git checkout "merged/$CIRCLE_BRANCH"
-            fi
+#     - run:
+#         name: Checkout merge branch
+#         command: |
+#           set -ex
+#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
+#           if [[ "$BRANCH" != "master" ]]; then
+#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
+#             git checkout "merged/$CIRCLE_BRANCH"
+#           fi
 
 binary_common: &binary_common
   parameters:
@@ -440,369 +440,578 @@ workflows:
           cu_version: "cpu"
 
   nightly:
-    triggers:
-      - schedule:
-          cron: "0 9 * * *"
-          filters:
-            branches:
-              only:
-                - master
     jobs:
       - circleci_consistency
       - binary_linux_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7_cpu
           python_version: '2.7'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py2.7_cpu
           subfolder: cpu/
       - binary_linux_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7u_cpu
           python_version: '2.7'
           unicode_abi: '1'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7u_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py2.7u_cpu
           subfolder: cpu/
       - binary_linux_wheel:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7_cu92
           python_version: '2.7'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7_cu92_upload
           requires:
           - nightly_binary_linux_wheel_py2.7_cu92
           subfolder: cu92/
       - binary_linux_wheel:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7u_cu92
           python_version: '2.7'
           unicode_abi: '1'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7u_cu92_upload
           requires:
           - nightly_binary_linux_wheel_py2.7u_cu92
           subfolder: cu92/
       - binary_linux_wheel:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7_cu100
           python_version: '2.7'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7_cu100_upload
           requires:
           - nightly_binary_linux_wheel_py2.7_cu100
           subfolder: cu100/
       - binary_linux_wheel:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7u_cu100
           python_version: '2.7'
           unicode_abi: '1'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py2.7u_cu100_upload
           requires:
           - nightly_binary_linux_wheel_py2.7u_cu100
           subfolder: cu100/
       - binary_linux_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.5_cpu
           python_version: '3.5'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.5_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py3.5_cpu
           subfolder: cpu/
       - binary_linux_wheel:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.5_cu92
           python_version: '3.5'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.5_cu92_upload
           requires:
           - nightly_binary_linux_wheel_py3.5_cu92
           subfolder: cu92/
       - binary_linux_wheel:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.5_cu100
           python_version: '3.5'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.5_cu100_upload
           requires:
           - nightly_binary_linux_wheel_py3.5_cu100
           subfolder: cu100/
       - binary_linux_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.6_cpu
           python_version: '3.6'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.6_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_cpu
           subfolder: cpu/
       - binary_linux_wheel:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.6_cu92
           python_version: '3.6'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.6_cu92_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_cu92
           subfolder: cu92/
       - binary_linux_wheel:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.6_cu100
           python_version: '3.6'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.6_cu100_upload
           requires:
           - nightly_binary_linux_wheel_py3.6_cu100
           subfolder: cu100/
       - binary_linux_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.7_cpu
           python_version: '3.7'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.7_cpu_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_cpu
           subfolder: cpu/
       - binary_linux_wheel:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.7_cu92
           python_version: '3.7'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.7_cu92_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_cu92
           subfolder: cu92/
       - binary_linux_wheel:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.7_cu100
           python_version: '3.7'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_wheel_py3.7_cu100_upload
           requires:
           - nightly_binary_linux_wheel_py3.7_cu100
           subfolder: cu100/
       - binary_macos_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py2.7_cpu
           python_version: '2.7'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py2.7_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py2.7_cpu
           subfolder: ''
       - binary_macos_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py2.7u_cpu
           python_version: '2.7'
           unicode_abi: '1'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py2.7u_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py2.7u_cpu
           subfolder: ''
       - binary_macos_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py3.5_cpu
           python_version: '3.5'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py3.5_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py3.5_cpu
           subfolder: ''
       - binary_macos_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py3.6_cpu
           python_version: '3.6'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py3.6_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py3.6_cpu
           subfolder: ''
       - binary_macos_wheel:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py3.7_cpu
           python_version: '3.7'
       - binary_wheel_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_wheel_py3.7_cpu_upload
           requires:
           - nightly_binary_macos_wheel_py3.7_cpu
           subfolder: ''
       - binary_linux_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py2.7_cpu
           python_version: '2.7'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py2.7_cpu_upload
           requires:
           - nightly_binary_linux_conda_py2.7_cpu
       - binary_linux_conda:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py2.7_cu92
           python_version: '2.7'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py2.7_cu92_upload
           requires:
           - nightly_binary_linux_conda_py2.7_cu92
       - binary_linux_conda:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py2.7_cu100
           python_version: '2.7'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py2.7_cu100_upload
           requires:
           - nightly_binary_linux_conda_py2.7_cu100
       - binary_linux_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.5_cpu
           python_version: '3.5'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.5_cpu_upload
           requires:
           - nightly_binary_linux_conda_py3.5_cpu
       - binary_linux_conda:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.5_cu92
           python_version: '3.5'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.5_cu92_upload
           requires:
           - nightly_binary_linux_conda_py3.5_cu92
       - binary_linux_conda:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.5_cu100
           python_version: '3.5'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.5_cu100_upload
           requires:
           - nightly_binary_linux_conda_py3.5_cu100
       - binary_linux_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.6_cpu
           python_version: '3.6'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.6_cpu_upload
           requires:
           - nightly_binary_linux_conda_py3.6_cpu
       - binary_linux_conda:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.6_cu92
           python_version: '3.6'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.6_cu92_upload
           requires:
           - nightly_binary_linux_conda_py3.6_cu92
       - binary_linux_conda:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.6_cu100
           python_version: '3.6'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.6_cu100_upload
           requires:
           - nightly_binary_linux_conda_py3.6_cu100
       - binary_linux_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.7_cpu
           python_version: '3.7'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.7_cpu_upload
           requires:
           - nightly_binary_linux_conda_py3.7_cpu
       - binary_linux_conda:
           cu_version: cu92
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.7_cu92
           python_version: '3.7'
           wheel_docker_image: soumith/manylinux-cuda92
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.7_cu92_upload
           requires:
           - nightly_binary_linux_conda_py3.7_cu92
       - binary_linux_conda:
           cu_version: cu100
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.7_cu100
           python_version: '3.7'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_linux_conda_py3.7_cu100_upload
           requires:
           - nightly_binary_linux_conda_py3.7_cu100
       - binary_macos_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py2.7_cpu
           python_version: '2.7'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py2.7_cpu_upload
           requires:
           - nightly_binary_macos_conda_py2.7_cpu
       - binary_macos_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py3.5_cpu
           python_version: '3.5'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py3.5_cpu_upload
           requires:
           - nightly_binary_macos_conda_py3.5_cpu
       - binary_macos_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py3.6_cpu
           python_version: '3.6'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py3.6_cpu_upload
           requires:
           - nightly_binary_macos_conda_py3.6_cpu
       - binary_macos_conda:
           cu_version: cpu
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py3.7_cpu
           python_version: '3.7'
       - binary_conda_upload:
           context: org-member
+          filters:
+            branches:
+              only: nightly
           name: nightly_binary_macos_conda_py3.7_cpu_upload
           requires:
           - nightly_binary_macos_conda_py3.7_cpu
\ No newline at end of file

.circleci/config.yml.in

@@ -14,15 +14,15 @@ commands:
     description: "checkout merge branch"
     steps:
       - checkout
-      - run:
-          name: Checkout merge branch
-          command: |
-            set -ex
-            BRANCH=$(git rev-parse --abbrev-ref HEAD)
-            if [[ "$BRANCH" != "master" ]]; then
-              git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
-              git checkout "merged/$CIRCLE_BRANCH"
-            fi
+#     - run:
+#         name: Checkout merge branch
+#         command: |
+#           set -ex
+#           BRANCH=$(git rev-parse --abbrev-ref HEAD)
+#           if [[ "$BRANCH" != "master" ]]; then
+#             git fetch --force origin ${CIRCLE_BRANCH}/merge:merged/${CIRCLE_BRANCH}
+#             git checkout "merged/$CIRCLE_BRANCH"
+#           fi
 
 binary_common: &binary_common
   parameters:
@@ -285,14 +285,7 @@ workflows:
           cu_version: "cpu"
 
   nightly:
-    triggers:
-      - schedule:
-          cron: "0 9 * * *"
-          filters:
-            branches:
-              only:
-                - master
 {%- endif %}
     jobs:
       - circleci_consistency
-      {{ workflows(prefix="nightly_", upload=True) }}
+      {{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}

.circleci/regenerate.py

@@ -19,33 +19,38 @@ import yaml
 import os.path
 
 
-def workflows(prefix='', upload=False, indentation=6):
+def workflows(prefix='', filter_branch=None, upload=False, indentation=6):
     w = []
     for btype in ["wheel", "conda"]:
         for os_type in ["linux", "macos"]:
             for python_version in ["2.7", "3.5", "3.6", "3.7"]:
                 for cu_version in (["cpu", "cu92", "cu100"] if os_type == "linux" else ["cpu"]):
                     for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
-                        w += workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix, upload)
+                        w += workflow_pair(
+                            btype, os_type, python_version, cu_version,
+                            unicode, prefix, upload, filter_branch=filter_branch)
 
     return indent(indentation, w)
 
 
-def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False):
+def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False, *, filter_branch=None):
 
     w = []
     unicode_suffix = "u" if unicode else ""
     base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"
 
-    w.append(generate_base_workflow(base_workflow_name, python_version, cu_version, unicode, os_type, btype))
+    w.append(generate_base_workflow(
+        base_workflow_name, python_version, cu_version,
+        unicode, os_type, btype, filter_branch=filter_branch))
 
     if upload:
-        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version))
+        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
 
     return w
 
 
-def generate_base_workflow(base_workflow_name, python_version, cu_version, unicode, os_type, btype):
+def generate_base_workflow(base_workflow_name, python_version, cu_version,
+                           unicode, os_type, btype, *, filter_branch=None):
 
     d = {
         "name": base_workflow_name,
@@ -59,10 +64,13 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version, unico
     if cu_version == "cu92":
         d["wheel_docker_image"] = "soumith/manylinux-cuda92"
 
+    if filter_branch is not None:
+        d["filters"] = {"branches": {"only": filter_branch}}
+
     return {f"binary_{os_type}_{btype}": d}
 
 
-def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version):
+def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
     d = {
         "name": f"{base_workflow_name}_upload",
         "context": "org-member",
@@ -72,6 +80,9 @@ def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version):
     if btype == 'wheel':
         d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"
 
+    if filter_branch is not None:
+        d["filters"] = {"branches": {"only": filter_branch}}
+
     return {f"binary_{btype}_upload": d}