use native python code generation logic (#1321)

use flake8 with Python 3 on .circleci directory

.circleci/.gitignore

+.idea

.circleci/config.yml.in

@@ -50,7 +50,7 @@ jobs:
       - checkout
       - run:
           command: |
-            pip install --user --progress-bar off jinja2
+            pip install --user --progress-bar off jinja2 pyyaml
             python .circleci/regenerate.py
             git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
 
@@ -253,47 +253,6 @@ jobs:
               aws s3 cp "$pkg" "s3://pytorch/whl/nightly/<< parameters.subfolder >>" --acl public-read
             done
 
-{%- macro workflow(btype, os, python_version, cu_version, unicode, prefix='', upload=False) %}
-      - binary_{{os}}_{{btype}}:
-          name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}
-          python_version: "{{python_version}}"
-          cu_version: "{{cu_version}}"
-{%-   if unicode %}
-          unicode_abi: "1"
-{%-   endif %}
-{%-   if cu_version == "cu92" %}
-          wheel_docker_image: "soumith/manylinux-cuda92"
-{%-   endif %}
-
-{%-   if upload %}
-      - binary_{{btype}}_upload:
-          name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}_upload
-          context: org-member
-          requires:
-            - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "u" if unicode }}_{{cu_version}}
-{%-     if btype == 'wheel' %}
-{%-       if os == 'macos' %}
-          subfolder: ""
-{%-       else %}
-          subfolder: "{{cu_version}}/"
-{%-       endif %}
-{%-     endif %}
-{%-   endif %}
-{%- endmacro %}
-
-{%- macro workflows(prefix='', upload=False) %}
-{%- for btype in ["wheel", "conda"] -%}
-{%-   for os in ["linux", "macos"] -%}
-{%-     for python_version in ["2.7", "3.5", "3.6", "3.7"] -%}
-{%-       for cu_version in (["cpu", "cu92", "cu100"] if os == "linux" else ["cpu"]) -%}
-{%-         for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]) -%}
-          {{ workflow(btype, os, python_version, cu_version, unicode, prefix=prefix, upload=upload) }}
-{%-         endfor -%}
-{%-     endfor -%}
-{%-     endfor -%}
-{%-   endfor -%}
-{%- endfor %}
-{%- endmacro %}
 
 workflows:
   build:

.circleci/regenerate.py

-#!/usr/bin/env python
+#!/usr/bin/env python3
+
+"""
+This script should use a very simple, functional programming style.
+Avoid Jinja macros in favor of native Python functions.
+
+Don't go overboard on code generation; use Python only to generate
+content that can't be easily declared statically using CircleCI's YAML API.
+
+Data declarations (e.g. the nested loops for defining the configuration matrix)
+should be at the top of the file for easy updating.
+
+See this comment for design rationale:
+https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
+"""
 
 import jinja2
+import yaml
 import os.path
 
-d = os.path.dirname(__file__)
-env = jinja2.Environment(
-    loader=jinja2.FileSystemLoader(d),
-    lstrip_blocks=True,
-    autoescape=False,
-)
-with open(os.path.join(d, 'config.yml'), 'w') as f:
-    f.write(env.get_template('config.yml.in').render())
+
+def workflows(prefix='', upload=False, indentation=6):
+    w = []
+    for btype in ["wheel", "conda"]:
+        for os_type in ["linux", "macos"]:
+            for python_version in ["2.7", "3.5", "3.6", "3.7"]:
+                for cu_version in (["cpu", "cu92", "cu100"] if os_type == "linux" else ["cpu"]):
+                    for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
+                        w += workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix, upload)
+
+    return indent(indentation, w)
+
+
+def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False):
+
+    w = []
+    unicode_suffix = "u" if unicode else ""
+    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"
+
+    w.append(generate_base_workflow(base_workflow_name, python_version, cu_version, unicode, os_type, btype))
+
+    if upload:
+        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version))
+
+    return w
+
+
+def generate_base_workflow(base_workflow_name, python_version, cu_version, unicode, os_type, btype):
+
+    d = {
+        "name": base_workflow_name,
+        "python_version": python_version,
+        "cu_version": cu_version,
+    }
+
+    if unicode:
+        d["unicode_abi"] = '1'
+
+    if cu_version == "cu92":
+        d["wheel_docker_image"] = "soumith/manylinux-cuda92"
+
+    return {f"binary_{os_type}_{btype}": d}
+
+
+def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version):
+    d = {
+        "name": f"{base_workflow_name}_upload",
+        "context": "org-member",
+        "requires": [base_workflow_name],
+    }
+
+    if btype == 'wheel':
+        d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"
+
+    return {f"binary_{btype}_upload": d}
+
+
+def indent(indentation, data_list):
+    return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())
+
+
+if __name__ == "__main__":
+    d = os.path.dirname(__file__)
+    env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(d),
+        lstrip_blocks=True,
+        autoescape=False,
+    )
+
+    with open(os.path.join(d, 'config.yml'), 'w') as f:
+        f.write(env.get_template('config.yml.in').render(workflows=workflows))

.travis.yml

@@ -18,7 +18,12 @@ matrix:
     - env: LINT_CHECK
       python: "2.7"
       install: pip install flake8
-      script: flake8
+      script: flake8 --exclude .circleci
+      after_success: []
+    - env: LINT_CHECK
+      python: "3.6"
+      install: pip install flake8
+      script: flake8 .circleci
       after_success: []
     - python: "2.7"
       env: IMAGE_BACKEND=Pillow-SIMD