regenerate.py 6.24 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16
17

import jinja2
18
import yaml
19
20
import os.path

21

moto's avatar
moto committed
22
23
PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]

24
25
DOC_VERSION = ('linux', '3.8')

moto's avatar
moto committed
26
27

def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
28
    w = []
29
    w += build_download_job(filter_branch)
30
    for btype in ["wheel", "conda"]:
peterjc123's avatar
peterjc123 committed
31
        for os_type in ["linux", "macos", "windows"]:
moto's avatar
moto committed
32
33
            for python_version in PYTHON_VERSIONS:
                w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload)
34

35
36
37
38
39
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
        w += build_doc_job(None)
        w += upload_doc_job('nightly')

40
41
42
    return indent(indentation, w)


43
44
45
46
47
48
49
50
51
52
def build_download_job(filter_branch):
    job = {
        "name": "download_third_parties_nix",
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"download_third_parties_nix": job}]


moto's avatar
moto committed
53
def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False):
54
55

    w = []
56
    base_workflow_name = "{prefix}binary_{os_type}_{btype}_py{python_version}".format(
57
58
59
60
        prefix=prefix,
        os_type=os_type,
        btype=btype,
        python_version=python_version,
61
    )
62

63
    w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype))
64
65
66

    if upload:

67
        is_py3_linux = os_type in ['linux', "windows"] and not python_version.startswith("2.")
68

69
        w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype))
70
71
72

        if filter_branch == 'nightly' and is_py3_linux:
            pydistro = 'pip' if btype == 'wheel' else 'conda'
73
            w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
74
75
76

    return w

77
78
79
80
81
82
83
84
85
86
87
88
89
90
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.8",
        "requires": ["binary_linux_wheel_py3.8",],
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"build_docs": job}]

def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
91
        "context": "org-member",
92
93
94
95
96
97
98
99
        "python_version": "3.8",
        "requires": ["build_docs",],
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"upload_docs": job}]

100

101
def generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype):
102
103
104
105
106
107

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
    }

108
109
110
    if os_type in ['linux', 'macos']:
        d['requires'] = ['download_third_parties_nix']

111
112
113
    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

114
    return {f"binary_{os_type}_{btype}": d}
115
116


moto's avatar
moto committed
117
def gen_filter_branch_tree(*branches):
118
119
120
121
122
123
124
125
126
127
    return {
        "branches": {
            "only": list(branches),
        },
        "tags": {
            # Using a raw string here to avoid having to escape
            # anything
            "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
        }
    }
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142


def generate_upload_workflow(base_workflow_name, filter_branch, btype):
    d = {
        "name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name),
        "context": "org-member",
        "requires": [base_workflow_name],
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

    return {"binary_{btype}_upload".format(btype=btype): d}


143
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = "smoke_test_{pydistro}".format(pydistro=pydistro)
    d = {
        "name": "{base_workflow_name}_{smoke_suffix}".format(
            base_workflow_name=base_workflow_name, smoke_suffix=smoke_suffix),
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

159
    return {"smoke_test_{os_type}_{pydistro}".format(os_type=os_type, pydistro=pydistro): d}
160
161
162
163
164
165


def indent(indentation, data_list):
    return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())


moto's avatar
moto committed
166
def unittest_workflows(indentation=6):
moto's avatar
moto committed
167
    jobs = []
168
    jobs += build_download_job(None)
moto's avatar
moto committed
169
    for os_type in ["linux", "windows", "macos"]:
moto's avatar
moto committed
170
        for device_type in ["cpu", "gpu"]:
moto's avatar
moto committed
171
172
173
            if os_type == "macos" and device_type == "gpu":
                continue

174
            for i, python_version in enumerate(PYTHON_VERSIONS):
moto's avatar
moto committed
175
176
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
moto's avatar
moto committed
177
178
                    "python_version": python_version,
                }
moto's avatar
moto committed
179

180
181
182
                if os_type != "windows":
                    job['requires'] = ['download_third_parties_nix']

moto's avatar
moto committed
183
                jobs.append({f"unittest_{os_type}_{device_type}": job})
184
185
186
187
188
189
190
191

                if i == 0 and os_type == "linux" and device_type == "cpu":
                    jobs.append({
                        f"stylecheck": {
                            "name": f"stylecheck_py{python_version}",
                            "python_version": python_version,
                        }
                    })
moto's avatar
moto committed
192
    return indent(indentation, jobs)
moto's avatar
moto committed
193
194


195
196
197
198
199
200
201
202
203
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
        autoescape=False,
    )

    with open(os.path.join(d, 'config.yml'), 'w') as f:
moto's avatar
moto committed
204
205
206
207
        f.write(env.get_template('config.yml.in').render(
            build_workflows=build_workflows,
            unittest_workflows=unittest_workflows,
        ))
moto's avatar
moto committed
208
        f.write("\n")