regenerate.py 8.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16
17

import jinja2
18
import yaml
19
20
import os.path

21

22
PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
23

24
25
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

26
27

def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
28
29
    w = []
    for btype in ["wheel", "conda"]:
30
        for os_type in ["linux", "macos", "win"]:
31
            python_versions = PYTHON_VERSIONS
32
33
            cu_versions_dict = {"linux": ["cpu", "cu101", "cu102", "cu111"],
                                "win": ["cpu", "cu101", "cu102", "cu111"],
34
35
                                "macos": ["cpu"]}
            cu_versions = cu_versions_dict[os_type]
36
37
            for python_version in python_versions:
                for cu_version in cu_versions:
38
                    for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
39
40
41
42
43
                        fb = filter_branch
                        if windows_latest_only and os_type == "win" and filter_branch is None and \
                            (python_version != python_versions[-1] or
                             (cu_version not in [cu_versions[0], cu_versions[-1]])):
                            fb = "master"
44
45
                        w += workflow_pair(
                            btype, os_type, python_version, cu_version,
46
                            unicode, prefix, upload, filter_branch=fb)
47

48
49
50
51
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
        w += build_doc_job(None)
        w += upload_doc_job('nightly')
52
53
54
    return indent(indentation, w)


55
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False, *, filter_branch=None):
56
57
58
59
60

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

61
62
63
    w.append(generate_base_workflow(
        base_workflow_name, python_version, cu_version,
        unicode, os_type, btype, filter_branch=filter_branch))
64
65

    if upload:
66
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
guyang3532's avatar
guyang3532 committed
67
68
69
        if filter_branch == 'nightly' and os_type in ['linux', 'win']:
            pydistro = 'pip' if btype == 'wheel' else 'conda'
            w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
70
71
72
73

    return w


74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
        "requires": ["binary_linux_wheel_py3.7_cpu", ],
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
        "requires": ["build_docs", ],
    }

    if filter_branch:
95
96
        job["filters"] = gen_filter_branch_tree(filter_branch,
                                                tags_list=RC_PATTERN)
97
98
99
    return [{"upload_docs": job}]


100
101
102
103
manylinux_images = {
    "cu92": "pytorch/manylinux-cuda92",
    "cu101": "pytorch/manylinux-cuda101",
    "cu102": "pytorch/manylinux-cuda102",
peterjc123's avatar
peterjc123 committed
104
    "cu110": "pytorch/manylinux-cuda110",
105
    "cu111": "pytorch/manylinux-cuda111",
106
    "cu112": "pytorch/manylinux-cuda112",
107
108
109
110
111
112
113
114
115
116
}


def get_manylinux_image(cu_version):
    cu_suffix = "102"
    if cu_version.startswith('cu'):
        cu_suffix = cu_version[len('cu'):]
    return f"pytorch/manylinux-cuda{cu_suffix}"


117
118
119
120
121
122
123
124
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
    if cu_version.startswith('cu'):
        cu_suffix = cu_version[len('cu'):]
    return f"pytorch/conda-builder:cuda{cu_suffix}"


125
126
def generate_base_workflow(base_workflow_name, python_version, cu_version,
                           unicode, os_type, btype, *, filter_branch=None):
127
128
129
130

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
131
        "cu_version": cu_version,
132
133
    }

134
    if os_type != "win" and unicode:
135
136
        d["unicode_abi"] = '1'

137
138
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
139
        d["conda_docker_image"] = get_conda_image(cu_version)
140

141
    if filter_branch is not None:
142
143
144
145
146
147
148
149
150
151
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
152

153
    w = f"binary_{os_type}_{btype}"
154
    return {w: d}
155
156


157
158
159
160
161
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
162
163


164
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
165
166
167
168
169
170
171
172
173
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

    if btype == 'wheel':
        d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"

174
    if filter_branch is not None:
175
176
177
178
179
180
181
182
183
184
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
185

186
187
188
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

    return {"smoke_test_{os_type}_{pydistro}".format(os_type=os_type, pydistro=pydistro): d}


207
def indent(indentation, data_list):
208
209
    return ("\n" + " " * indentation).join(
        yaml.dump(data_list, default_flow_style=False).splitlines())
210
211


212
213
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
214
    for os_type in ["linux", "windows", "macos"]:
215
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
216
217
            if os_type == "macos" and device_type == "gpu":
                continue
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

                if device_type == 'gpu':
                    if python_version != "3.8":
                        job['filters'] = gen_filter_branch_tree('master', 'nightly')
                    job['cu_version'] = 'cu101'
                else:
                    job['cu_version'] = 'cpu'

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


236
237
238
239
def cmake_workflows(indentation=6):
    jobs = []
    python_version = '3.8'
    for os_type in ['linux', 'windows', 'macos']:
240
241
        # Skip OSX CUDA
        device_types = ['cpu', 'gpu'] if os_type != 'macos' else ['cpu']
242
243
244
245
246
247
248
        for device in device_types:
            job = {
                'name': f'cmake_{os_type}_{device}',
                'python_version': python_version
            }

            job['cu_version'] = 'cu101' if device == 'gpu' else 'cpu'
249
            if device == 'gpu' and os_type == 'linux':
250
251
252
253
254
                job['wheel_docker_image'] = 'pytorch/manylinux-cuda101'
            jobs.append({f'cmake_{os_type}_{device}': job})
    return indent(indentation, jobs)


255
256
257
258
259
260
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
        autoescape=False,
261
        keep_trailing_newline=True,
262
263
264
    )

    with open(os.path.join(d, 'config.yml'), 'w') as f:
265
266
267
        f.write(env.get_template('config.yml.in').render(
            build_workflows=build_workflows,
            unittest_workflows=unittest_workflows,
268
            cmake_workflows=cmake_workflows,
269
        ))