regenerate.py 10.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16
17

import jinja2
18
import yaml
19
20
import os.path

21

22
PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
23

24
25
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

26
27

def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
28
29
    w = []
    for btype in ["wheel", "conda"]:
30
        for os_type in ["linux", "macos", "win"]:
31
            python_versions = PYTHON_VERSIONS
32
            cu_versions_dict = {"linux": ["cpu", "cu101", "cu102", "cu111", "rocm4.0.1", "rocm4.1"],
33
                                "win": ["cpu", "cu101", "cu102", "cu111"],
34
35
                                "macos": ["cpu"]}
            cu_versions = cu_versions_dict[os_type]
36
37
            for python_version in python_versions:
                for cu_version in cu_versions:
Jeff Daily's avatar
Jeff Daily committed
38
39
40
                    # ROCm conda packages not yet supported
                    if cu_version.startswith('rocm') and btype == "conda":
                        continue
41
                    for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
42
43
44
45
46
                        fb = filter_branch
                        if windows_latest_only and os_type == "win" and filter_branch is None and \
                            (python_version != python_versions[-1] or
                             (cu_version not in [cu_versions[0], cu_versions[-1]])):
                            fb = "master"
47
48
                        w += workflow_pair(
                            btype, os_type, python_version, cu_version,
49
                            unicode, prefix, upload, filter_branch=fb)
50

51
52
53
54
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
        w += build_doc_job(None)
        w += upload_doc_job('nightly')
55
56
57
    return indent(indentation, w)


58
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False, *, filter_branch=None):
59
60
61
62
63

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

64
65
66
    w.append(generate_base_workflow(
        base_workflow_name, python_version, cu_version,
        unicode, os_type, btype, filter_branch=filter_branch))
67
68

    if upload:
69
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
guyang3532's avatar
guyang3532 committed
70
71
72
        if filter_branch == 'nightly' and os_type in ['linux', 'win']:
            pydistro = 'pip' if btype == 'wheel' else 'conda'
            w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
73
74
75
76

    return w


77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
        "requires": ["binary_linux_wheel_py3.7_cpu", ],
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
        "requires": ["build_docs", ],
    }

    if filter_branch:
98
99
        job["filters"] = gen_filter_branch_tree(filter_branch,
                                                tags_list=RC_PATTERN)
100
101
102
    return [{"upload_docs": job}]


103
104
105
106
manylinux_images = {
    "cu92": "pytorch/manylinux-cuda92",
    "cu101": "pytorch/manylinux-cuda101",
    "cu102": "pytorch/manylinux-cuda102",
peterjc123's avatar
peterjc123 committed
107
    "cu110": "pytorch/manylinux-cuda110",
108
    "cu111": "pytorch/manylinux-cuda111",
109
    "cu112": "pytorch/manylinux-cuda112",
110
111
112
113
}


def get_manylinux_image(cu_version):
Jeff Daily's avatar
Jeff Daily committed
114
115
116
    if cu_version == "cpu":
        return "pytorch/manylinux-cuda102"
    elif cu_version.startswith('cu'):
117
        cu_suffix = cu_version[len('cu'):]
Jeff Daily's avatar
Jeff Daily committed
118
119
120
121
        return f"pytorch/manylinux-cuda{cu_suffix}"
    elif cu_version.startswith('rocm'):
        rocm_suffix = cu_version[len('rocm'):]
        return f"pytorch/manylinux-rocm:{rocm_suffix}"
122
123


124
125
126
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
Jeff Daily's avatar
Jeff Daily committed
127
    elif cu_version.startswith('cu'):
128
        cu_suffix = cu_version[len('cu'):]
Jeff Daily's avatar
Jeff Daily committed
129
        return f"pytorch/conda-builder:cuda{cu_suffix}"
130
131


132
133
def generate_base_workflow(base_workflow_name, python_version, cu_version,
                           unicode, os_type, btype, *, filter_branch=None):
134
135
136
137

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
138
        "cu_version": cu_version,
139
140
    }

141
    if os_type != "win" and unicode:
142
143
        d["unicode_abi"] = '1'

144
145
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
Jeff Daily's avatar
Jeff Daily committed
146
147
148
        # ROCm conda packages not yet supported
        if "rocm" not in cu_version:
            d["conda_docker_image"] = get_conda_image(cu_version)
149

150
    if filter_branch is not None:
151
152
153
154
155
156
157
158
159
160
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
161

162
    w = f"binary_{os_type}_{btype}"
163
    return {w: d}
164
165


166
167
168
169
170
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
171
172


173
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
174
175
176
177
178
179
180
181
182
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

    if btype == 'wheel':
        d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"

183
    if filter_branch is not None:
184
185
186
187
188
189
190
191
192
193
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
194

195
196
197
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

    return {"smoke_test_{os_type}_{pydistro}".format(os_type=os_type, pydistro=pydistro): d}


216
def indent(indentation, data_list):
217
218
    return ("\n" + " " * indentation).join(
        yaml.dump(data_list, default_flow_style=False).splitlines())
219
220


221
222
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
223
    for os_type in ["linux", "windows", "macos"]:
224
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
225
226
            if os_type == "macos" and device_type == "gpu":
                continue
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

                if device_type == 'gpu':
                    if python_version != "3.8":
                        job['filters'] = gen_filter_branch_tree('master', 'nightly')
                    job['cu_version'] = 'cu101'
                else:
                    job['cu_version'] = 'cpu'

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


245
246
247
248
def cmake_workflows(indentation=6):
    jobs = []
    python_version = '3.8'
    for os_type in ['linux', 'windows', 'macos']:
249
250
        # Skip OSX CUDA
        device_types = ['cpu', 'gpu'] if os_type != 'macos' else ['cpu']
251
252
253
254
255
256
257
        for device in device_types:
            job = {
                'name': f'cmake_{os_type}_{device}',
                'python_version': python_version
            }

            job['cu_version'] = 'cu101' if device == 'gpu' else 'cpu'
258
            if device == 'gpu' and os_type == 'linux':
259
260
261
262
263
                job['wheel_docker_image'] = 'pytorch/manylinux-cuda101'
            jobs.append({f'cmake_{os_type}_{device}': job})
    return indent(indentation, jobs)


264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
def ios_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""
    for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]:
        name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}'
        build_job_names.append(name)
        build_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}',
            'ios_arch': arch,
            'ios_platform': platform,
            'name': name,
        }
        if nightly:
            build_job['filters'] = gen_filter_branch_tree('nightly')
        jobs.append({'binary_ios_build': build_job})

    if nightly:
        upload_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload',
            'context': 'org-member',
            'filters': gen_filter_branch_tree('nightly'),
            'requires': build_job_names,
        }
        jobs.append({'binary_ios_upload': upload_job})
    return indent(indentation, jobs)


293
294
295
296
297
298
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
        autoescape=False,
299
        keep_trailing_newline=True,
300
301
302
    )

    with open(os.path.join(d, 'config.yml'), 'w') as f:
303
304
305
        f.write(env.get_template('config.yml.in').render(
            build_workflows=build_workflows,
            unittest_workflows=unittest_workflows,
306
            cmake_workflows=cmake_workflows,
307
            ios_workflows=ios_workflows,
308
        ))