"notebooks/train_sfno.ipynb" did not exist on "45b371b799fbbc571f0e50a53e0e82770960ac52"
regenerate.py 11.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16
17

import jinja2
18
from jinja2 import select_autoescape
19
import yaml
20
21
import os.path

22

23
PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
24

25
26
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

27
28

def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
29
30
    w = []
    for btype in ["wheel", "conda"]:
31
        for os_type in ["linux", "macos", "win"]:
32
            python_versions = PYTHON_VERSIONS
33
34
            cu_versions_dict = {"linux": ["cpu", "cu102", "cu111", "cu113", "rocm4.1", "rocm4.2"],
                                "win": ["cpu", "cu102", "cu111", "cu113"],
35
36
                                "macos": ["cpu"]}
            cu_versions = cu_versions_dict[os_type]
37
38
            for python_version in python_versions:
                for cu_version in cu_versions:
Jeff Daily's avatar
Jeff Daily committed
39
40
41
                    # ROCm conda packages not yet supported
                    if cu_version.startswith('rocm') and btype == "conda":
                        continue
Nikita Shulga's avatar
Nikita Shulga committed
42
                    for unicode in [False]:
43
44
45
46
                        fb = filter_branch
                        if windows_latest_only and os_type == "win" and filter_branch is None and \
                            (python_version != python_versions[-1] or
                             (cu_version not in [cu_versions[0], cu_versions[-1]])):
47
                            fb = "main"
48
49
50
51
52
                        if not fb and (os_type == 'linux' and
                                       cu_version == 'cpu' and
                                       btype == 'wheel' and
                                       python_version == '3.7'):
                            # the fields must match the build_docs "requires" dependency
53
                            fb = "/.*/"
54
55
                        w += workflow_pair(
                            btype, os_type, python_version, cu_version,
56
                            unicode, prefix, upload, filter_branch=fb)
57

58
59
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
60
        w += build_doc_job('/.*/')
61
        w += upload_doc_job('nightly')
62
63
64
    return indent(indentation, w)


65
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False, *, filter_branch=None):
66
67
68
69
70

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

71
72
73
    w.append(generate_base_workflow(
        base_workflow_name, python_version, cu_version,
        unicode, os_type, btype, filter_branch=filter_branch))
74
75

    if upload:
76
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
guyang3532's avatar
guyang3532 committed
77
78
79
        if filter_branch == 'nightly' and os_type in ['linux', 'win']:
            pydistro = 'pip' if btype == 'wheel' else 'conda'
            w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
80
81
82
83

    return w


84
85
86
87
88
89
90
91
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
        "requires": ["binary_linux_wheel_py3.7_cpu", ],
    }

    if filter_branch:
92
93
        job["filters"] = gen_filter_branch_tree(filter_branch,
                                                tags_list=RC_PATTERN)
94
95
96
97
98
99
100
101
102
103
104
105
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
        "requires": ["build_docs", ],
    }

    if filter_branch:
106
107
        job["filters"] = gen_filter_branch_tree(filter_branch,
                                                tags_list=RC_PATTERN)
108
109
110
    return [{"upload_docs": job}]


111
112
113
114
manylinux_images = {
    "cu92": "pytorch/manylinux-cuda92",
    "cu101": "pytorch/manylinux-cuda101",
    "cu102": "pytorch/manylinux-cuda102",
peterjc123's avatar
peterjc123 committed
115
    "cu110": "pytorch/manylinux-cuda110",
116
    "cu111": "pytorch/manylinux-cuda111",
117
    "cu112": "pytorch/manylinux-cuda112",
118
    "cu113": "pytorch/manylinux-cuda113",
119
120
121
122
}


def get_manylinux_image(cu_version):
Jeff Daily's avatar
Jeff Daily committed
123
124
125
    if cu_version == "cpu":
        return "pytorch/manylinux-cuda102"
    elif cu_version.startswith('cu'):
126
        cu_suffix = cu_version[len('cu'):]
Jeff Daily's avatar
Jeff Daily committed
127
128
129
130
        return f"pytorch/manylinux-cuda{cu_suffix}"
    elif cu_version.startswith('rocm'):
        rocm_suffix = cu_version[len('rocm'):]
        return f"pytorch/manylinux-rocm:{rocm_suffix}"
131
132


133
134
135
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
Jeff Daily's avatar
Jeff Daily committed
136
    elif cu_version.startswith('cu'):
137
        cu_suffix = cu_version[len('cu'):]
Jeff Daily's avatar
Jeff Daily committed
138
        return f"pytorch/conda-builder:cuda{cu_suffix}"
139
140


141
142
def generate_base_workflow(base_workflow_name, python_version, cu_version,
                           unicode, os_type, btype, *, filter_branch=None):
143
144
145
146

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
147
        "cu_version": cu_version,
148
149
    }

150
    if os_type != "win" and unicode:
151
152
        d["unicode_abi"] = '1'

153
154
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
Jeff Daily's avatar
Jeff Daily committed
155
156
157
        # ROCm conda packages not yet supported
        if "rocm" not in cu_version:
            d["conda_docker_image"] = get_conda_image(cu_version)
158

159
    if filter_branch is not None:
160
161
162
163
164
165
166
167
168
169
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
170

171
    w = f"binary_{os_type}_{btype}"
172
    return {w: d}
173
174


175
176
177
178
179
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
180
181


182
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
183
184
185
186
187
188
189
190
191
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

    if btype == 'wheel':
        d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"

192
    if filter_branch is not None:
193
194
195
196
197
198
199
200
201
202
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
203

204
205
206
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Nikita Shulga's avatar
Nikita Shulga committed
222
    return {f"smoke_test_{os_type}_{pydistro}": d}
guyang3532's avatar
guyang3532 committed
223
224


225
def indent(indentation, data_list):
226
227
    return ("\n" + " " * indentation).join(
        yaml.dump(data_list, default_flow_style=False).splitlines())
228
229


230
231
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
232
    for os_type in ["linux", "windows", "macos"]:
233
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
234
235
            if os_type == "macos" and device_type == "gpu":
                continue
236
237
238
239
240
241
242
243
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

                if device_type == 'gpu':
                    if python_version != "3.8":
244
                        job['filters'] = gen_filter_branch_tree('main', 'nightly')
245
                    job['cu_version'] = 'cu102'
246
247
248
249
250
251
252
253
                else:
                    job['cu_version'] = 'cpu'

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


254
255
256
257
def cmake_workflows(indentation=6):
    jobs = []
    python_version = '3.8'
    for os_type in ['linux', 'windows', 'macos']:
258
259
        # Skip OSX CUDA
        device_types = ['cpu', 'gpu'] if os_type != 'macos' else ['cpu']
260
261
262
263
264
265
        for device in device_types:
            job = {
                'name': f'cmake_{os_type}_{device}',
                'python_version': python_version
            }

266
            job['cu_version'] = 'cu102' if device == 'gpu' else 'cpu'
267
            if device == 'gpu' and os_type == 'linux':
268
                job['wheel_docker_image'] = 'pytorch/manylinux-cuda102'
269
270
271
272
            jobs.append({f'cmake_{os_type}_{device}': job})
    return indent(indentation, jobs)


273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
def ios_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""
    for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]:
        name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}'
        build_job_names.append(name)
        build_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}',
            'ios_arch': arch,
            'ios_platform': platform,
            'name': name,
        }
        if nightly:
            build_job['filters'] = gen_filter_branch_tree('nightly')
        jobs.append({'binary_ios_build': build_job})

    if nightly:
        upload_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload',
            'context': 'org-member',
            'filters': gen_filter_branch_tree('nightly'),
            'requires': build_job_names,
        }
        jobs.append({'binary_ios_upload': upload_job})
    return indent(indentation, jobs)


302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
def android_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""

    name = f'{name_prefix}binary_libtorchvision_ops_android'
    build_job_names.append(name)
    build_job = {
        'build_environment': f'{env_prefix}binary-libtorchvision_ops-android',
        'name': name,
    }

    if nightly:
        upload_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-android-upload',
            'context': 'org-member',
            'filters': gen_filter_branch_tree('nightly'),
            'name': f'{name_prefix}binary_libtorchvision_ops_android_upload'
        }
        jobs.append({'binary_android_upload': upload_job})
    else:
        jobs.append({'binary_android_build': build_job})
    return indent(indentation, jobs)


328
329
330
331
332
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
333
        autoescape=select_autoescape(enabled_extensions=('html', 'xml')),
334
        keep_trailing_newline=True,
335
336
337
    )

    with open(os.path.join(d, 'config.yml'), 'w') as f:
338
339
340
        f.write(env.get_template('config.yml.in').render(
            build_workflows=build_workflows,
            unittest_workflows=unittest_workflows,
341
            cmake_workflows=cmake_workflows,
342
            ios_workflows=ios_workflows,
343
            android_workflows=android_workflows,
344
        ))