regenerate.py 12.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16

17
18
import os.path

19
import jinja2
20
import yaml
21
from jinja2 import select_autoescape
22

23

24
PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
25

26
27
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

28

29
def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
30
31
    w = []
    for btype in ["wheel", "conda"]:
32
        for os_type in ["linux", "macos", "win"]:
33
            python_versions = PYTHON_VERSIONS
34
            cu_versions_dict = {
35
                "linux": ["cpu", "cu116", "cu117", "rocm5.2", "rocm5.3"],
Andrey Talman's avatar
Andrey Talman committed
36
                "win": ["cpu", "cu116", "cu117"],
37
38
                "macos": ["cpu"],
            }
39
            cu_versions = cu_versions_dict[os_type]
40
41
            for python_version in python_versions:
                for cu_version in cu_versions:
Jeff Daily's avatar
Jeff Daily committed
42
                    # ROCm conda packages not yet supported
43
                    if cu_version.startswith("rocm") and btype == "conda":
Jeff Daily's avatar
Jeff Daily committed
44
                        continue
Nikita Shulga's avatar
Nikita Shulga committed
45
                    for unicode in [False]:
46
                        fb = filter_branch
47
48
49
50
51
52
53
54
55
                        if (
                            windows_latest_only
                            and os_type == "win"
                            and filter_branch is None
                            and (
                                python_version != python_versions[-1]
                                or (cu_version not in [cu_versions[0], cu_versions[-1]])
                            )
                        ):
56
                            fb = "main"
57
58
59
                        if not fb and (
                            os_type == "linux" and cu_version == "cpu" and btype == "wheel" and python_version == "3.7"
                        ):
60
                            # the fields must match the build_docs "requires" dependency
61
                            fb = "/.*/"
62
63
64
65
66
67
68
69

                        # Disable all Linux Wheels Workflows from CircleCI
                        # since those will now be done through Nova. We'll keep
                        # around the py3.7 Linux Wheels build since the docs
                        # job depends on it.
                        if os_type == "linux" and btype == "wheel" and python_version != "3.7":
                            continue

70
71
72
73
                        # Disable all Macos Wheels Workflows from CircleCI.
                        if os_type == "macos" and btype == "wheel":
                            continue

74
75
76
77
                        # Disable all non-Windows Conda workflows
                        if os_type != "win" and btype == "conda":
                            continue

78
                        w += workflow_pair(
79
80
                            btype, os_type, python_version, cu_version, unicode, prefix, upload, filter_branch=fb
                        )
81

82
83
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
84
85
        w += build_doc_job("/.*/")
        w += upload_doc_job("nightly")
86
87
88
    return indent(indentation, w)


89
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix="", upload=False, *, filter_branch=None):
90
91
92
93
94

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

95
96
97
98
99
    w.append(
        generate_base_workflow(
            base_workflow_name, python_version, cu_version, unicode, os_type, btype, filter_branch=filter_branch
        )
    )
100

101
102
103
104
105
    # For the remaining py3.7 Linux Wheels job left around for the docs build,
    # we'll disable uploads.
    if os_type == "linux" and btype == "wheel":
        upload = False

106
    if upload:
107
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
108
109
110
111
        # disable smoke tests, they are broken and needs to be fixed
        # if filter_branch == "nightly" and os_type in ["linux", "win"]:
        #     pydistro = "pip" if btype == "wheel" else "conda"
        #     w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
112
113
114
115

    return w


116
117
118
119
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
120
121
122
        "requires": [
            "binary_linux_wheel_py3.7_cpu",
        ],
123
124
125
    }

    if filter_branch:
126
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
127
128
129
130
131
132
133
134
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
135
136
137
        "requires": [
            "build_docs",
        ],
138
139
140
    }

    if filter_branch:
141
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
142
143
144
    return [{"upload_docs": job}]


145
manylinux_images = {
146
    "cu116": "pytorch/manylinux-cuda116",
ptrblck's avatar
ptrblck committed
147
    "cu117": "pytorch/manylinux-cuda117",
148
149
150
151
}


def get_manylinux_image(cu_version):
Jeff Daily's avatar
Jeff Daily committed
152
    if cu_version == "cpu":
153
        return "pytorch/manylinux-cpu"
154
155
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
156
        return f"pytorch/manylinux-cuda{cu_suffix}"
157
158
    elif cu_version.startswith("rocm"):
        rocm_suffix = cu_version[len("rocm") :]
Jeff Daily's avatar
Jeff Daily committed
159
        return f"pytorch/manylinux-rocm:{rocm_suffix}"
160
161


162
163
164
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
165
166
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
167
        return f"pytorch/conda-builder:cuda{cu_suffix}"
168
169


170
171
172
def generate_base_workflow(
    base_workflow_name, python_version, cu_version, unicode, os_type, btype, *, filter_branch=None
):
173
174
175
176

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
177
        "cu_version": cu_version,
178
179
    }

180
    if os_type != "win" and unicode:
181
        d["unicode_abi"] = "1"
182

183
184
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
Jeff Daily's avatar
Jeff Daily committed
185
186
187
        # ROCm conda packages not yet supported
        if "rocm" not in cu_version:
            d["conda_docker_image"] = get_conda_image(cu_version)
188

189
    if filter_branch is not None:
190
        d["filters"] = {
191
            "branches": {"only": filter_branch},
192
193
194
195
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
196
            },
197
        }
198

199
    w = f"binary_{os_type}_{btype}"
200
    return {w: d}
201
202


203
204
205
206
207
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
208
209


210
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
211
212
213
214
215
216
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

217
218
    if btype == "wheel":
        d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
219

220
    if filter_branch is not None:
221
        d["filters"] = {
222
            "branches": {"only": filter_branch},
223
224
225
226
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
227
            },
228
        }
229

230
231
232
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Nikita Shulga's avatar
Nikita Shulga committed
248
    return {f"smoke_test_{os_type}_{pydistro}": d}
guyang3532's avatar
guyang3532 committed
249
250


251
def indent(indentation, data_list):
252
    return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines())
253
254


255
256
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
257
    for os_type in ["linux", "windows", "macos"]:
258
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
259
260
            if os_type == "macos" and device_type == "gpu":
                continue
261
262
            if os_type == "linux" and device_type == "cpu":
                continue
263
264
265
266
267
268
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

269
                if device_type == "gpu":
270
                    if python_version != "3.8":
271
                        job["filters"] = gen_filter_branch_tree("main", "nightly")
Andrey Talman's avatar
Andrey Talman committed
272
                    job["cu_version"] = "cu116"
273
                else:
274
                    job["cu_version"] = "cpu"
275
276
277
278
279
280

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


281
282
def cmake_workflows(indentation=6):
    jobs = []
283
284
    python_version = "3.8"
    for os_type in ["linux", "windows", "macos"]:
285
        # Skip OSX CUDA
286
        device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"]
287
        for device in device_types:
288
            job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version}
289

Andrey Talman's avatar
Andrey Talman committed
290
            job["cu_version"] = "cu116" if device == "gpu" else "cpu"
291
            if device == "gpu" and os_type == "linux":
Andrey Talman's avatar
Andrey Talman committed
292
                job["wheel_docker_image"] = "pytorch/manylinux-cuda116"
293
            jobs.append({f"cmake_{os_type}_{device}": job})
294
295
296
    return indent(indentation, jobs)


297
298
299
300
301
def ios_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""
302
303
    for arch, platform in [("x86_64", "SIMULATOR"), ("arm64", "OS")]:
        name = f"{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}"
304
305
        build_job_names.append(name)
        build_job = {
306
307
308
309
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}",
            "ios_arch": arch,
            "ios_platform": platform,
            "name": name,
310
311
        }
        if nightly:
312
313
            build_job["filters"] = gen_filter_branch_tree("nightly")
        jobs.append({"binary_ios_build": build_job})
314
315
316

    if nightly:
        upload_job = {
317
318
319
320
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "requires": build_job_names,
321
        }
322
        jobs.append({"binary_ios_upload": upload_job})
323
324
325
    return indent(indentation, jobs)


326
327
328
329
330
331
def android_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""

332
    name = f"{name_prefix}binary_libtorchvision_ops_android"
333
334
    build_job_names.append(name)
    build_job = {
335
336
        "build_environment": f"{env_prefix}binary-libtorchvision_ops-android",
        "name": name,
337
338
339
340
    }

    if nightly:
        upload_job = {
341
342
343
344
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-android-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "name": f"{name_prefix}binary_libtorchvision_ops_android_upload",
345
        }
346
        jobs.append({"binary_android_upload": upload_job})
347
    else:
348
        jobs.append({"binary_android_build": build_job})
349
350
351
    return indent(indentation, jobs)


352
353
354
355
356
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
357
        autoescape=select_autoescape(enabled_extensions=("html", "xml")),
358
        keep_trailing_newline=True,
359
360
    )

361
362
363
364
365
366
367
368
369
370
    with open(os.path.join(d, "config.yml"), "w") as f:
        f.write(
            env.get_template("config.yml.in").render(
                build_workflows=build_workflows,
                unittest_workflows=unittest_workflows,
                cmake_workflows=cmake_workflows,
                ios_workflows=ios_workflows,
                android_workflows=android_workflows,
            )
        )