regenerate.py 11.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16

17
18
import os.path

19
import jinja2
20
import yaml
21
from jinja2 import select_autoescape
22

23

24
PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
25

26
27
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

28

29
def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
30
31
    w = []
    for btype in ["wheel", "conda"]:
32
        for os_type in ["linux", "macos", "win"]:
33
            python_versions = PYTHON_VERSIONS
34
            cu_versions_dict = {
35
                "linux": ["cpu", "cu102", "cu113", "cu115", "rocm4.3.1", "rocm4.5.2"],
36
                "win": ["cpu", "cu113", "cu115"],
37
38
                "macos": ["cpu"],
            }
39
            cu_versions = cu_versions_dict[os_type]
40
41
            for python_version in python_versions:
                for cu_version in cu_versions:
Jeff Daily's avatar
Jeff Daily committed
42
                    # ROCm conda packages not yet supported
43
                    if cu_version.startswith("rocm") and btype == "conda":
Jeff Daily's avatar
Jeff Daily committed
44
                        continue
Nikita Shulga's avatar
Nikita Shulga committed
45
                    for unicode in [False]:
46
                        fb = filter_branch
47
48
49
50
51
52
53
54
55
                        if (
                            windows_latest_only
                            and os_type == "win"
                            and filter_branch is None
                            and (
                                python_version != python_versions[-1]
                                or (cu_version not in [cu_versions[0], cu_versions[-1]])
                            )
                        ):
56
                            fb = "main"
57
58
59
                        if not fb and (
                            os_type == "linux" and cu_version == "cpu" and btype == "wheel" and python_version == "3.7"
                        ):
60
                            # the fields must match the build_docs "requires" dependency
61
                            fb = "/.*/"
62
                        w += workflow_pair(
63
64
                            btype, os_type, python_version, cu_version, unicode, prefix, upload, filter_branch=fb
                        )
65

66
67
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
68
69
        w += build_doc_job("/.*/")
        w += upload_doc_job("nightly")
70
71
72
    return indent(indentation, w)


73
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix="", upload=False, *, filter_branch=None):
74
75
76
77
78

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

79
80
81
82
83
    w.append(
        generate_base_workflow(
            base_workflow_name, python_version, cu_version, unicode, os_type, btype, filter_branch=filter_branch
        )
    )
84
85

    if upload:
86
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
87
88
89
90
        # disable smoke tests, they are broken and needs to be fixed
        # if filter_branch == "nightly" and os_type in ["linux", "win"]:
        #     pydistro = "pip" if btype == "wheel" else "conda"
        #     w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
91
92
93
94

    return w


95
96
97
98
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
99
100
101
        "requires": [
            "binary_linux_wheel_py3.7_cpu",
        ],
102
103
104
    }

    if filter_branch:
105
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
106
107
108
109
110
111
112
113
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
114
115
116
        "requires": [
            "build_docs",
        ],
117
118
119
    }

    if filter_branch:
120
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
121
122
123
    return [{"upload_docs": job}]


124
125
126
127
manylinux_images = {
    "cu92": "pytorch/manylinux-cuda92",
    "cu101": "pytorch/manylinux-cuda101",
    "cu102": "pytorch/manylinux-cuda102",
peterjc123's avatar
peterjc123 committed
128
    "cu110": "pytorch/manylinux-cuda110",
129
    "cu111": "pytorch/manylinux-cuda111",
130
    "cu112": "pytorch/manylinux-cuda112",
131
    "cu113": "pytorch/manylinux-cuda113",
132
    "cu115": "pytorch/manylinux-cuda115",
133
134
135
136
}


def get_manylinux_image(cu_version):
Jeff Daily's avatar
Jeff Daily committed
137
138
    if cu_version == "cpu":
        return "pytorch/manylinux-cuda102"
139
140
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
141
        return f"pytorch/manylinux-cuda{cu_suffix}"
142
143
    elif cu_version.startswith("rocm"):
        rocm_suffix = cu_version[len("rocm") :]
Jeff Daily's avatar
Jeff Daily committed
144
        return f"pytorch/manylinux-rocm:{rocm_suffix}"
145
146


147
148
149
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
150
151
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
152
        return f"pytorch/conda-builder:cuda{cu_suffix}"
153
154


155
156
157
def generate_base_workflow(
    base_workflow_name, python_version, cu_version, unicode, os_type, btype, *, filter_branch=None
):
158
159
160
161

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
162
        "cu_version": cu_version,
163
164
    }

165
    if os_type != "win" and unicode:
166
        d["unicode_abi"] = "1"
167

168
169
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
Jeff Daily's avatar
Jeff Daily committed
170
171
172
        # ROCm conda packages not yet supported
        if "rocm" not in cu_version:
            d["conda_docker_image"] = get_conda_image(cu_version)
173

174
    if filter_branch is not None:
175
        d["filters"] = {
176
            "branches": {"only": filter_branch},
177
178
179
180
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
181
            },
182
        }
183

184
    w = f"binary_{os_type}_{btype}"
185
    return {w: d}
186
187


188
189
190
191
192
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
193
194


195
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
196
197
198
199
200
201
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

202
203
    if btype == "wheel":
        d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
204

205
    if filter_branch is not None:
206
        d["filters"] = {
207
            "branches": {"only": filter_branch},
208
209
210
211
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
212
            },
213
        }
214

215
216
217
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Nikita Shulga's avatar
Nikita Shulga committed
233
    return {f"smoke_test_{os_type}_{pydistro}": d}
guyang3532's avatar
guyang3532 committed
234
235


236
def indent(indentation, data_list):
237
    return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines())
238
239


240
241
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
242
    for os_type in ["linux", "windows", "macos"]:
243
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
244
245
            if os_type == "macos" and device_type == "gpu":
                continue
246
247
248
249
250
251
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

252
                if device_type == "gpu":
253
                    if python_version != "3.8":
254
255
                        job["filters"] = gen_filter_branch_tree("main", "nightly")
                    job["cu_version"] = "cu102"
256
                else:
257
                    job["cu_version"] = "cpu"
258
259
260
261
262
263

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


264
265
def cmake_workflows(indentation=6):
    jobs = []
266
267
    python_version = "3.8"
    for os_type in ["linux", "windows", "macos"]:
268
        # Skip OSX CUDA
269
        device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"]
270
        for device in device_types:
271
            job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version}
272

273
            job["cu_version"] = "cu113" if device == "gpu" else "cpu"
274
            if device == "gpu" and os_type == "linux":
275
                job["wheel_docker_image"] = "pytorch/manylinux-cuda113"
276
            jobs.append({f"cmake_{os_type}_{device}": job})
277
278
279
    return indent(indentation, jobs)


280
281
282
283
284
def ios_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""
285
286
    for arch, platform in [("x86_64", "SIMULATOR"), ("arm64", "OS")]:
        name = f"{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}"
287
288
        build_job_names.append(name)
        build_job = {
289
290
291
292
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}",
            "ios_arch": arch,
            "ios_platform": platform,
            "name": name,
293
294
        }
        if nightly:
295
296
            build_job["filters"] = gen_filter_branch_tree("nightly")
        jobs.append({"binary_ios_build": build_job})
297
298
299

    if nightly:
        upload_job = {
300
301
302
303
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "requires": build_job_names,
304
        }
305
        jobs.append({"binary_ios_upload": upload_job})
306
307
308
    return indent(indentation, jobs)


309
310
311
312
313
314
def android_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""

315
    name = f"{name_prefix}binary_libtorchvision_ops_android"
316
317
    build_job_names.append(name)
    build_job = {
318
319
        "build_environment": f"{env_prefix}binary-libtorchvision_ops-android",
        "name": name,
320
321
322
323
    }

    if nightly:
        upload_job = {
324
325
326
327
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-android-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "name": f"{name_prefix}binary_libtorchvision_ops_android_upload",
328
        }
329
        jobs.append({"binary_android_upload": upload_job})
330
    else:
331
        jobs.append({"binary_android_build": build_job})
332
333
334
    return indent(indentation, jobs)


335
336
337
338
339
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
340
        autoescape=select_autoescape(enabled_extensions=("html", "xml")),
341
        keep_trailing_newline=True,
342
343
    )

344
345
346
347
348
349
350
351
352
353
    with open(os.path.join(d, "config.yml"), "w") as f:
        f.write(
            env.get_template("config.yml.in").render(
                build_workflows=build_workflows,
                unittest_workflows=unittest_workflows,
                cmake_workflows=cmake_workflows,
                ios_workflows=ios_workflows,
                android_workflows=android_workflows,
            )
        )