regenerate.py 11.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16

17
18
import os.path

19
import jinja2
20
import yaml
21
from jinja2 import select_autoescape
22

23

24
PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
25

26
27
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

28

29
def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
30
31
    w = []
    for btype in ["wheel", "conda"]:
32
        for os_type in ["linux", "macos", "win"]:
33
            python_versions = PYTHON_VERSIONS
34
            cu_versions_dict = {
35
36
                "linux": ["cpu", "cu102", "cu113", "cu116", "rocm4.5.2", "rocm5.0"],
                "win": ["cpu", "cu113", "cu116"],
37
38
                "macos": ["cpu"],
            }
39
            cu_versions = cu_versions_dict[os_type]
40
41
            for python_version in python_versions:
                for cu_version in cu_versions:
Jeff Daily's avatar
Jeff Daily committed
42
                    # ROCm conda packages not yet supported
43
                    if cu_version.startswith("rocm") and btype == "conda":
Jeff Daily's avatar
Jeff Daily committed
44
                        continue
Nikita Shulga's avatar
Nikita Shulga committed
45
                    for unicode in [False]:
46
                        fb = filter_branch
47
48
49
50
51
52
53
54
55
                        if (
                            windows_latest_only
                            and os_type == "win"
                            and filter_branch is None
                            and (
                                python_version != python_versions[-1]
                                or (cu_version not in [cu_versions[0], cu_versions[-1]])
                            )
                        ):
56
                            fb = "main"
57
58
59
                        if not fb and (
                            os_type == "linux" and cu_version == "cpu" and btype == "wheel" and python_version == "3.7"
                        ):
60
                            # the fields must match the build_docs "requires" dependency
61
                            fb = "/.*/"
62
                        w += workflow_pair(
63
64
                            btype, os_type, python_version, cu_version, unicode, prefix, upload, filter_branch=fb
                        )
65

66
67
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
68
69
        w += build_doc_job("/.*/")
        w += upload_doc_job("nightly")
70
71
72
    return indent(indentation, w)


73
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix="", upload=False, *, filter_branch=None):
74
75
76
77
78

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

79
80
81
82
83
    w.append(
        generate_base_workflow(
            base_workflow_name, python_version, cu_version, unicode, os_type, btype, filter_branch=filter_branch
        )
    )
84
85

    if upload:
86
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
87
88
89
90
        # disable smoke tests, they are broken and needs to be fixed
        # if filter_branch == "nightly" and os_type in ["linux", "win"]:
        #     pydistro = "pip" if btype == "wheel" else "conda"
        #     w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
91
92
93
94

    return w


95
96
97
98
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
99
100
101
        "requires": [
            "binary_linux_wheel_py3.7_cpu",
        ],
102
103
104
    }

    if filter_branch:
105
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
106
107
108
109
110
111
112
113
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
114
115
116
        "requires": [
            "build_docs",
        ],
117
118
119
    }

    if filter_branch:
120
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
121
122
123
    return [{"upload_docs": job}]


124
125
manylinux_images = {
    "cu102": "pytorch/manylinux-cuda102",
126
    "cu113": "pytorch/manylinux-cuda113",
127
    "cu116": "pytorch/manylinux-cuda116",
128
129
130
131
}


def get_manylinux_image(cu_version):
Jeff Daily's avatar
Jeff Daily committed
132
133
    if cu_version == "cpu":
        return "pytorch/manylinux-cuda102"
134
135
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
136
        return f"pytorch/manylinux-cuda{cu_suffix}"
137
138
    elif cu_version.startswith("rocm"):
        rocm_suffix = cu_version[len("rocm") :]
Jeff Daily's avatar
Jeff Daily committed
139
        return f"pytorch/manylinux-rocm:{rocm_suffix}"
140
141


142
143
144
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
145
146
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
147
        return f"pytorch/conda-builder:cuda{cu_suffix}"
148
149


150
151
152
def generate_base_workflow(
    base_workflow_name, python_version, cu_version, unicode, os_type, btype, *, filter_branch=None
):
153
154
155
156

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
157
        "cu_version": cu_version,
158
159
    }

160
    if os_type != "win" and unicode:
161
        d["unicode_abi"] = "1"
162

163
164
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
Jeff Daily's avatar
Jeff Daily committed
165
166
167
        # ROCm conda packages not yet supported
        if "rocm" not in cu_version:
            d["conda_docker_image"] = get_conda_image(cu_version)
168

169
    if filter_branch is not None:
170
        d["filters"] = {
171
            "branches": {"only": filter_branch},
172
173
174
175
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
176
            },
177
        }
178

179
    w = f"binary_{os_type}_{btype}"
180
    return {w: d}
181
182


183
184
185
186
187
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
188
189


190
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
191
192
193
194
195
196
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

197
198
    if btype == "wheel":
        d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
199

200
    if filter_branch is not None:
201
        d["filters"] = {
202
            "branches": {"only": filter_branch},
203
204
205
206
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
207
            },
208
        }
209

210
211
212
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Nikita Shulga's avatar
Nikita Shulga committed
228
    return {f"smoke_test_{os_type}_{pydistro}": d}
guyang3532's avatar
guyang3532 committed
229
230


231
def indent(indentation, data_list):
232
    return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines())
233
234


235
236
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
237
    for os_type in ["linux", "windows", "macos"]:
238
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
239
240
            if os_type == "macos" and device_type == "gpu":
                continue
241
242
243
244
245
246
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

247
                if device_type == "gpu":
248
                    if python_version != "3.8":
249
250
                        job["filters"] = gen_filter_branch_tree("main", "nightly")
                    job["cu_version"] = "cu102"
251
                else:
252
                    job["cu_version"] = "cpu"
253
254
255
256
257
258

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


259
260
def cmake_workflows(indentation=6):
    jobs = []
261
262
    python_version = "3.8"
    for os_type in ["linux", "windows", "macos"]:
263
        # Skip OSX CUDA
264
        device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"]
265
        for device in device_types:
266
            job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version}
267

268
            job["cu_version"] = "cu113" if device == "gpu" else "cpu"
269
            if device == "gpu" and os_type == "linux":
270
                job["wheel_docker_image"] = "pytorch/manylinux-cuda113"
271
            jobs.append({f"cmake_{os_type}_{device}": job})
272
273
274
    return indent(indentation, jobs)


275
276
277
278
279
def ios_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""
280
281
    for arch, platform in [("x86_64", "SIMULATOR"), ("arm64", "OS")]:
        name = f"{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}"
282
283
        build_job_names.append(name)
        build_job = {
284
285
286
287
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}",
            "ios_arch": arch,
            "ios_platform": platform,
            "name": name,
288
289
        }
        if nightly:
290
291
            build_job["filters"] = gen_filter_branch_tree("nightly")
        jobs.append({"binary_ios_build": build_job})
292
293
294

    if nightly:
        upload_job = {
295
296
297
298
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "requires": build_job_names,
299
        }
300
        jobs.append({"binary_ios_upload": upload_job})
301
302
303
    return indent(indentation, jobs)


304
305
306
307
308
309
def android_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""

310
    name = f"{name_prefix}binary_libtorchvision_ops_android"
311
312
    build_job_names.append(name)
    build_job = {
313
314
        "build_environment": f"{env_prefix}binary-libtorchvision_ops-android",
        "name": name,
315
316
317
318
    }

    if nightly:
        upload_job = {
319
320
321
322
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-android-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "name": f"{name_prefix}binary_libtorchvision_ops_android_upload",
323
        }
324
        jobs.append({"binary_android_upload": upload_job})
325
    else:
326
        jobs.append({"binary_android_build": build_job})
327
328
329
    return indent(indentation, jobs)


330
331
332
333
334
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
335
        autoescape=select_autoescape(enabled_extensions=("html", "xml")),
336
        keep_trailing_newline=True,
337
338
    )

339
340
341
342
343
344
345
346
347
348
    with open(os.path.join(d, "config.yml"), "w") as f:
        f.write(
            env.get_template("config.yml.in").render(
                build_workflows=build_workflows,
                unittest_workflows=unittest_workflows,
                cmake_workflows=cmake_workflows,
                ios_workflows=ios_workflows,
                android_workflows=android_workflows,
            )
        )