regenerate.py 12.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16

17
18
import os.path

19
import jinja2
20
import yaml
21
from jinja2 import select_autoescape
22

23

24
PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
25

26
27
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

28

29
def build_workflows(prefix="", filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
30
31
    w = []
    for btype in ["wheel", "conda"]:
32
        for os_type in ["linux", "macos", "win"]:
33
            python_versions = PYTHON_VERSIONS
34
            cu_versions_dict = {
35
                "linux": ["cpu", "cu116", "cu117", "rocm5.2", "rocm5.3"],
Andrey Talman's avatar
Andrey Talman committed
36
                "win": ["cpu", "cu116", "cu117"],
37
38
                "macos": ["cpu"],
            }
39
            cu_versions = cu_versions_dict[os_type]
40
41
            for python_version in python_versions:
                for cu_version in cu_versions:
Jeff Daily's avatar
Jeff Daily committed
42
                    # ROCm conda packages not yet supported
43
                    if cu_version.startswith("rocm") and btype == "conda":
Jeff Daily's avatar
Jeff Daily committed
44
                        continue
Nikita Shulga's avatar
Nikita Shulga committed
45
                    for unicode in [False]:
46
                        fb = filter_branch
47
48
49
50
51
52
53
54
55
                        if (
                            windows_latest_only
                            and os_type == "win"
                            and filter_branch is None
                            and (
                                python_version != python_versions[-1]
                                or (cu_version not in [cu_versions[0], cu_versions[-1]])
                            )
                        ):
56
                            fb = "main"
57
58
59
                        if not fb and (
                            os_type == "linux" and cu_version == "cpu" and btype == "wheel" and python_version == "3.7"
                        ):
60
                            # the fields must match the build_docs "requires" dependency
61
                            fb = "/.*/"
62
63
64
65
66
67
68
69

                        # Disable all Linux Wheels Workflows from CircleCI
                        # since those will now be done through Nova. We'll keep
                        # around the py3.7 Linux Wheels build since the docs
                        # job depends on it.
                        if os_type == "linux" and btype == "wheel" and python_version != "3.7":
                            continue

70
71
72
73
                        # Disable all Macos Wheels Workflows from CircleCI.
                        if os_type == "macos" and btype == "wheel":
                            continue

74
                        w += workflow_pair(
75
76
                            btype, os_type, python_version, cu_version, unicode, prefix, upload, filter_branch=fb
                        )
77

78
79
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
80
81
        w += build_doc_job("/.*/")
        w += upload_doc_job("nightly")
82
83
84
    return indent(indentation, w)


85
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix="", upload=False, *, filter_branch=None):
86
87
88
89
90

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

91
92
93
94
95
    w.append(
        generate_base_workflow(
            base_workflow_name, python_version, cu_version, unicode, os_type, btype, filter_branch=filter_branch
        )
    )
96

97
98
99
100
101
    # For the remaining py3.7 Linux Wheels job left around for the docs build,
    # we'll disable uploads.
    if os_type == "linux" and btype == "wheel":
        upload = False

102
    if upload:
103
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
104
105
106
107
        # disable smoke tests, they are broken and needs to be fixed
        # if filter_branch == "nightly" and os_type in ["linux", "win"]:
        #     pydistro = "pip" if btype == "wheel" else "conda"
        #     w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
108
109
110
111

    return w


112
113
114
115
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
116
117
118
        "requires": [
            "binary_linux_wheel_py3.7_cpu",
        ],
119
120
121
    }

    if filter_branch:
122
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
123
124
125
126
127
128
129
130
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
131
132
133
        "requires": [
            "build_docs",
        ],
134
135
136
    }

    if filter_branch:
137
        job["filters"] = gen_filter_branch_tree(filter_branch, tags_list=RC_PATTERN)
138
139
140
    return [{"upload_docs": job}]


141
manylinux_images = {
142
    "cu116": "pytorch/manylinux-cuda116",
ptrblck's avatar
ptrblck committed
143
    "cu117": "pytorch/manylinux-cuda117",
144
145
146
147
}


def get_manylinux_image(cu_version):
Jeff Daily's avatar
Jeff Daily committed
148
    if cu_version == "cpu":
149
        return "pytorch/manylinux-cpu"
150
151
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
152
        return f"pytorch/manylinux-cuda{cu_suffix}"
153
154
    elif cu_version.startswith("rocm"):
        rocm_suffix = cu_version[len("rocm") :]
Jeff Daily's avatar
Jeff Daily committed
155
        return f"pytorch/manylinux-rocm:{rocm_suffix}"
156
157


158
159
160
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
161
162
    elif cu_version.startswith("cu"):
        cu_suffix = cu_version[len("cu") :]
Jeff Daily's avatar
Jeff Daily committed
163
        return f"pytorch/conda-builder:cuda{cu_suffix}"
164
165


166
167
168
def generate_base_workflow(
    base_workflow_name, python_version, cu_version, unicode, os_type, btype, *, filter_branch=None
):
169
170
171
172

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
173
        "cu_version": cu_version,
174
175
    }

176
    if os_type != "win" and unicode:
177
        d["unicode_abi"] = "1"
178

179
180
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
Jeff Daily's avatar
Jeff Daily committed
181
182
183
        # ROCm conda packages not yet supported
        if "rocm" not in cu_version:
            d["conda_docker_image"] = get_conda_image(cu_version)
184

185
    if filter_branch is not None:
186
        d["filters"] = {
187
            "branches": {"only": filter_branch},
188
189
190
191
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
192
            },
193
        }
194

195
    w = f"binary_{os_type}_{btype}"
196
    return {w: d}
197
198


199
200
201
202
203
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
204
205


206
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
207
208
209
210
211
212
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

213
214
    if btype == "wheel":
        d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
215

216
    if filter_branch is not None:
217
        d["filters"] = {
218
            "branches": {"only": filter_branch},
219
220
221
222
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
223
            },
224
        }
225

226
227
228
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Nikita Shulga's avatar
Nikita Shulga committed
244
    return {f"smoke_test_{os_type}_{pydistro}": d}
guyang3532's avatar
guyang3532 committed
245
246


247
def indent(indentation, data_list):
248
    return ("\n" + " " * indentation).join(yaml.dump(data_list, default_flow_style=False).splitlines())
249
250


251
252
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
253
    for os_type in ["linux", "windows", "macos"]:
254
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
255
256
            if os_type == "macos" and device_type == "gpu":
                continue
257
258
            if os_type == "linux" and device_type == "cpu":
                continue
259
260
261
262
263
264
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

265
                if device_type == "gpu":
266
                    if python_version != "3.8":
267
                        job["filters"] = gen_filter_branch_tree("main", "nightly")
Andrey Talman's avatar
Andrey Talman committed
268
                    job["cu_version"] = "cu116"
269
                else:
270
                    job["cu_version"] = "cpu"
271
272
273
274
275
276

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


277
278
def cmake_workflows(indentation=6):
    jobs = []
279
280
    python_version = "3.8"
    for os_type in ["linux", "windows", "macos"]:
281
        # Skip OSX CUDA
282
        device_types = ["cpu", "gpu"] if os_type != "macos" else ["cpu"]
283
        for device in device_types:
284
            job = {"name": f"cmake_{os_type}_{device}", "python_version": python_version}
285

Andrey Talman's avatar
Andrey Talman committed
286
            job["cu_version"] = "cu116" if device == "gpu" else "cpu"
287
            if device == "gpu" and os_type == "linux":
Andrey Talman's avatar
Andrey Talman committed
288
                job["wheel_docker_image"] = "pytorch/manylinux-cuda116"
289
            jobs.append({f"cmake_{os_type}_{device}": job})
290
291
292
    return indent(indentation, jobs)


293
294
295
296
297
def ios_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""
298
299
    for arch, platform in [("x86_64", "SIMULATOR"), ("arm64", "OS")]:
        name = f"{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}"
300
301
        build_job_names.append(name)
        build_job = {
302
303
304
305
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}",
            "ios_arch": arch,
            "ios_platform": platform,
            "name": name,
306
307
        }
        if nightly:
308
309
            build_job["filters"] = gen_filter_branch_tree("nightly")
        jobs.append({"binary_ios_build": build_job})
310
311
312

    if nightly:
        upload_job = {
313
314
315
316
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "requires": build_job_names,
317
        }
318
        jobs.append({"binary_ios_upload": upload_job})
319
320
321
    return indent(indentation, jobs)


322
323
324
325
326
327
def android_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""

328
    name = f"{name_prefix}binary_libtorchvision_ops_android"
329
330
    build_job_names.append(name)
    build_job = {
331
332
        "build_environment": f"{env_prefix}binary-libtorchvision_ops-android",
        "name": name,
333
334
335
336
    }

    if nightly:
        upload_job = {
337
338
339
340
            "build_environment": f"{env_prefix}binary-libtorchvision_ops-android-upload",
            "context": "org-member",
            "filters": gen_filter_branch_tree("nightly"),
            "name": f"{name_prefix}binary_libtorchvision_ops_android_upload",
341
        }
342
        jobs.append({"binary_android_upload": upload_job})
343
    else:
344
        jobs.append({"binary_android_build": build_job})
345
346
347
    return indent(indentation, jobs)


348
349
350
351
352
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
353
        autoescape=select_autoescape(enabled_extensions=("html", "xml")),
354
        keep_trailing_newline=True,
355
356
    )

357
358
359
360
361
362
363
364
365
366
    with open(os.path.join(d, "config.yml"), "w") as f:
        f.write(
            env.get_template("config.yml.in").render(
                build_workflows=build_workflows,
                unittest_workflows=unittest_workflows,
                cmake_workflows=cmake_workflows,
                ios_workflows=ios_workflows,
                android_workflows=android_workflows,
            )
        )