regenerate.py 8.58 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16

17
18
import os.path

19
import jinja2
20
import yaml
21
from jinja2 import select_autoescape
22

23

24
PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
25
CU_VERSIONS_DICT = {
26
    "linux": ["cpu", "cu102", "cu116", "cu117", "rocm5.1.1", "rocm5.2"],
27
    "windows": ["cpu", "cu116", "cu117"],
28
29
    "macos": ["cpu"],
}
Nikita Shulga's avatar
Nikita Shulga committed
30

moto's avatar
moto committed
31

32
DOC_VERSION = ("linux", "3.8")
33

moto's avatar
moto committed
34

35
def build_workflows(prefix="", upload=False, filter_branch=None, indentation=6):
36
    w = []
37
    w += build_download_job(filter_branch)
38
39
    for os_type in ["linux", "macos", "windows"]:
        w += build_ffmpeg_job(os_type, filter_branch)
40
    for btype in ["wheel", "conda"]:
peterjc123's avatar
peterjc123 committed
41
        for os_type in ["linux", "macos", "windows"]:
moto's avatar
moto committed
42
            for python_version in PYTHON_VERSIONS:
Nikita Shulga's avatar
Nikita Shulga committed
43
                for cu_version in CU_VERSIONS_DICT[os_type]:
Matti Picus's avatar
Matti Picus committed
44
                    fb = filter_branch
45
                    if cu_version.startswith("rocm") and btype == "conda":
Matti Picus's avatar
Matti Picus committed
46
                        continue
47
48
49
                    if not fb and (
                        os_type == "linux" and btype == "wheel" and python_version == "3.8" and cu_version == "cpu"
                    ):
Matti Picus's avatar
Matti Picus committed
50
                        # the fields must match the build_docs "requires" dependency
51
                        fb = "/.*/"
Matti Picus's avatar
Matti Picus committed
52
                    w += build_workflow_pair(btype, os_type, python_version, cu_version, fb, prefix, upload)
53

54
55
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
56
57
        w += build_doc_job("/.*/")
        w += upload_doc_job("nightly")
58
59
        w += docstring_parameters_sync_job(None)

60
61
62
    return indent(indentation, w)


63
64
def build_download_job(filter_branch):
    job = {
65
        "name": "download_third_parties",
66
67
68
69
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
70
71
72
73
74
75
76
77
78
79
80
81
82
    return [{"download_third_parties": job}]


def build_ffmpeg_job(os_type, filter_branch):
    job = {
        "name": f"build_ffmpeg_{os_type}",
        "requires": ["download_third_parties"],
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    job["python_version"] = "foo"
    return [{f"build_ffmpeg_{os_type}": job}]
83
84


85
def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branch, prefix="", upload=False):
86
87

    w = []
Nikita Shulga's avatar
Nikita Shulga committed
88
89
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}_{cu_version}"
    w.append(generate_base_workflow(base_workflow_name, python_version, cu_version, filter_branch, os_type, btype))
90
91
92

    if upload:

93
        w.append(generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version))
94

moto's avatar
moto committed
95
96
97
98
99
    if os_type != "macos":
        pydistro = "pip" if btype == "wheel" else "conda"
        w.append(
            generate_smoketest_workflow(
                pydistro, base_workflow_name, filter_branch, python_version, cu_version, os_type
100
            )
moto's avatar
moto committed
101
        )
102
103
104

    return w

moto's avatar
moto committed
105

106
107
108
109
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.8",
110
        "cuda_version": "cu116",
111
        "requires": [
112
            "binary_linux_conda_py3.8_cu116",
113
        ],
114
115
116
117
118
119
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"build_docs": job}]

moto's avatar
moto committed
120

121
122
123
def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
124
        "context": "org-member",
125
        "python_version": "3.8",
126
127
128
        "requires": [
            "build_docs",
        ],
129
130
131
132
133
134
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"upload_docs": job}]

135

136
137
138
139
def docstring_parameters_sync_job(filter_branch):
    job = {
        "name": "docstring_parameters_sync",
        "python_version": "3.8",
140
141
142
        "requires": [
            "binary_linux_wheel_py3.8_cpu",
        ],
143
144
145
146
147
148
149
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"docstring_parameters_sync": job}]


Nikita Shulga's avatar
Nikita Shulga committed
150
def generate_base_workflow(base_workflow_name, python_version, cu_version, filter_branch, os_type, btype):
151
152
153
154

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
Nikita Shulga's avatar
Nikita Shulga committed
155
        "cuda_version": cu_version,
156
        "requires": [f"build_ffmpeg_{os_type}"],
157
158
    }

159
160
161
162
163
    if btype == "conda":
        d["conda_docker_image"] = f'pytorch/conda-builder:{cu_version.replace("cu1","cuda1")}'
    elif cu_version.startswith("cu"):
        d["wheel_docker_image"] = f'pytorch/manylinux-{cu_version.replace("cu1","cuda1")}'
    elif cu_version.startswith("rocm"):
164
        d["wheel_docker_image"] = f"pytorch/manylinux-rocm:{cu_version[len('rocm'):]}"
165

166
167
168
    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

169
    return {f"binary_{os_type}_{btype}": d}
170
171


moto's avatar
moto committed
172
def gen_filter_branch_tree(*branches):
173
174
175
176
177
178
179
180
    return {
        "branches": {
            "only": list(branches),
        },
        "tags": {
            # Using a raw string here to avoid having to escape
            # anything
            "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
181
        },
182
    }
183
184


185
def generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version):
186
187
188
189
190
191
    d = {
        "name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name),
        "context": "org-member",
        "requires": [base_workflow_name],
    }

192
193
    if btype == "wheel":
        d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
194

195
196
197
198
199
200
    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

    return {"binary_{btype}_upload".format(btype=btype): d}


Nikita Shulga's avatar
Nikita Shulga committed
201
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, cu_version, os_type):
202

Nikita Shulga's avatar
Nikita Shulga committed
203
    smoke_suffix = f"smoke_test_{pydistro}".format(pydistro=pydistro)
204
    d = {
Nikita Shulga's avatar
Nikita Shulga committed
205
        "name": f"{base_workflow_name}_{smoke_suffix}",
moto's avatar
moto committed
206
        "requires": [base_workflow_name],
207
        "python_version": python_version,
Nikita Shulga's avatar
Nikita Shulga committed
208
        "cuda_version": cu_version,
209
210
211
212
213
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Caroline Chen's avatar
Caroline Chen committed
214
    smoke_name = f"smoke_test_{os_type}_{pydistro}"
215
    if pydistro == "conda" and (os_type == "linux" or os_type == "windows") and cu_version != "cpu":
Caroline Chen's avatar
Caroline Chen committed
216
217
        smoke_name += "_gpu"
    return {smoke_name: d}
218
219
220
221
222
223


def indent(indentation, data_list):
    return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())


224
225
226
227
228
229
230
231
def unittest_python_versions(os):
    return {
        "windows": PYTHON_VERSIONS[:1],
        "macos": PYTHON_VERSIONS[:1],
        "linux": PYTHON_VERSIONS,
    }.get(os)


moto's avatar
moto committed
232
def unittest_workflows(indentation=6):
moto's avatar
moto committed
233
    jobs = []
234
    jobs += build_download_job(None)
moto's avatar
moto committed
235
    for os_type in ["linux", "windows", "macos"]:
moto's avatar
moto committed
236
        for device_type in ["cpu", "gpu"]:
moto's avatar
moto committed
237
238
239
            if os_type == "macos" and device_type == "gpu":
                continue

240
            for i, python_version in enumerate(unittest_python_versions(os_type)):
moto's avatar
moto committed
241
242
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
moto's avatar
moto committed
243
                    "python_version": python_version,
244
                    "cuda_version": "cpu" if device_type == "cpu" else "cu116",
245
                    "requires": ["download_third_parties"],
moto's avatar
moto committed
246
                }
moto's avatar
moto committed
247
248

                jobs.append({f"unittest_{os_type}_{device_type}": job})
249
250

                if i == 0 and os_type == "linux" and device_type == "cpu":
251
252
253
254
255
256
257
                    jobs.append(
                        {
                            "stylecheck": {
                                "name": f"stylecheck_py{python_version}",
                                "python_version": python_version,
                                "cuda_version": "cpu",
                            }
258
                        }
259
                    )
moto's avatar
moto committed
260
    return indent(indentation, jobs)
moto's avatar
moto committed
261
262


263
264
265
266
267
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
268
        autoescape=select_autoescape(enabled_extensions=("html", "xml")),
269
270
    )

271
272
273
274
275
276
277
    with open(os.path.join(d, "config.yml"), "w") as f:
        f.write(
            env.get_template("config.yml.in").render(
                build_workflows=build_workflows,
                unittest_workflows=unittest_workflows,
            )
        )
moto's avatar
moto committed
278
        f.write("\n")