regenerate.py 6.46 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16

17
18
import os.path

19
import jinja2
20
import yaml
21
from jinja2 import select_autoescape
22

23

24
PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"]
25
26


27
28
def build_download_job(filter_branch):
    job = {
29
        "name": "download_third_parties",
30
31
32
33
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
34
35
36
37
38
39
40
41
42
43
44
45
46
    return [{"download_third_parties": job}]


def build_ffmpeg_job(os_type, filter_branch):
    job = {
        "name": f"build_ffmpeg_{os_type}",
        "requires": ["download_third_parties"],
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    job["python_version"] = "foo"
    return [{f"build_ffmpeg_{os_type}": job}]
47
48


49
def build_workflow_pair(btype, os_type, python_version, cu_version, filter_branch, prefix="", upload=False):
50
51

    w = []
Nikita Shulga's avatar
Nikita Shulga committed
52
53
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}_{cu_version}"
    w.append(generate_base_workflow(base_workflow_name, python_version, cu_version, filter_branch, os_type, btype))
54
55

    if upload:
56
        w.append(generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version))
57

moto's avatar
moto committed
58
59
60
61
62
    if os_type != "macos":
        pydistro = "pip" if btype == "wheel" else "conda"
        w.append(
            generate_smoketest_workflow(
                pydistro, base_workflow_name, filter_branch, python_version, cu_version, os_type
63
            )
moto's avatar
moto committed
64
        )
65
66
67

    return w

moto's avatar
moto committed
68

69
70
71
72
def docstring_parameters_sync_job(filter_branch):
    job = {
        "name": "docstring_parameters_sync",
        "python_version": "3.8",
73
74
75
        "requires": [
            "binary_linux_wheel_py3.8_cpu",
        ],
76
77
78
79
80
81
82
    }

    if filter_branch:
        job["filters"] = gen_filter_branch_tree(filter_branch)
    return [{"docstring_parameters_sync": job}]


Nikita Shulga's avatar
Nikita Shulga committed
83
def generate_base_workflow(base_workflow_name, python_version, cu_version, filter_branch, os_type, btype):
84
85
86
87

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
Nikita Shulga's avatar
Nikita Shulga committed
88
        "cuda_version": cu_version,
89
        "requires": [f"build_ffmpeg_{os_type}"],
90
91
    }

92
93
94
95
96
    if btype == "conda":
        d["conda_docker_image"] = f'pytorch/conda-builder:{cu_version.replace("cu1","cuda1")}'
    elif cu_version.startswith("cu"):
        d["wheel_docker_image"] = f'pytorch/manylinux-{cu_version.replace("cu1","cuda1")}'
    elif cu_version.startswith("rocm"):
97
        d["wheel_docker_image"] = f"pytorch/manylinux-rocm:{cu_version[len('rocm'):]}"
98

99
100
101
    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

102
    return {f"binary_{os_type}_{btype}": d}
103
104


moto's avatar
moto committed
105
def gen_filter_branch_tree(*branches):
106
107
108
109
110
111
112
113
    return {
        "branches": {
            "only": list(branches),
        },
        "tags": {
            # Using a raw string here to avoid having to escape
            # anything
            "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
114
        },
115
    }
116
117


118
def generate_upload_workflow(base_workflow_name, filter_branch, os_type, btype, cu_version):
119
120
121
122
123
124
    d = {
        "name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name),
        "context": "org-member",
        "requires": [base_workflow_name],
    }

125
126
    if btype == "wheel":
        d["subfolder"] = "" if os_type == "macos" else cu_version + "/"
127

128
129
130
131
132
133
    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

    return {"binary_{btype}_upload".format(btype=btype): d}


Nikita Shulga's avatar
Nikita Shulga committed
134
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, cu_version, os_type):
135

Nikita Shulga's avatar
Nikita Shulga committed
136
    smoke_suffix = f"smoke_test_{pydistro}".format(pydistro=pydistro)
137
    d = {
Nikita Shulga's avatar
Nikita Shulga committed
138
        "name": f"{base_workflow_name}_{smoke_suffix}",
moto's avatar
moto committed
139
        "requires": [base_workflow_name],
140
        "python_version": python_version,
Nikita Shulga's avatar
Nikita Shulga committed
141
        "cuda_version": cu_version,
142
143
144
145
146
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Caroline Chen's avatar
Caroline Chen committed
147
    smoke_name = f"smoke_test_{os_type}_{pydistro}"
148
    if pydistro == "conda" and (os_type == "linux" or os_type == "windows") and cu_version != "cpu":
Caroline Chen's avatar
Caroline Chen committed
149
150
        smoke_name += "_gpu"
    return {smoke_name: d}
151
152
153
154
155
156


def indent(indentation, data_list):
    return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines())


157
158
159
160
161
162
163
164
def unittest_python_versions(os):
    return {
        "windows": PYTHON_VERSIONS[:1],
        "macos": PYTHON_VERSIONS[:1],
        "linux": PYTHON_VERSIONS,
    }.get(os)


moto's avatar
moto committed
165
def unittest_workflows(indentation=6):
moto's avatar
moto committed
166
    jobs = []
167
    jobs += build_download_job(None)
moto's avatar
moto committed
168
    for os_type in ["linux", "windows", "macos"]:
moto's avatar
moto committed
169
        for device_type in ["cpu", "gpu"]:
170
            if os_type != "windows" and device_type == "gpu":
moto's avatar
moto committed
171
172
                continue

173
            for i, python_version in enumerate(unittest_python_versions(os_type)):
moto's avatar
moto committed
174
175
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
moto's avatar
moto committed
176
                    "python_version": python_version,
177
                    "cuda_version": "cpu" if device_type == "cpu" else "cu117",
178
                    "requires": ["download_third_parties"],
moto's avatar
moto committed
179
                }
moto's avatar
moto committed
180
181

                jobs.append({f"unittest_{os_type}_{device_type}": job})
182
183

                if i == 0 and os_type == "linux" and device_type == "cpu":
184
185
186
187
188
189
190
                    jobs.append(
                        {
                            "stylecheck": {
                                "name": f"stylecheck_py{python_version}",
                                "python_version": python_version,
                                "cuda_version": "cpu",
                            }
191
                        }
192
                    )
moto's avatar
moto committed
193
    return indent(indentation, jobs)
moto's avatar
moto committed
194
195


196
197
198
199
200
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
201
        autoescape=select_autoescape(enabled_extensions=("html", "xml")),
202
203
    )

204
205
206
207
208
209
    with open(os.path.join(d, "config.yml"), "w") as f:
        f.write(
            env.get_template("config.yml.in").render(
                unittest_workflows=unittest_workflows,
            )
        )
moto's avatar
moto committed
210
        f.write("\n")