regenerate.py 7.39 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16
17

import jinja2
18
import yaml
19
20
import os.path

21

22
23
24
25
PYTHON_VERSIONS = ["3.6", "3.7", "3.8"]


def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
26
27
    w = []
    for btype in ["wheel", "conda"]:
28
        for os_type in ["linux", "macos", "win"]:
29
            python_versions = PYTHON_VERSIONS
peterjc123's avatar
peterjc123 committed
30
31
            cu_versions_dict = {"linux": ["cpu", "cu92", "cu101", "cu102", "cu110"],
                                "win": ["cpu", "cu101", "cu102", "cu110"],
32
33
                                "macos": ["cpu"]}
            cu_versions = cu_versions_dict[os_type]
34
35
            for python_version in python_versions:
                for cu_version in cu_versions:
36
                    for unicode in ([False, True] if btype == "wheel" and python_version == "2.7" else [False]):
37
38
39
40
41
                        fb = filter_branch
                        if windows_latest_only and os_type == "win" and filter_branch is None and \
                            (python_version != python_versions[-1] or
                             (cu_version not in [cu_versions[0], cu_versions[-1]])):
                            fb = "master"
42
43
                        w += workflow_pair(
                            btype, os_type, python_version, cu_version,
44
                            unicode, prefix, upload, filter_branch=fb)
45
46
47
48

    return indent(indentation, w)


49
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False, *, filter_branch=None):
50
51
52
53
54

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

55
56
57
    w.append(generate_base_workflow(
        base_workflow_name, python_version, cu_version,
        unicode, os_type, btype, filter_branch=filter_branch))
58
59

    if upload:
60
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
guyang3532's avatar
guyang3532 committed
61
62
63
        if filter_branch == 'nightly' and os_type in ['linux', 'win']:
            pydistro = 'pip' if btype == 'wheel' else 'conda'
            w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
64
65
66
67

    return w


68
69
70
71
manylinux_images = {
    "cu92": "pytorch/manylinux-cuda92",
    "cu101": "pytorch/manylinux-cuda101",
    "cu102": "pytorch/manylinux-cuda102",
peterjc123's avatar
peterjc123 committed
72
    "cu110": "pytorch/manylinux-cuda110",
73
74
75
76
77
78
79
80
81
82
}


def get_manylinux_image(cu_version):
    cu_suffix = "102"
    if cu_version.startswith('cu'):
        cu_suffix = cu_version[len('cu'):]
    return f"pytorch/manylinux-cuda{cu_suffix}"


83
84
def generate_base_workflow(base_workflow_name, python_version, cu_version,
                           unicode, os_type, btype, *, filter_branch=None):
85
86
87
88

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
89
        "cu_version": cu_version,
90
91
    }

92
    if os_type != "win" and unicode:
93
94
        d["unicode_abi"] = '1'

95
96
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
97

98
    if filter_branch is not None:
99
100
101
102
103
104
105
106
107
108
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
109

110
    w = f"binary_{os_type}_{btype}"
111
    return {w: d}
112
113


114
115
116
117
def gen_filter_branch_tree(*branches):
    return {"branches": {"only": [b for b in branches]}}


118
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
119
120
121
122
123
124
125
126
127
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

    if btype == 'wheel':
        d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"

128
    if filter_branch is not None:
129
130
131
132
133
134
135
136
137
138
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
139

140
141
142
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

    return {"smoke_test_{os_type}_{pydistro}".format(os_type=os_type, pydistro=pydistro): d}


161
def indent(indentation, data_list):
162
163
    return ("\n" + " " * indentation).join(
        yaml.dump(data_list, default_flow_style=False).splitlines())
164
165


166
167
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
168
    for os_type in ["linux", "windows", "macos"]:
169
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
170
171
            if os_type == "macos" and device_type == "gpu":
                continue
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

                if device_type == 'gpu':
                    if python_version != "3.8":
                        job['filters'] = gen_filter_branch_tree('master', 'nightly')
                    job['cu_version'] = 'cu101'
                else:
                    job['cu_version'] = 'cpu'

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


190
191
192
193
def cmake_workflows(indentation=6):
    jobs = []
    python_version = '3.8'
    for os_type in ['linux', 'windows', 'macos']:
194
195
        # Skip OSX CUDA
        device_types = ['cpu', 'gpu'] if os_type != 'macos' else ['cpu']
196
197
198
199
200
201
202
        for device in device_types:
            job = {
                'name': f'cmake_{os_type}_{device}',
                'python_version': python_version
            }

            job['cu_version'] = 'cu101' if device == 'gpu' else 'cpu'
203
            if device == 'gpu' and os_type == 'linux':
204
205
206
207
208
                job['wheel_docker_image'] = 'pytorch/manylinux-cuda101'
            jobs.append({f'cmake_{os_type}_{device}': job})
    return indent(indentation, jobs)


209
210
211
212
213
214
215
216
217
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
        autoescape=False,
    )

    with open(os.path.join(d, 'config.yml'), 'w') as f:
218
219
220
        f.write(env.get_template('config.yml.in').render(
            build_workflows=build_workflows,
            unittest_workflows=unittest_workflows,
221
            cmake_workflows=cmake_workflows,
222
        ))