"lmdeploy/turbomind/chat.py" did not exist on "5ea40abf613e47bb56a0c06f695644d55671f585"
regenerate.py 11.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3

"""
This script should use a very simple, functional programming style.
Avoid Jinja macros in favor of native Python functions.

Don't go overboard on code generation; use Python only to generate
content that can't be easily declared statically using CircleCI's YAML API.

Data declarations (e.g. the nested loops for defining the configuration matrix)
should be at the top of the file for easy updating.

See this comment for design rationale:
https://github.com/pytorch/vision/pull/1321#issuecomment-531033978
"""
16
17

import jinja2
18
from jinja2 import select_autoescape
19
import yaml
20
21
import os.path

22

23
PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
24

25
26
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"

27
28

def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6, windows_latest_only=False):
29
30
    w = []
    for btype in ["wheel", "conda"]:
31
        for os_type in ["linux", "macos", "win"]:
32
            python_versions = PYTHON_VERSIONS
33
            cu_versions_dict = {"linux": ["cpu", "cu102", "cu111", "rocm4.1", "rocm4.2"],
Nikita Shulga's avatar
Nikita Shulga committed
34
                                "win": ["cpu", "cu102", "cu111"],
35
36
                                "macos": ["cpu"]}
            cu_versions = cu_versions_dict[os_type]
37
38
            for python_version in python_versions:
                for cu_version in cu_versions:
Jeff Daily's avatar
Jeff Daily committed
39
40
41
                    # ROCm conda packages not yet supported
                    if cu_version.startswith('rocm') and btype == "conda":
                        continue
Nikita Shulga's avatar
Nikita Shulga committed
42
                    for unicode in [False]:
43
44
45
46
47
                        fb = filter_branch
                        if windows_latest_only and os_type == "win" and filter_branch is None and \
                            (python_version != python_versions[-1] or
                             (cu_version not in [cu_versions[0], cu_versions[-1]])):
                            fb = "master"
48
49
50
51
52
                        if not fb and (os_type == 'linux' and
                                       cu_version == 'cpu' and
                                       btype == 'wheel' and
                                       python_version == '3.7'):
                            # the fields must match the build_docs "requires" dependency
53
                            fb = "/.*/"
54
55
                        w += workflow_pair(
                            btype, os_type, python_version, cu_version,
56
                            unicode, prefix, upload, filter_branch=fb)
57

58
59
    if not filter_branch:
        # Build on every pull request, but upload only on nightly and tags
60
        w += build_doc_job('/.*/')
61
        w += upload_doc_job('nightly')
62
63
64
    return indent(indentation, w)


65
def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix='', upload=False, *, filter_branch=None):
66
67
68
69
70

    w = []
    unicode_suffix = "u" if unicode else ""
    base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}{unicode_suffix}_{cu_version}"

71
72
73
    w.append(generate_base_workflow(
        base_workflow_name, python_version, cu_version,
        unicode, os_type, btype, filter_branch=filter_branch))
74
75

    if upload:
76
        w.append(generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, filter_branch=filter_branch))
guyang3532's avatar
guyang3532 committed
77
78
79
        if filter_branch == 'nightly' and os_type in ['linux', 'win']:
            pydistro = 'pip' if btype == 'wheel' else 'conda'
            w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type))
80
81
82
83

    return w


84
85
86
87
88
89
90
91
def build_doc_job(filter_branch):
    job = {
        "name": "build_docs",
        "python_version": "3.7",
        "requires": ["binary_linux_wheel_py3.7_cpu", ],
    }

    if filter_branch:
92
93
        job["filters"] = gen_filter_branch_tree(filter_branch,
                                                tags_list=RC_PATTERN)
94
95
96
97
98
99
100
101
102
103
104
105
    return [{"build_docs": job}]


def upload_doc_job(filter_branch):
    job = {
        "name": "upload_docs",
        "context": "org-member",
        "python_version": "3.7",
        "requires": ["build_docs", ],
    }

    if filter_branch:
106
107
        job["filters"] = gen_filter_branch_tree(filter_branch,
                                                tags_list=RC_PATTERN)
108
109
110
    return [{"upload_docs": job}]


111
112
113
114
manylinux_images = {
    "cu92": "pytorch/manylinux-cuda92",
    "cu101": "pytorch/manylinux-cuda101",
    "cu102": "pytorch/manylinux-cuda102",
peterjc123's avatar
peterjc123 committed
115
    "cu110": "pytorch/manylinux-cuda110",
116
    "cu111": "pytorch/manylinux-cuda111",
117
    "cu112": "pytorch/manylinux-cuda112",
118
119
120
121
}


def get_manylinux_image(cu_version):
Jeff Daily's avatar
Jeff Daily committed
122
123
124
    if cu_version == "cpu":
        return "pytorch/manylinux-cuda102"
    elif cu_version.startswith('cu'):
125
        cu_suffix = cu_version[len('cu'):]
Jeff Daily's avatar
Jeff Daily committed
126
127
128
129
        return f"pytorch/manylinux-cuda{cu_suffix}"
    elif cu_version.startswith('rocm'):
        rocm_suffix = cu_version[len('rocm'):]
        return f"pytorch/manylinux-rocm:{rocm_suffix}"
130
131


132
133
134
def get_conda_image(cu_version):
    if cu_version == "cpu":
        return "pytorch/conda-builder:cpu"
Jeff Daily's avatar
Jeff Daily committed
135
    elif cu_version.startswith('cu'):
136
        cu_suffix = cu_version[len('cu'):]
Jeff Daily's avatar
Jeff Daily committed
137
        return f"pytorch/conda-builder:cuda{cu_suffix}"
138
139


140
141
def generate_base_workflow(base_workflow_name, python_version, cu_version,
                           unicode, os_type, btype, *, filter_branch=None):
142
143
144
145

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
146
        "cu_version": cu_version,
147
148
    }

149
    if os_type != "win" and unicode:
150
151
        d["unicode_abi"] = '1'

152
153
    if os_type != "win":
        d["wheel_docker_image"] = get_manylinux_image(cu_version)
Jeff Daily's avatar
Jeff Daily committed
154
155
156
        # ROCm conda packages not yet supported
        if "rocm" not in cu_version:
            d["conda_docker_image"] = get_conda_image(cu_version)
157

158
    if filter_branch is not None:
159
160
161
162
163
164
165
166
167
168
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
169

170
    w = f"binary_{os_type}_{btype}"
171
    return {w: d}
172
173


174
175
176
177
178
def gen_filter_branch_tree(*branches, tags_list=None):
    filter_dict = {"branches": {"only": [b for b in branches]}}
    if tags_list is not None:
        filter_dict["tags"] = {"only": tags_list}
    return filter_dict
179
180


181
def generate_upload_workflow(base_workflow_name, os_type, btype, cu_version, *, filter_branch=None):
182
183
184
185
186
187
188
189
190
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

    if btype == 'wheel':
        d["subfolder"] = "" if os_type == 'macos' else cu_version + "/"

191
    if filter_branch is not None:
192
193
194
195
196
197
198
199
200
201
        d["filters"] = {
            "branches": {
                "only": filter_branch
            },
            "tags": {
                # Using a raw string here to avoid having to escape
                # anything
                "only": r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
            }
        }
202

203
204
205
    return {f"binary_{btype}_upload": d}


guyang3532's avatar
guyang3532 committed
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version, os_type):

    required_build_suffix = "_upload"
    required_build_name = base_workflow_name + required_build_suffix

    smoke_suffix = f"smoke_test_{pydistro}"
    d = {
        "name": f"{base_workflow_name}_{smoke_suffix}",
        "requires": [required_build_name],
        "python_version": python_version,
    }

    if filter_branch:
        d["filters"] = gen_filter_branch_tree(filter_branch)

Nikita Shulga's avatar
Nikita Shulga committed
221
    return {f"smoke_test_{os_type}_{pydistro}": d}
guyang3532's avatar
guyang3532 committed
222
223


224
def indent(indentation, data_list):
225
226
    return ("\n" + " " * indentation).join(
        yaml.dump(data_list, default_flow_style=False).splitlines())
227
228


229
230
def unittest_workflows(indentation=6):
    jobs = []
Francisco Massa's avatar
Francisco Massa committed
231
    for os_type in ["linux", "windows", "macos"]:
232
        for device_type in ["cpu", "gpu"]:
Francisco Massa's avatar
Francisco Massa committed
233
234
            if os_type == "macos" and device_type == "gpu":
                continue
235
236
237
238
239
240
241
242
243
            for i, python_version in enumerate(PYTHON_VERSIONS):
                job = {
                    "name": f"unittest_{os_type}_{device_type}_py{python_version}",
                    "python_version": python_version,
                }

                if device_type == 'gpu':
                    if python_version != "3.8":
                        job['filters'] = gen_filter_branch_tree('master', 'nightly')
244
                    job['cu_version'] = 'cu102'
245
246
247
248
249
250
251
252
                else:
                    job['cu_version'] = 'cpu'

                jobs.append({f"unittest_{os_type}_{device_type}": job})

    return indent(indentation, jobs)


253
254
255
256
def cmake_workflows(indentation=6):
    jobs = []
    python_version = '3.8'
    for os_type in ['linux', 'windows', 'macos']:
257
258
        # Skip OSX CUDA
        device_types = ['cpu', 'gpu'] if os_type != 'macos' else ['cpu']
259
260
261
262
263
264
        for device in device_types:
            job = {
                'name': f'cmake_{os_type}_{device}',
                'python_version': python_version
            }

265
            job['cu_version'] = 'cu102' if device == 'gpu' else 'cpu'
266
            if device == 'gpu' and os_type == 'linux':
267
                job['wheel_docker_image'] = 'pytorch/manylinux-cuda102'
268
269
270
271
            jobs.append({f'cmake_{os_type}_{device}': job})
    return indent(indentation, jobs)


272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
def ios_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""
    for arch, platform in [('x86_64', 'SIMULATOR'), ('arm64', 'OS')]:
        name = f'{name_prefix}binary_libtorchvision_ops_ios_12.0.0_{arch}'
        build_job_names.append(name)
        build_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-{arch}',
            'ios_arch': arch,
            'ios_platform': platform,
            'name': name,
        }
        if nightly:
            build_job['filters'] = gen_filter_branch_tree('nightly')
        jobs.append({'binary_ios_build': build_job})

    if nightly:
        upload_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-ios-12.0.0-upload',
            'context': 'org-member',
            'filters': gen_filter_branch_tree('nightly'),
            'requires': build_job_names,
        }
        jobs.append({'binary_ios_upload': upload_job})
    return indent(indentation, jobs)


301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
def android_workflows(indentation=6, nightly=False):
    jobs = []
    build_job_names = []
    name_prefix = "nightly_" if nightly else ""
    env_prefix = "nightly-" if nightly else ""

    name = f'{name_prefix}binary_libtorchvision_ops_android'
    build_job_names.append(name)
    build_job = {
        'build_environment': f'{env_prefix}binary-libtorchvision_ops-android',
        'name': name,
    }

    if nightly:
        upload_job = {
            'build_environment': f'{env_prefix}binary-libtorchvision_ops-android-upload',
            'context': 'org-member',
            'filters': gen_filter_branch_tree('nightly'),
            'name': f'{name_prefix}binary_libtorchvision_ops_android_upload'
        }
        jobs.append({'binary_android_upload': upload_job})
    else:
        jobs.append({'binary_android_build': build_job})
    return indent(indentation, jobs)


327
328
329
330
331
if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
332
        autoescape=select_autoescape(enabled_extensions=('html', 'xml')),
333
        keep_trailing_newline=True,
334
335
336
    )

    with open(os.path.join(d, 'config.yml'), 'w') as f:
337
338
339
        f.write(env.get_template('config.yml.in').render(
            build_workflows=build_workflows,
            unittest_workflows=unittest_workflows,
340
            cmake_workflows=cmake_workflows,
341
            ios_workflows=ios_workflows,
342
            android_workflows=android_workflows,
343
        ))