regenerate.py 4.09 KB
Newer Older
facebook-github-bot's avatar
facebook-github-bot committed
1
2
3
4
5
6
7
8
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.

"""
This script is adapted from the torchvision one.
"""

import os.path
9

facebook-github-bot's avatar
facebook-github-bot committed
10
11
12
13
import jinja2
import yaml


Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
14
15
# The CUDA versions which have pytorch conda packages available for linux for each
# version of pytorch.
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
16
# Pytorch 1.4 also supports cuda 10.0 but we no longer build for cuda 10.0 at all.
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
17
CONDA_CUDA_VERSIONS = {
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
18
    "1.4": ["cu92", "cu101"],
19
20
    "1.5.0": ["cu92", "cu101", "cu102"],
    "1.5.1": ["cu92", "cu101", "cu102"],
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
21
    "1.6.0": ["cu92", "cu101", "cu102"],
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
22
    "1.7.0": ["cu101", "cu102", "cu110"],
23
    "1.7.1": ["cu101", "cu102", "cu110"],
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
24
25
    "1.8.0": ["cu101", "cu102", "cu111"],
    "1.8.1": ["cu101", "cu102", "cu111"],
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
26
27
28
}


29
30
31
32
33
34
35
def pytorch_versions_for_python(python_version):
    if python_version in ["3.6", "3.7", "3.8"]:
        return list(CONDA_CUDA_VERSIONS)
    pytorch_without_py39 = ["1.4", "1.5.0", "1.5.1", "1.6.0", "1.7.0"]
    return [i for i in CONDA_CUDA_VERSIONS if i not in pytorch_without_py39]


facebook-github-bot's avatar
facebook-github-bot committed
36
37
38
def workflows(prefix="", filter_branch=None, upload=False, indentation=6):
    w = []
    for btype in ["conda"]:
39
40
        for python_version in ["3.6", "3.7", "3.8", "3.9"]:
            for pytorch_version in pytorch_versions_for_python(python_version):
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
41
42
43
44
45
46
47
48
49
50
                for cu_version in CONDA_CUDA_VERSIONS[pytorch_version]:
                    w += workflow_pair(
                        btype=btype,
                        python_version=python_version,
                        pytorch_version=pytorch_version,
                        cu_version=cu_version,
                        prefix=prefix,
                        upload=upload,
                        filter_branch=filter_branch,
                    )
facebook-github-bot's avatar
facebook-github-bot committed
51
52
53
54
55

    return indent(indentation, w)


def workflow_pair(
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
56
57
58
59
60
61
62
63
    *,
    btype,
    python_version,
    pytorch_version,
    cu_version,
    prefix="",
    upload=False,
    filter_branch,
facebook-github-bot's avatar
facebook-github-bot committed
64
65
66
):

    w = []
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
67
68
69
    py = python_version.replace(".", "")
    pyt = pytorch_version.replace(".", "")
    base_workflow_name = f"{prefix}linux_{btype}_py{py}_{cu_version}_pyt{pyt}"
facebook-github-bot's avatar
facebook-github-bot committed
70
71
72
73
74

    w.append(
        generate_base_workflow(
            base_workflow_name=base_workflow_name,
            python_version=python_version,
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
75
            pytorch_version=pytorch_version,
facebook-github-bot's avatar
facebook-github-bot committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
            cu_version=cu_version,
            btype=btype,
            filter_branch=filter_branch,
        )
    )

    if upload:
        w.append(
            generate_upload_workflow(
                base_workflow_name=base_workflow_name,
                btype=btype,
                cu_version=cu_version,
                filter_branch=filter_branch,
            )
        )

    return w


def generate_base_workflow(
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
96
97
98
99
100
101
102
    *,
    base_workflow_name,
    python_version,
    cu_version,
    pytorch_version,
    btype,
    filter_branch=None,
facebook-github-bot's avatar
facebook-github-bot committed
103
104
105
106
107
108
):

    d = {
        "name": base_workflow_name,
        "python_version": python_version,
        "cu_version": cu_version,
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
109
        "pytorch_version": pytorch_version,
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
110
        "context": "DOCKERHUB_TOKEN",
facebook-github-bot's avatar
facebook-github-bot committed
111
112
113
114
115
116
117
118
    }

    if filter_branch is not None:
        d["filters"] = {"branches": {"only": filter_branch}}

    return {f"binary_linux_{btype}": d}


119
def generate_upload_workflow(*, base_workflow_name, btype, cu_version, filter_branch):
facebook-github-bot's avatar
facebook-github-bot committed
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
    d = {
        "name": f"{base_workflow_name}_upload",
        "context": "org-member",
        "requires": [base_workflow_name],
    }

    if btype == "wheel":
        d["subfolder"] = cu_version + "/"

    if filter_branch is not None:
        d["filters"] = {"branches": {"only": filter_branch}}

    return {f"binary_{btype}_upload": d}


def indent(indentation, data_list):
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
136
137
    if len(data_list) == 0:
        return ""
facebook-github-bot's avatar
facebook-github-bot committed
138
139
140
141
142
143
144
145
    return ("\n" + " " * indentation).join(
        yaml.dump(data_list, default_flow_style=False).splitlines()
    )


if __name__ == "__main__":
    d = os.path.dirname(__file__)
    env = jinja2.Environment(
Jeremy Reizenstein's avatar
Jeremy Reizenstein committed
146
147
148
149
        loader=jinja2.FileSystemLoader(d),
        lstrip_blocks=True,
        autoescape=False,
        keep_trailing_newline=True,
facebook-github-bot's avatar
facebook-github-bot committed
150
151
152
153
    )

    with open(os.path.join(d, "config.yml"), "w") as f:
        f.write(env.get_template("config.in.yml").render(workflows=workflows))