check_copies.py 18 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import glob
import os
import re
20
21

import black
22
23
24
25
26


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
TRANSFORMERS_PATH = "src/transformers"
27
PATH_TO_DOCS = "docs/source"
28
REPO_PATH = "."
29

30
# Mapping for files that are full copies of others (keys are copies, values the file to keep them up to data with)
31
32
33
34
FULL_COPIES = {
    "examples/tensorflow/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",
    "examples/flax/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",
}
35

36

37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
LOCALIZED_READMES = {
    # If the introduction or the conclusion of the list change, the prompts may need to be updated.
    "README.md": {
        "start_prompt": "🤗 Transformers currently provides the following architectures",
        "end_prompt": "1. Want to contribute a new model?",
        "format_model_list": "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by {paper_authors}.{supplements}",
    },
    "README_zh-hans.md": {
        "start_prompt": "🤗 Transformers 目前支持如下的架构",
        "end_prompt": "1. 想要贡献新的模型?",
        "format_model_list": "**[{title}]({model_link})** (来自 {paper_affiliations}) 伴随论文 {paper_title_link} 由 {paper_authors} 发布。{supplements}",
    },
    "README_zh-hant.md": {
        "start_prompt": "🤗 Transformers 目前支援以下的架構",
        "end_prompt": "1. 想要貢獻新的模型?",
        "format_model_list": "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by {paper_authors}.{supplements}",
    },
}


57
def _should_continue(line, indent):
58
    return line.startswith(indent) or len(line) <= 1 or re.search(r"^\s*\)(\s*->.*:|:)\s*$", line) is not None
59
60


61
def find_code_in_transformers(object_name):
Patrick von Platen's avatar
Patrick von Platen committed
62
    """Find and return the code source code of `object_name`."""
63
64
65
66
67
68
69
    parts = object_name.split(".")
    i = 0

    # First let's find the module where our object lives.
    module = parts[i]
    while i < len(parts) and not os.path.isfile(os.path.join(TRANSFORMERS_PATH, f"{module}.py")):
        i += 1
70
71
        if i < len(parts):
            module = os.path.join(module, parts[i])
72
73
74
75
76
    if i >= len(parts):
        raise ValueError(
            f"`object_name` should begin with the name of a module of transformers but got {object_name}."
        )

77
    with open(os.path.join(TRANSFORMERS_PATH, f"{module}.py"), "r", encoding="utf-8", newline="\n") as f:
78
79
80
81
82
83
        lines = f.readlines()

    # Now let's find the class / func in the code!
    indent = ""
    line_index = 0
    for name in parts[i + 1 :]:
84
85
86
        while (
            line_index < len(lines) and re.search(fr"^{indent}(class|def)\s+{name}(\(|\:)", lines[line_index]) is None
        ):
87
88
89
90
91
92
93
94
95
            line_index += 1
        indent += "    "
        line_index += 1

    if line_index >= len(lines):
        raise ValueError(f" {object_name} does not match any function or class in {module}.")

    # We found the beginning of the class / func, now let's find the end (when the indent diminishes).
    start_index = line_index
96
    while line_index < len(lines) and _should_continue(lines[line_index], indent):
97
98
99
100
101
102
103
104
105
106
        line_index += 1
    # Clean up empty lines at the end (if any).
    while len(lines[line_index - 1]) <= 1:
        line_index -= 1

    code_lines = lines[start_index:line_index]
    return "".join(code_lines)


_re_copy_warning = re.compile(r"^(\s*)#\s*Copied from\s+transformers\.(\S+\.\S+)\s*($|\S.*$)")
107
_re_replace_pattern = re.compile(r"^\s*(\S+)->(\S+)(\s+.*|$)")
108
109


110
111
112
113
114
115
116
def get_indent(code):
    lines = code.split("\n")
    idx = 0
    while idx < len(lines) and len(lines[idx]) == 0:
        idx += 1
    if idx < len(lines):
        return re.search(r"^(\s*)\S", lines[idx]).groups()[0]
117
118
119
120
121
122
123
124
125
126
127
128
    return ""


def blackify(code):
    """
    Applies the black part of our `make style` command to `code`.
    """
    has_indent = len(get_indent(code)) > 0
    if has_indent:
        code = f"class Bla:\n{code}"
    result = black.format_str(code, mode=black.FileMode([black.TargetVersion.PY35], line_length=119))
    return result[len("class Bla:\n") :] if has_indent else result
129
130


131
132
133
134
135
136
def is_copy_consistent(filename, overwrite=False):
    """
    Check if the code commented as a copy in `filename` matches the original.

    Return the differences or overwrites the content depending on `overwrite`.
    """
137
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
138
        lines = f.readlines()
139
    diffs = []
140
    line_index = 0
141
    # Not a for loop cause `lines` is going to change (if `overwrite=True`).
142
143
144
145
146
147
148
149
150
    while line_index < len(lines):
        search = _re_copy_warning.search(lines[line_index])
        if search is None:
            line_index += 1
            continue

        # There is some copied code here, let's retrieve the original.
        indent, object_name, replace_pattern = search.groups()
        theoretical_code = find_code_in_transformers(object_name)
151
        theoretical_indent = get_indent(theoretical_code)
152
153
154
155
156
157
158
159
160
161
162
163

        start_index = line_index + 1 if indent == theoretical_indent else line_index + 2
        indent = theoretical_indent
        line_index = start_index

        # Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
        should_continue = True
        while line_index < len(lines) and should_continue:
            line_index += 1
            if line_index >= len(lines):
                break
            line = lines[line_index]
164
            should_continue = _should_continue(line, indent) and re.search(f"^{indent}# End copy", line) is None
165
166
167
168
169
170
171
172
173
        # Clean up empty lines at the end (if any).
        while len(lines[line_index - 1]) <= 1:
            line_index -= 1

        observed_code_lines = lines[start_index:line_index]
        observed_code = "".join(observed_code_lines)

        # Before comparing, use the `replace_pattern` on the original code.
        if len(replace_pattern) > 0:
174
175
176
177
178
179
            patterns = replace_pattern.replace("with", "").split(",")
            patterns = [_re_replace_pattern.search(p) for p in patterns]
            for pattern in patterns:
                if pattern is None:
                    continue
                obj1, obj2, option = pattern.groups()
180
                theoretical_code = re.sub(obj1, obj2, theoretical_code)
181
182
183
                if option.strip() == "all-casing":
                    theoretical_code = re.sub(obj1.lower(), obj2.lower(), theoretical_code)
                    theoretical_code = re.sub(obj1.upper(), obj2.upper(), theoretical_code)
184

185
186
187
188
189
            # Blackify after replacement. To be able to do that, we need the header (class or function definition)
            # from the previous line
            theoretical_code = blackify(lines[start_index - 1] + theoretical_code)
            theoretical_code = theoretical_code[len(lines[start_index - 1]) :]

190
191
        # Test for a diff and act accordingly.
        if observed_code != theoretical_code:
192
            diffs.append([object_name, start_index])
193
194
195
196
            if overwrite:
                lines = lines[:start_index] + [theoretical_code] + lines[line_index:]
                line_index = start_index + 1

197
    if overwrite and len(diffs) > 0:
198
199
        # Warn the user a file has been modified.
        print(f"Detected changes, rewriting {filename}.")
200
        with open(filename, "w", encoding="utf-8", newline="\n") as f:
201
            f.writelines(lines)
202
    return diffs
203
204
205
206
207
208


def check_copies(overwrite: bool = False):
    all_files = glob.glob(os.path.join(TRANSFORMERS_PATH, "**/*.py"), recursive=True)
    diffs = []
    for filename in all_files:
209
210
        new_diffs = is_copy_consistent(filename, overwrite)
        diffs += [f"- {filename}: copy does not match {d[0]} at line {d[1]}" for d in new_diffs]
211
212
213
    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
214
            "Found the following copy inconsistencies:\n"
215
            + diff
216
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."
217
        )
218
219
220
    check_model_list_copy(overwrite=overwrite)


221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
def check_full_copies(overwrite: bool = False):
    diffs = []
    for target, source in FULL_COPIES.items():
        with open(source, "r", encoding="utf-8") as f:
            source_code = f.read()
        with open(target, "r", encoding="utf-8") as f:
            target_code = f.read()
        if source_code != target_code:
            if overwrite:
                with open(target, "w", encoding="utf-8") as f:
                    print(f"Replacing the content of {target} by the one of {source}.")
                    f.write(source_code)
            else:
                diffs.append(f"- {target}: copy does not match {source}.")

    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
            "Found the following copy inconsistencies:\n"
            + diff
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."
        )


245
def get_model_list(filename, start_prompt, end_prompt):
Patrick von Platen's avatar
Patrick von Platen committed
246
    """Extracts the model list from the README."""
247
    with open(os.path.join(REPO_PATH, filename), "r", encoding="utf-8", newline="\n") as f:
248
249
250
        lines = f.readlines()
    # Find the start of the list.
    start_index = 0
251
    while not lines[start_index].startswith(start_prompt):
252
253
254
255
256
257
258
        start_index += 1
    start_index += 1

    result = []
    current_line = ""
    end_index = start_index

259
    while not lines[end_index].startswith(end_prompt):
260
261
262
263
264
265
266
267
268
269
270
271
272
273
        if lines[end_index].startswith("1."):
            if len(current_line) > 1:
                result.append(current_line)
            current_line = lines[end_index]
        elif len(lines[end_index]) > 1:
            current_line = f"{current_line[:-1]} {lines[end_index].lstrip()}"
        end_index += 1
    if len(current_line) > 1:
        result.append(current_line)

    return "".join(result)


def split_long_line_with_indent(line, max_per_line, indent):
Patrick von Platen's avatar
Patrick von Platen committed
274
    """Split the `line` so that it doesn't go over `max_per_line` and adds `indent` to new lines."""
275
276
277
278
279
280
281
282
283
284
285
286
287
288
    words = line.split(" ")
    lines = []
    current_line = words[0]
    for word in words[1:]:
        if len(f"{current_line} {word}") > max_per_line:
            lines.append(current_line)
            current_line = " " * indent + word
        else:
            current_line = f"{current_line} {word}"
    lines.append(current_line)
    return "\n".join(lines)


def convert_to_rst(model_list, max_per_line=None):
Patrick von Platen's avatar
Patrick von Platen committed
289
    """Convert `model_list` to rst format."""
290
    # Convert **[description](link)** to `description <link>`__
291
292
293
294
295
296
297
298
299
300
301
    def _rep_link(match):
        title, link = match.groups()
        # Keep hard links for the models not released yet
        if "master" in link or not link.startswith("https://huggingface.co/transformers"):
            return f"`{title} <{link}>`__"
        # Convert links to relative links otherwise
        else:
            link = link[len("https://huggingface.co/transformers/") : -len(".html")]
            return f":doc:`{title} <{link}>`"

    model_list = re.sub(r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\*", _rep_link, model_list)
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

    # Convert [description](link) to `description <link>`__
    model_list = re.sub(r"\[([^\]]*)\]\(([^\)]*)\)", r"`\1 <\2>`__", model_list)

    # Enumerate the lines properly
    lines = model_list.split("\n")
    result = []
    for i, line in enumerate(lines):
        line = re.sub(r"^\s*(\d+)\.", f"{i+1}.", line)
        # Split the lines that are too long
        if max_per_line is not None and len(line) > max_per_line:
            prompt = re.search(r"^(\s*\d+\.\s+)\S", line)
            indent = len(prompt.groups()[0]) if prompt is not None else 0
            line = split_long_line_with_indent(line, max_per_line, indent)

        result.append(line)
    return "\n".join(result)


321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
def convert_to_localized_md(model_list, localized_model_list, format_str):
    """Convert `model_list` to each localized README."""

    def _rep(match):
        title, model_link, paper_affiliations, paper_title_link, paper_authors, supplements = match.groups()
        return format_str.format(
            title=title,
            model_link=model_link,
            paper_affiliations=paper_affiliations,
            paper_title_link=paper_title_link,
            paper_authors=paper_authors,
            supplements=" " + supplements.strip() if len(supplements) != 0 else "",
        )

    # This regex captures metadata from an English model description, including model title, model link,
    # affiliations of the paper, title of the paper, authors of the paper, and supplemental data (see DistilBERT for example).
    _re_capture_meta = re.compile(
        r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\* \(from ([^)]*)\)[^\[]*([^\)]*\)).*?by (.*?[A-Za-z\*]{2,}?)\. (.*)$"
    )

    num_models_equal = True

    if len(localized_model_list) == 0:
        localized_model_index = {}
    else:
        try:
            localized_model_index = {
                re.search(r"\*\*\[([^\]]*)", line).groups()[0]: line
                for line in localized_model_list.strip().split("\n")
            }
        except AttributeError:
            raise AttributeError("A model name in localized READMEs cannot be recognized.")

    for model in model_list.strip().split("\n"):
        title = re.search(r"\*\*\[([^\]]*)", model).groups()[0]
        if title not in localized_model_index:
            num_models_equal = False
            # Add an anchor white space behind a model description string for regex.
            # If metadata cannot be captured, the English version will be directly copied.
            localized_model_index[title] = re.sub(_re_capture_meta, _rep, model + " ")

    sorted_index = sorted(localized_model_index.items(), key=lambda x: x[0].lower())

    return num_models_equal, "\n".join(map(lambda x: x[1], sorted_index)) + "\n"


Sylvain Gugger's avatar
Sylvain Gugger committed
367
368
369
370
371
372
def _find_text_in_file(filename, start_prompt, end_prompt):
    """
    Find the text in `filename` between a line beginning with `start_prompt` and before `end_prompt`, removing empty
    lines.
    """
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
373
        lines = f.readlines()
Sylvain Gugger's avatar
Sylvain Gugger committed
374
    # Find the start prompt.
375
    start_index = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
376
    while not lines[start_index].startswith(start_prompt):
377
378
379
380
        start_index += 1
    start_index += 1

    end_index = start_index
Sylvain Gugger's avatar
Sylvain Gugger committed
381
    while not lines[end_index].startswith(end_prompt):
382
383
384
385
386
387
388
389
        end_index += 1
    end_index -= 1

    while len(lines[start_index]) <= 1:
        start_index += 1
    while len(lines[end_index]) <= 1:
        end_index -= 1
    end_index += 1
Sylvain Gugger's avatar
Sylvain Gugger committed
390
391
    return "".join(lines[start_index:end_index]), start_index, end_index, lines

392

Sylvain Gugger's avatar
Sylvain Gugger committed
393
def check_model_list_copy(overwrite=False, max_per_line=119):
Patrick von Platen's avatar
Patrick von Platen committed
394
    """Check the model lists in the README and index.rst are consistent and maybe `overwrite`."""
395
396

    # If the introduction or the conclusion of the list change, the prompts may need to be updated.
Sylvain Gugger's avatar
Sylvain Gugger committed
397
398
399
    rst_list, start_index, end_index, lines = _find_text_in_file(
        filename=os.path.join(PATH_TO_DOCS, "index.rst"),
        start_prompt="    This list is updated automatically from the README",
400
        end_prompt="Supported frameworks",
Sylvain Gugger's avatar
Sylvain Gugger committed
401
    )
402
403
404
405
406
407
408
    md_list = get_model_list(
        filename="README.md",
        start_prompt=LOCALIZED_READMES["README.md"]["start_prompt"],
        end_prompt=LOCALIZED_READMES["README.md"]["end_prompt"],
    )

    converted_rst_list = convert_to_rst(md_list, max_per_line=max_per_line)
409

410
411
412
413
414
415
416
417
418
419
420
421
    converted_md_lists = []
    for filename, value in LOCALIZED_READMES.items():
        _start_prompt = value["start_prompt"]
        _end_prompt = value["end_prompt"]
        _format_model_list = value["format_model_list"]

        localized_md_list = get_model_list(filename, _start_prompt, _end_prompt)
        num_models_equal, converted_md_list = convert_to_localized_md(md_list, localized_md_list, _format_model_list)

        converted_md_lists.append((filename, num_models_equal, converted_md_list, _start_prompt, _end_prompt))

    if converted_rst_list != rst_list:
422
        if overwrite:
423
            with open(os.path.join(PATH_TO_DOCS, "index.rst"), "w", encoding="utf-8", newline="\n") as f:
424
                f.writelines(lines[:start_index] + [converted_rst_list] + lines[end_index:])
425
426
        else:
            raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
427
428
429
430
                "The model list in the README changed and the list in `index.rst` has not been updated. Run "
                "`make fix-copies` to fix this."
            )

431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
    for converted_md_list in converted_md_lists:
        filename, num_models_equal, converted_md, _start_prompt, _end_prompt = converted_md_list
        if not num_models_equal:
            if overwrite:
                _, start_index, end_index, lines = _find_text_in_file(
                    filename=os.path.join(REPO_PATH, filename), start_prompt=_start_prompt, end_prompt=_end_prompt
                )
                with open(os.path.join(REPO_PATH, filename), "w", encoding="utf-8", newline="\n") as f:
                    f.writelines(lines[:start_index] + [converted_md] + lines[end_index:])

            else:
                raise ValueError(
                    f"The model list in the README changed and the list in `{filename}` has not been updated. Run "
                    "`make fix-copies` to fix this."
                )

Sylvain Gugger's avatar
Sylvain Gugger committed
447

448
449
450
451
452
453
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
    args = parser.parse_args()

    check_copies(args.fix_and_overwrite)
454
    check_full_copies(args.fix_and_overwrite)