check_copies.py 29.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
15
16
17
"""
Utility that checks whether the copies defined in the library match the original or not. This includes:
- All code commented with `# Copied from` comments,
18
- The list of models in the main README.md matches the ones in the localized READMEs,
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
- Files that are registered as full copies of one another in the `FULL_COPIES` constant of this script.

This also checks the list of models in the README is complete (has all models) and add a line to complete if there is
a model missing.

Use from the root of the repo with:

```bash
python utils/check_copies.py
```

for a check that will error in case of inconsistencies (used by `make repo-consistency`) or

```bash
python utils/check_copies.py --fix_and_overwrite
```

for a check that will fix all inconsistencies automatically (used by `make fix-copies`).
"""
38
39
40
41
42

import argparse
import glob
import os
import re
Sylvain Gugger's avatar
Sylvain Gugger committed
43
from typing import List, Optional, Tuple
44
45

import black
46
from doc_builder.style_doc import style_docstrings_in_code
47

48
49
from transformers.utils import direct_transformers_import

50
51
52
53

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
TRANSFORMERS_PATH = "src/transformers"
54
MODEL_TEST_PATH = "tests/models"
55
PATH_TO_DOCS = "docs/source/en"
56
REPO_PATH = "."
57

58
# Mapping for files that are full copies of others (keys are copies, values the file to keep them up to data with)
59
60
61
62
FULL_COPIES = {
    "examples/tensorflow/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",
    "examples/flax/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",
}
63

64

65
66
67
68
69
LOCALIZED_READMES = {
    # If the introduction or the conclusion of the list change, the prompts may need to be updated.
    "README.md": {
        "start_prompt": "🤗 Transformers currently provides the following architectures",
        "end_prompt": "1. Want to contribute a new model?",
Sylvain Gugger's avatar
Sylvain Gugger committed
70
71
72
73
        "format_model_list": (
            "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"
            " {paper_authors}.{supplements}"
        ),
74
75
76
77
    },
    "README_zh-hans.md": {
        "start_prompt": "🤗 Transformers 目前支持如下的架构",
        "end_prompt": "1. 想要贡献新的模型?",
Sylvain Gugger's avatar
Sylvain Gugger committed
78
79
80
81
        "format_model_list": (
            "**[{title}]({model_link})** (来自 {paper_affiliations}) 伴随论文 {paper_title_link} 由 {paper_authors}"
            " 发布。{supplements}"
        ),
82
83
84
85
    },
    "README_zh-hant.md": {
        "start_prompt": "🤗 Transformers 目前支援以下的架構",
        "end_prompt": "1. 想要貢獻新的模型?",
Sylvain Gugger's avatar
Sylvain Gugger committed
86
87
88
89
        "format_model_list": (
            "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"
            " {paper_authors}.{supplements}"
        ),
90
    },
91
92
93
    "README_ko.md": {
        "start_prompt": "🤗 Transformers는 다음 모델들을 제공합니다",
        "end_prompt": "1. 새로운 모델을 올리고 싶나요?",
Sylvain Gugger's avatar
Sylvain Gugger committed
94
        "format_model_list": (
Arthur's avatar
Arthur committed
95
96
            "**[{title}]({model_link})** ({paper_affiliations} 에서 제공)은 {paper_authors}.{supplements}의"
            " {paper_title_link}논문과 함께 발표했습니다."
Sylvain Gugger's avatar
Sylvain Gugger committed
97
        ),
98
    },
99
100
101
102
103
104
105
106
    "README_es.md": {
        "start_prompt": "🤗 Transformers actualmente proporciona las siguientes arquitecturas",
        "end_prompt": "1. ¿Quieres aportar un nuevo modelo?",
        "format_model_list": (
            "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"
            " {paper_authors}.{supplements}"
        ),
    },
107
108
109
110
    "README_ja.md": {
        "start_prompt": "🤗Transformersは現在、以下のアーキテクチャを提供しています",
        "end_prompt": "1. 新しいモデルを投稿したいですか?",
        "format_model_list": (
111
112
            "**[{title}]({model_link})** ({paper_affiliations} から) {paper_authors}.{supplements} から公開された研究論文"
            " {paper_title_link}"
113
114
        ),
    },
115
116
117
118
    "README_hd.md": {
        "start_prompt": "🤗 ट्रांसफॉर्मर वर्तमान में निम्नलिखित आर्किटेक्चर का समर्थन करते हैं",
        "end_prompt": "1. एक नए मॉडल में योगदान देना चाहते हैं?",
        "format_model_list": (
119
120
            "**[{title}]({model_link})** ({paper_affiliations} से) {paper_authors}.{supplements} द्वारा"
            "अनुसंधान पत्र {paper_title_link} के साथ जारी किया गया"
121
122
        ),
    },
123
124
125
}


126
# This is to make sure the transformers module imported is the one in the repo.
127
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
128
129


Sylvain Gugger's avatar
Sylvain Gugger committed
130
def _should_continue(line: str, indent: str) -> bool:
131
132
133
    # Helper function. Returns `True` if `line` is empty, starts with the `indent` or is the end parenthesis of a
    # function definition
    return line.startswith(indent) or len(line.strip()) == 0 or re.search(r"^\s*\)(\s*->.*:|:)\s*$", line) is not None
134
135


136
def find_code_in_transformers(object_name: str, base_path: str = None) -> str:
Sylvain Gugger's avatar
Sylvain Gugger committed
137
138
139
140
    """
    Find and return the source code of an object.

    Args:
141
142
143
144
        object_name (`str`):
            The name of the object we want the source code of.
        base_path (`str`, *optional*):
            The path to the base folder where files are checked. If not set, it will be set to `TRANSFORMERS_PATH`.
Sylvain Gugger's avatar
Sylvain Gugger committed
145
146
147
148

    Returns:
        `str`: The source code of the object.
    """
149
150
151
    parts = object_name.split(".")
    i = 0

152
153
154
155
156
157
158
159
160
161
162
163
    # We can't set this as the default value in the argument, otherwise `CopyCheckTester` will fail, as it uses a
    # patched temp directory.
    if base_path is None:
        base_path = TRANSFORMERS_PATH

    # Detail: the `Copied from` statement is originally designed to work with the last part of `TRANSFORMERS_PATH`,
    # (which is `transformers`). The same should be applied for `MODEL_TEST_PATH`. However, its last part is `models`
    # (to only check and search in it) which is a bit confusing. So we keep the copied statement staring with
    # `tests.models.` and change it to `tests` here.
    if base_path == MODEL_TEST_PATH:
        base_path = "tests"

164
165
    # First let's find the module where our object lives.
    module = parts[i]
166
    while i < len(parts) and not os.path.isfile(os.path.join(base_path, f"{module}.py")):
167
        i += 1
168
169
        if i < len(parts):
            module = os.path.join(module, parts[i])
170
171
172
173
174
    if i >= len(parts):
        raise ValueError(
            f"`object_name` should begin with the name of a module of transformers but got {object_name}."
        )

175
    with open(os.path.join(base_path, f"{module}.py"), "r", encoding="utf-8", newline="\n") as f:
176
177
178
179
180
181
        lines = f.readlines()

    # Now let's find the class / func in the code!
    indent = ""
    line_index = 0
    for name in parts[i + 1 :]:
182
        while (
183
            line_index < len(lines) and re.search(rf"^{indent}(class|def)\s+{name}(\(|\:)", lines[line_index]) is None
184
        ):
185
186
187
188
189
190
191
192
            line_index += 1
        indent += "    "
        line_index += 1

    if line_index >= len(lines):
        raise ValueError(f" {object_name} does not match any function or class in {module}.")

    # We found the beginning of the class / func, now let's find the end (when the indent diminishes).
193
    start_index = line_index - 1
194
    while line_index < len(lines) and _should_continue(lines[line_index], indent):
195
196
197
198
199
200
201
202
203
204
        line_index += 1
    # Clean up empty lines at the end (if any).
    while len(lines[line_index - 1]) <= 1:
        line_index -= 1

    code_lines = lines[start_index:line_index]
    return "".join(code_lines)


_re_copy_warning = re.compile(r"^(\s*)#\s*Copied from\s+transformers\.(\S+\.\S+)\s*($|\S.*$)")
205
_re_copy_warning_for_test_file = re.compile(r"^(\s*)#\s*Copied from\s+tests\.(\S+\.\S+)\s*($|\S.*$)")
206
_re_replace_pattern = re.compile(r"^\s*(\S+)->(\S+)(\s+.*|$)")
207
_re_fill_pattern = re.compile(r"<FILL\s+[^>]*>")
208
209


Sylvain Gugger's avatar
Sylvain Gugger committed
210
211
212
213
214
215
216
217
218
219
def get_indent(code: str) -> str:
    """
    Find the indent in the first non empty line in a code sample.

    Args:
        code (`str`): The code to inspect.

    Returns:
        `str`: The indent looked at (as string).
    """
220
221
222
223
224
225
    lines = code.split("\n")
    idx = 0
    while idx < len(lines) and len(lines[idx]) == 0:
        idx += 1
    if idx < len(lines):
        return re.search(r"^(\s*)\S", lines[idx]).groups()[0]
226
227
228
    return ""


Sylvain Gugger's avatar
Sylvain Gugger committed
229
def blackify(code: str) -> str:
230
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
231
232
233
234
235
236
237
    Applies the black part of our `make style` command to some code.

    Args:
        code (`str`): The code to format.

    Returns:
        `str`: The formatted code.
238
239
240
241
    """
    has_indent = len(get_indent(code)) > 0
    if has_indent:
        code = f"class Bla:\n{code}"
242
    mode = black.Mode(target_versions={black.TargetVersion.PY37}, line_length=119)
243
    result = black.format_str(code, mode=mode)
244
    result, _ = style_docstrings_in_code(result)
245
    return result[len("class Bla:\n") :] if has_indent else result
246
247


Sylvain Gugger's avatar
Sylvain Gugger committed
248
def check_codes_match(observed_code: str, theoretical_code: str) -> Optional[int]:
249
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
250
251
252
253
254
255
256
257
258
    Checks if two version of a code match with the exception of the class/function name.

    Args:
        observed_code (`str`): The code found.
        theoretical_code (`str`): The code to match.

    Returns:
        `Optional[int]`: The index of the first line where there is a difference (if any) and `None` if the codes
        match.
259
260
261
262
    """
    observed_code_header = observed_code.split("\n")[0]
    theoretical_code_header = theoretical_code.split("\n")[0]

Sylvain Gugger's avatar
Sylvain Gugger committed
263
    # Catch the function/class name: it is expected that those do not match.
264
265
266
267
268
269
270
271
    _re_class_match = re.compile(r"class\s+([^\(:]+)(?:\(|:)")
    _re_func_match = re.compile(r"def\s+([^\(]+)\(")
    for re_pattern in [_re_class_match, _re_func_match]:
        if re_pattern.match(observed_code_header) is not None:
            observed_obj_name = re_pattern.search(observed_code_header).groups()[0]
            theoretical_name = re_pattern.search(theoretical_code_header).groups()[0]
            theoretical_code_header = theoretical_code_header.replace(theoretical_name, observed_obj_name)

Sylvain Gugger's avatar
Sylvain Gugger committed
272
    # Find the first diff. Line 0 is special since we need to compare with the function/class names ignored.
273
274
275
276
277
278
279
280
281
282
283
    diff_index = 0
    if theoretical_code_header != observed_code_header:
        return 0

    diff_index = 1
    for observed_line, theoretical_line in zip(observed_code.split("\n")[1:], theoretical_code.split("\n")[1:]):
        if observed_line != theoretical_line:
            return diff_index
        diff_index += 1


Sylvain Gugger's avatar
Sylvain Gugger committed
284
def is_copy_consistent(filename: str, overwrite: bool = False) -> Optional[List[Tuple[str, int]]]:
285
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
286
    Check if the code commented as a copy in a file matches the original.
287

Sylvain Gugger's avatar
Sylvain Gugger committed
288
289
290
291
292
293
294
295
296
    Args:
        filename (`str`):
            The name of the file to check.
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.

    Returns:
        `Optional[List[Tuple[str, int]]]`: If `overwrite=False`, returns the list of differences as tuples `(str, int)`
        with the name of the object having a diff and the line number where theere is the first diff.
297
    """
298
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
299
        lines = f.readlines()
300
    diffs = []
301
    line_index = 0
302
    # Not a for loop cause `lines` is going to change (if `overwrite=True`).
303
    while line_index < len(lines):
304
305
306
307
308
        search_re = _re_copy_warning
        if filename.startswith("tests"):
            search_re = _re_copy_warning_for_test_file

        search = search_re.search(lines[line_index])
309
310
311
312
313
314
        if search is None:
            line_index += 1
            continue

        # There is some copied code here, let's retrieve the original.
        indent, object_name, replace_pattern = search.groups()
315
316
317

        base_path = TRANSFORMERS_PATH if not filename.startswith("tests") else MODEL_TEST_PATH
        theoretical_code = find_code_in_transformers(object_name, base_path=base_path)
318
        theoretical_indent = get_indent(theoretical_code)
319

320
321
        start_index = line_index + 1 if indent == theoretical_indent else line_index
        line_index = start_index + 1
322

323
324
        subcode = "\n".join(theoretical_code.split("\n")[1:])
        indent = get_indent(subcode)
325
326
327
328
329
330
331
        # Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
        should_continue = True
        while line_index < len(lines) and should_continue:
            line_index += 1
            if line_index >= len(lines):
                break
            line = lines[line_index]
332
333
            # There is a special pattern `# End copy` to stop early. It's not documented cause it shouldn't really be
            # used.
334
            should_continue = _should_continue(line, indent) and re.search(f"^{indent}# End copy", line) is None
335
336
337
338
339
340
341
342
343
        # Clean up empty lines at the end (if any).
        while len(lines[line_index - 1]) <= 1:
            line_index -= 1

        observed_code_lines = lines[start_index:line_index]
        observed_code = "".join(observed_code_lines)

        # Before comparing, use the `replace_pattern` on the original code.
        if len(replace_pattern) > 0:
344
345
346
347
348
349
            patterns = replace_pattern.replace("with", "").split(",")
            patterns = [_re_replace_pattern.search(p) for p in patterns]
            for pattern in patterns:
                if pattern is None:
                    continue
                obj1, obj2, option = pattern.groups()
350
                theoretical_code = re.sub(obj1, obj2, theoretical_code)
351
352
353
                if option.strip() == "all-casing":
                    theoretical_code = re.sub(obj1.lower(), obj2.lower(), theoretical_code)
                    theoretical_code = re.sub(obj1.upper(), obj2.upper(), theoretical_code)
354

355
            theoretical_code = blackify(theoretical_code)
356

357
        # Test for a diff and act accordingly.
358
359
360
        diff_index = check_codes_match(observed_code, theoretical_code)
        if diff_index is not None:
            diffs.append([object_name, diff_index + start_index + 1])
361
362
363
364
            if overwrite:
                lines = lines[:start_index] + [theoretical_code] + lines[line_index:]
                line_index = start_index + 1

365
    if overwrite and len(diffs) > 0:
366
367
        # Warn the user a file has been modified.
        print(f"Detected changes, rewriting {filename}.")
368
        with open(filename, "w", encoding="utf-8", newline="\n") as f:
369
            f.writelines(lines)
370
    return diffs
371
372
373


def check_copies(overwrite: bool = False):
374
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
375
    Check every file is copy-consistent with the original. Also check the model list in the main README and other
376
    READMEs are consistent.
Sylvain Gugger's avatar
Sylvain Gugger committed
377
378
379
380

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
381
    """
382
    all_files = glob.glob(os.path.join(TRANSFORMERS_PATH, "**/*.py"), recursive=True)
383
384
385
    all_test_files = glob.glob(os.path.join(MODEL_TEST_PATH, "**/*.py"), recursive=True)
    all_files = list(all_files) + list(all_test_files)

386
387
    diffs = []
    for filename in all_files:
388
389
        new_diffs = is_copy_consistent(filename, overwrite)
        diffs += [f"- {filename}: copy does not match {d[0]} at line {d[1]}" for d in new_diffs]
390
391
392
    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
393
            "Found the following copy inconsistencies:\n"
394
            + diff
395
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."
396
        )
397
398
399
    check_model_list_copy(overwrite=overwrite)


400
def check_full_copies(overwrite: bool = False):
401
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
402
403
404
405
406
    Check the files that are full copies of others (as indicated in `FULL_COPIES`) are copy-consistent.

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
407
    """
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
    diffs = []
    for target, source in FULL_COPIES.items():
        with open(source, "r", encoding="utf-8") as f:
            source_code = f.read()
        with open(target, "r", encoding="utf-8") as f:
            target_code = f.read()
        if source_code != target_code:
            if overwrite:
                with open(target, "w", encoding="utf-8") as f:
                    print(f"Replacing the content of {target} by the one of {source}.")
                    f.write(source_code)
            else:
                diffs.append(f"- {target}: copy does not match {source}.")

    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
            "Found the following copy inconsistencies:\n"
            + diff
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."
        )


Sylvain Gugger's avatar
Sylvain Gugger committed
431
432
433
434
435
436
437
438
439
440
441
442
def get_model_list(filename: str, start_prompt: str, end_prompt: str) -> str:
    """
    Extracts the model list from a README.

    Args:
        filename (`str`): The name of the README file to check.
        start_prompt (`str`): The string to look for that introduces the model list.
        end_prompt (`str`): The string to look for that ends the model list.

    Returns:
        `str`: The model list.
    """
443
    with open(os.path.join(REPO_PATH, filename), "r", encoding="utf-8", newline="\n") as f:
444
445
446
        lines = f.readlines()
    # Find the start of the list.
    start_index = 0
447
    while not lines[start_index].startswith(start_prompt):
448
449
450
451
452
453
454
        start_index += 1
    start_index += 1

    result = []
    current_line = ""
    end_index = start_index

Sylvain Gugger's avatar
Sylvain Gugger committed
455
    # Keep going until the end of the list.
456
    while not lines[end_index].startswith(end_prompt):
457
458
459
460
461
462
463
464
465
466
467
468
469
        if lines[end_index].startswith("1."):
            if len(current_line) > 1:
                result.append(current_line)
            current_line = lines[end_index]
        elif len(lines[end_index]) > 1:
            current_line = f"{current_line[:-1]} {lines[end_index].lstrip()}"
        end_index += 1
    if len(current_line) > 1:
        result.append(current_line)

    return "".join(result)


Sylvain Gugger's avatar
Sylvain Gugger committed
470
def convert_to_localized_md(model_list: str, localized_model_list: str, format_str: str) -> Tuple[bool, str]:
471
472
473
474
475
476
477
478
479
480
481
482
483
484
    """
    Compare the model list from the main README to the one in a localized README.

    Args:
        model_list (`str`): The model list in the main README.
        localized_model_list (`str`): The model list in one of the localized README.
        format_str (`str`):
            The template for a model entry in the localized README (look at the `format_model_list` in the entries of
            `LOCALIZED_READMES` for examples).

    Returns:
        `Tuple[bool, str]`: A tuple where the first value indicates if the READMEs match or not, and the second value
        is the correct localized README.
    """
485
486
487
488
489
490
491
492
493
494
495
496
497

    def _rep(match):
        title, model_link, paper_affiliations, paper_title_link, paper_authors, supplements = match.groups()
        return format_str.format(
            title=title,
            model_link=model_link,
            paper_affiliations=paper_affiliations,
            paper_title_link=paper_title_link,
            paper_authors=paper_authors,
            supplements=" " + supplements.strip() if len(supplements) != 0 else "",
        )

    # This regex captures metadata from an English model description, including model title, model link,
498
499
    # affiliations of the paper, title of the paper, authors of the paper, and supplemental data (see DistilBERT for
    # example).
500
501
502
    _re_capture_meta = re.compile(
        r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\* \(from ([^)]*)\)[^\[]*([^\)]*\)).*?by (.*?[A-Za-z\*]{2,}?)\. (.*)$"
    )
503
504
    # This regex is used to synchronize link.
    _re_capture_title_link = re.compile(r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\*")
505
506
507
508
509
510
511
512
513
514
515
516

    if len(localized_model_list) == 0:
        localized_model_index = {}
    else:
        try:
            localized_model_index = {
                re.search(r"\*\*\[([^\]]*)", line).groups()[0]: line
                for line in localized_model_list.strip().split("\n")
            }
        except AttributeError:
            raise AttributeError("A model name in localized READMEs cannot be recognized.")

517
518
519
    model_keys = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in model_list.strip().split("\n")]

    # We exclude keys in localized README not in the main one.
520
    readmes_match = not any(k not in model_keys for k in localized_model_index)
521
522
    localized_model_index = {k: v for k, v in localized_model_index.items() if k in model_keys}

523
    for model in model_list.strip().split("\n"):
524
        title, model_link = _re_capture_title_link.search(model).groups()
525
        if title not in localized_model_index:
526
            readmes_match = False
527
528
            # Add an anchor white space behind a model description string for regex.
            # If metadata cannot be captured, the English version will be directly copied.
529
            localized_model_index[title] = _re_capture_meta.sub(_rep, model + " ")
530
531
532
533
534
        elif _re_fill_pattern.search(localized_model_index[title]) is not None:
            update = _re_capture_meta.sub(_rep, model + " ")
            if update != localized_model_index[title]:
                readmes_match = False
                localized_model_index[title] = update
535
536
537
538
539
        else:
            # Synchronize link
            localized_model_index[title] = _re_capture_title_link.sub(
                f"**[{title}]({model_link})**", localized_model_index[title], count=1
            )
540
541
542

    sorted_index = sorted(localized_model_index.items(), key=lambda x: x[0].lower())

543
    return readmes_match, "\n".join((x[1] for x in sorted_index)) + "\n"
544
545


Sylvain Gugger's avatar
Sylvain Gugger committed
546
def _find_text_in_file(filename: str, start_prompt: str, end_prompt: str) -> Tuple[str, int, int, List[str]]:
Sylvain Gugger's avatar
Sylvain Gugger committed
547
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
548
549
550
551
552
553
554
555
556
557
    Find the text in a file between two prompts.

    Args:
        filename (`str`): The name of the file to look into.
        start_prompt (`str`): The string to look for that introduces the content looked for.
        end_prompt (`str`): The string to look for that ends the content looked for.

    Returns:
        Tuple[str, int, int, List[str]]: The content between the two prompts, the index of the start line in the
        original file, the index of the end line in the original file and the list of lines of that file.
Sylvain Gugger's avatar
Sylvain Gugger committed
558
559
    """
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
560
        lines = f.readlines()
Sylvain Gugger's avatar
Sylvain Gugger committed
561
    # Find the start prompt.
562
    start_index = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
563
    while not lines[start_index].startswith(start_prompt):
564
565
566
567
        start_index += 1
    start_index += 1

    end_index = start_index
Sylvain Gugger's avatar
Sylvain Gugger committed
568
    while not lines[end_index].startswith(end_prompt):
569
570
571
572
573
574
575
576
        end_index += 1
    end_index -= 1

    while len(lines[start_index]) <= 1:
        start_index += 1
    while len(lines[end_index]) <= 1:
        end_index -= 1
    end_index += 1
Sylvain Gugger's avatar
Sylvain Gugger committed
577
578
    return "".join(lines[start_index:end_index]), start_index, end_index, lines

579

Sylvain Gugger's avatar
Sylvain Gugger committed
580
def check_model_list_copy(overwrite: bool = False):
581
582
    """
    Check the model lists in the README is consistent with the ones in the other READMES and also with `index.nmd`.
Sylvain Gugger's avatar
Sylvain Gugger committed
583
584
585
586

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
587
    """
588
589
590
591
    # Fix potential doc links in the README
    with open(os.path.join(REPO_PATH, "README.md"), "r", encoding="utf-8", newline="\n") as f:
        readme = f.read()
    new_readme = readme.replace("https://huggingface.co/transformers", "https://huggingface.co/docs/transformers")
592
    new_readme = new_readme.replace(
593
594
595
596
597
598
599
600
601
602
603
        "https://huggingface.co/docs/main/transformers", "https://huggingface.co/docs/transformers/main"
    )
    if new_readme != readme:
        if overwrite:
            with open(os.path.join(REPO_PATH, "README.md"), "w", encoding="utf-8", newline="\n") as f:
                f.write(new_readme)
        else:
            raise ValueError(
                "The main README contains wrong links to the documentation of Transformers. Run `make fix-copies` to "
                "automatically fix them."
            )
604
605
606
607
608
609
610

    md_list = get_model_list(
        filename="README.md",
        start_prompt=LOCALIZED_READMES["README.md"]["start_prompt"],
        end_prompt=LOCALIZED_READMES["README.md"]["end_prompt"],
    )

611
    # Build the converted Markdown.
612
613
614
615
616
617
618
    converted_md_lists = []
    for filename, value in LOCALIZED_READMES.items():
        _start_prompt = value["start_prompt"]
        _end_prompt = value["end_prompt"]
        _format_model_list = value["format_model_list"]

        localized_md_list = get_model_list(filename, _start_prompt, _end_prompt)
619
        readmes_match, converted_md_list = convert_to_localized_md(md_list, localized_md_list, _format_model_list)
620

621
        converted_md_lists.append((filename, readmes_match, converted_md_list, _start_prompt, _end_prompt))
622

Sylvain Gugger's avatar
Sylvain Gugger committed
623
    # Compare the converted Markdowns
624
    for converted_md_list in converted_md_lists:
625
        filename, readmes_match, converted_md, _start_prompt, _end_prompt = converted_md_list
626

627
628
629
630
631
632
633
634
        if filename == "README.md":
            continue
        if overwrite:
            _, start_index, end_index, lines = _find_text_in_file(
                filename=os.path.join(REPO_PATH, filename), start_prompt=_start_prompt, end_prompt=_end_prompt
            )
            with open(os.path.join(REPO_PATH, filename), "w", encoding="utf-8", newline="\n") as f:
                f.writelines(lines[:start_index] + [converted_md] + lines[end_index:])
635
        elif not readmes_match:
636
637
638
639
            raise ValueError(
                f"The model list in the README changed and the list in `{filename}` has not been updated. Run "
                "`make fix-copies` to fix this."
            )
640

Sylvain Gugger's avatar
Sylvain Gugger committed
641

642
# Map a model name with the name it has in the README for the check_readme check
643
644
645
646
647
648
SPECIAL_MODEL_NAMES = {
    "Bert Generation": "BERT For Sequence Generation",
    "BigBird": "BigBird-RoBERTa",
    "Data2VecAudio": "Data2Vec",
    "Data2VecText": "Data2Vec",
    "Data2VecVision": "Data2Vec",
649
    "DonutSwin": "Swin Transformer",
650
    "Marian": "MarianMT",
651
    "MaskFormerSwin": "Swin Transformer",
652
653
654
    "OpenAI GPT-2": "GPT-2",
    "OpenAI GPT": "GPT",
    "Perceiver": "Perceiver IO",
655
    "SAM": "Segment Anything",
656
657
658
659
660
661
662
663
664
665
666
667
668
669
    "ViT": "Vision Transformer (ViT)",
}

# Update this list with the models that shouldn't be in the README. This only concerns modular models or those who do
# not have an associated paper.
MODELS_NOT_IN_README = [
    "BertJapanese",
    "Encoder decoder",
    "FairSeq Machine-Translation",
    "HerBERT",
    "RetriBERT",
    "Speech Encoder decoder",
    "Speech2Text",
    "Speech2Text2",
amyeroberts's avatar
amyeroberts committed
670
    "TimmBackbone",
671
672
673
674
    "Vision Encoder decoder",
    "VisionTextDualEncoder",
]

675
# Template for new entries to add in the main README when we have missing models.
676
README_TEMPLATE = (
677
678
    "1. **[{model_name}](https://huggingface.co/docs/main/transformers/model_doc/{model_type})** (from "
    "<FILL INSTITUTION>) released with the paper [<FILL PAPER TITLE>](<FILL ARKIV LINK>) by <FILL AUTHORS>."
679
680
681
)


Sylvain Gugger's avatar
Sylvain Gugger committed
682
def check_readme(overwrite: bool = False):
683
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
684
685
686
687
688
    Check if the main README contains all the models in the library or not.

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to add an entry for the missing models using `README_TEMPLATE`.
689
    """
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
    info = LOCALIZED_READMES["README.md"]
    models, start_index, end_index, lines = _find_text_in_file(
        os.path.join(REPO_PATH, "README.md"),
        info["start_prompt"],
        info["end_prompt"],
    )
    models_in_readme = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in models.strip().split("\n")]

    model_names_mapping = transformers_module.models.auto.configuration_auto.MODEL_NAMES_MAPPING
    absents = [
        (key, name)
        for key, name in model_names_mapping.items()
        if SPECIAL_MODEL_NAMES.get(name, name) not in models_in_readme
    ]
    # Remove exceptions
    absents = [(key, name) for key, name in absents if name not in MODELS_NOT_IN_README]
    if len(absents) > 0 and not overwrite:
        print(absents)
        raise ValueError(
            "The main README doesn't contain all models, run `make fix-copies` to fill it with the missing model(s)"
            " then complete the generated entries.\nIf the model is not supposed to be in the main README, add it to"
            " the list `MODELS_NOT_IN_README` in utils/check_copies.py.\nIf it has a different name in the repo than"
            " in the README, map the correspondence in `SPECIAL_MODEL_NAMES` in utils/check_copies.py."
        )

    new_models = [README_TEMPLATE.format(model_name=name, model_type=key) for key, name in absents]

    all_models = models.strip().split("\n") + new_models
    all_models = sorted(all_models, key=lambda x: re.search(r"\*\*\[([^\]]*)", x).groups()[0].lower())
    all_models = "\n".join(all_models) + "\n"

    if all_models != models:
        if overwrite:
            print("Fixing the main README.")
            with open(os.path.join(REPO_PATH, "README.md"), "w", encoding="utf-8", newline="\n") as f:
                f.writelines(lines[:start_index] + [all_models] + lines[end_index:])
        else:
            raise ValueError("The main README model list is not properly sorted. Run `make fix-copies` to fix this.")


730
731
732
733
734
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
    args = parser.parse_args()

735
    check_readme(args.fix_and_overwrite)
736
    check_copies(args.fix_and_overwrite)
737
    check_full_copies(args.fix_and_overwrite)