check_copies.py 47.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
15
16
17
"""
Utility that checks whether the copies defined in the library match the original or not. This includes:
- All code commented with `# Copied from` comments,
18
- The list of models in the main README.md matches the ones in the localized READMEs,
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
- Files that are registered as full copies of one another in the `FULL_COPIES` constant of this script.

This also checks the list of models in the README is complete (has all models) and add a line to complete if there is
a model missing.

Use from the root of the repo with:

```bash
python utils/check_copies.py
```

for a check that will error in case of inconsistencies (used by `make repo-consistency`) or

```bash
python utils/check_copies.py --fix_and_overwrite
```

for a check that will fix all inconsistencies automatically (used by `make fix-copies`).
"""
38
39
40
41
42

import argparse
import glob
import os
import re
43
import subprocess
Yih-Dar's avatar
Yih-Dar committed
44
45
from collections import OrderedDict
from typing import List, Optional, Tuple, Union
46

47
48
from transformers.utils import direct_transformers_import

49
50
51
52

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_copies.py
TRANSFORMERS_PATH = "src/transformers"
53
MODEL_TEST_PATH = "tests/models"
54
PATH_TO_DOCS = "docs/source/en"
55
REPO_PATH = "."
56

57
# Mapping for files that are full copies of others (keys are copies, values the file to keep them up to data with)
58
59
60
61
FULL_COPIES = {
    "examples/tensorflow/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",
    "examples/flax/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",
}
62

63

64
65
66
67
68
LOCALIZED_READMES = {
    # If the introduction or the conclusion of the list change, the prompts may need to be updated.
    "README.md": {
        "start_prompt": "🤗 Transformers currently provides the following architectures",
        "end_prompt": "1. Want to contribute a new model?",
Sylvain Gugger's avatar
Sylvain Gugger committed
69
70
71
72
        "format_model_list": (
            "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"
            " {paper_authors}.{supplements}"
        ),
73
74
75
76
    },
    "README_zh-hans.md": {
        "start_prompt": "🤗 Transformers 目前支持如下的架构",
        "end_prompt": "1. 想要贡献新的模型?",
Sylvain Gugger's avatar
Sylvain Gugger committed
77
78
79
80
        "format_model_list": (
            "**[{title}]({model_link})** (来自 {paper_affiliations}) 伴随论文 {paper_title_link} 由 {paper_authors}"
            " 发布。{supplements}"
        ),
81
82
83
84
    },
    "README_zh-hant.md": {
        "start_prompt": "🤗 Transformers 目前支援以下的架構",
        "end_prompt": "1. 想要貢獻新的模型?",
Sylvain Gugger's avatar
Sylvain Gugger committed
85
86
87
88
        "format_model_list": (
            "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"
            " {paper_authors}.{supplements}"
        ),
89
    },
90
91
92
    "README_ko.md": {
        "start_prompt": "🤗 Transformers는 다음 모델들을 제공합니다",
        "end_prompt": "1. 새로운 모델을 올리고 싶나요?",
Sylvain Gugger's avatar
Sylvain Gugger committed
93
        "format_model_list": (
Arthur's avatar
Arthur committed
94
95
            "**[{title}]({model_link})** ({paper_affiliations} 에서 제공)은 {paper_authors}.{supplements}의"
            " {paper_title_link}논문과 함께 발표했습니다."
Sylvain Gugger's avatar
Sylvain Gugger committed
96
        ),
97
    },
98
99
100
101
102
103
104
105
    "README_es.md": {
        "start_prompt": "🤗 Transformers actualmente proporciona las siguientes arquitecturas",
        "end_prompt": "1. ¿Quieres aportar un nuevo modelo?",
        "format_model_list": (
            "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"
            " {paper_authors}.{supplements}"
        ),
    },
106
107
108
109
    "README_ja.md": {
        "start_prompt": "🤗Transformersは現在、以下のアーキテクチャを提供しています",
        "end_prompt": "1. 新しいモデルを投稿したいですか?",
        "format_model_list": (
110
111
            "**[{title}]({model_link})** ({paper_affiliations} から) {paper_authors}.{supplements} から公開された研究論文"
            " {paper_title_link}"
112
113
        ),
    },
114
115
116
117
    "README_hd.md": {
        "start_prompt": "🤗 ट्रांसफॉर्मर वर्तमान में निम्नलिखित आर्किटेक्चर का समर्थन करते हैं",
        "end_prompt": "1. एक नए मॉडल में योगदान देना चाहते हैं?",
        "format_model_list": (
118
119
            "**[{title}]({model_link})** ({paper_affiliations} से) {paper_authors}.{supplements} द्वारा"
            "अनुसंधान पत्र {paper_title_link} के साथ जारी किया गया"
120
121
        ),
    },
122
123
124
}


125
# This is to make sure the transformers module imported is the one in the repo.
126
transformers_module = direct_transformers_import(TRANSFORMERS_PATH)
127
128


Yih-Dar's avatar
Yih-Dar committed
129
130
131
132
133
def _is_definition_header_ending_line(line: str) -> bool:
    # Helper function. Returns `True` if `line` is the end parenthesis of a class/function definition
    return re.search(r"^\s*\)(\s*->.*:|:)\s*$", line) is not None


Sylvain Gugger's avatar
Sylvain Gugger committed
134
def _should_continue(line: str, indent: str) -> bool:
135
    # Helper function. Returns `True` if `line` is empty, starts with the `indent` or is the end parenthesis of a
Yih-Dar's avatar
Yih-Dar committed
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
    # class/function definition
    return line.startswith(indent) or len(line.strip()) == 0 or _is_definition_header_ending_line(line)


def _sanity_check_splits(splits_1, splits_2, is_class):
    """Check the two (inner) block structures of the corresponding code block given by `split_code_into_blocks` match.

    For the case of `class`, they must be of one of the following 3 cases:

        - a single block without name:

            class foo:
                a = 1

        - a consecutive sequence of (1 or more) blocks with name

            class foo:

                def f(x):
                    return x

        - a block without name, followed by a consecutive sequence of (1 or more) blocks with name

            class foo:
                a = 1

                def f(x):
                    return x

                def g(x):
                    return None

    The 2 code snippets that give `splits_1` and `splits_2` have to be in the same case to pass this check, but the
    number of blocks with name in the consecutive sequence is not taken into account.

    For the case of `function or method`, we don't require it to be in one of the above 3 cases. However, the structure
    of`splits_1` and `splits_2` have to match exactly. In particular, the number of blocks with name in a consecutive
    sequence is taken into account.
    """
    block_names_1 = []
    block_names_2 = []

    for block in splits_1[1:]:
        if block[0].startswith("_block_without_name_"):
            block_names_1.append("block_without_name")
        elif not block[0].startswith("_empty_block_") and (
            not is_class or len(block_names_1) == 0 or block_names_1[-1].startswith("block_without_name")
        ):
            block_names_1.append("block_with_name")

    for block in splits_2[1:]:
        if block[0].startswith("_block_without_name_"):
            block_names_2.append("block_without_name")
        elif not block[0].startswith("_empty_block_") and (
            not is_class or len(block_names_2) == 0 or block_names_2[-1].startswith("block_without_name")
        ):
            block_names_2.append("block_with_name")

    if is_class:
        if block_names_1 not in [
            ["block_without_name"],
            ["block_with_name"],
            ["block_without_name", "block_with_name"],
        ]:
            raise ValueError(
                "For a class, it must have a specific structure. See the docstring of `_sanity_check_splits` in the file `utils/check_copies.py`"
            )

    if block_names_1 != block_names_2:
        raise ValueError("The structures in the 2 code blocks differ.")


def find_block_end(lines: List[str], start_index: int, indent: int) -> int:
    """
    Find the end of the class/func block starting at `start_index` in a source code (defined by `lines`).

    Args:
        lines (`List[str]`):
            The source code, represented by a list of lines.
        start_index (`int`):
            The starting index of the target class/func block.
        indent (`int`):
            The indent of the class/func body.

    Returns:
        `int`: The index of the block's ending line plus by 1 (i.e. exclusive).
    """
    indent = " " * indent
    # enter the block body
    line_index = start_index + 1

    while line_index < len(lines) and _should_continue(lines[line_index], indent):
        line_index += 1
    # Clean up empty lines at the end (if any).
    while len(lines[line_index - 1]) <= 1:
        line_index -= 1

    return line_index


def split_code_into_blocks(
    lines: List[str], start_index: int, end_index: int, indent: int, backtrace: bool = False
) -> List[Tuple[str, int, int]]:
    """
    Split the class/func block starting at `start_index` in a source code (defined by `lines`) into *inner blocks*.

    The block's header is included as the first element. The contiguous regions (without empty lines) that are not
    inside any inner block are included as blocks. The contiguous regions of empty lines that are not inside any inner
    block are also included as (dummy) blocks.
245

Yih-Dar's avatar
Yih-Dar committed
246
247
248
249
250
251
252
253
254
255
256
257
    Args:
        lines (`List[str]`):
            The source code, represented by a list of lines.
        start_index (`int`):
            The starting index of the target class/func block.
        end_index (`int`):
            The ending index of the target class/func block.
        indent (`int`):
            The indent of the class/func body.
        backtrace (`bool`, *optional*, defaults to `False`):
            Whether or not to include the lines before the inner class/func block's header (e.g. comments, decorators,
            etc.) until an empty line is encountered.
258

Yih-Dar's avatar
Yih-Dar committed
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
    Returns:
        `List[Tuple[str, int, int]]`: A list of elements with the form `(block_name, start_index, end_index)`.
    """
    splits = []
    # `indent - 4` is the indent level of the target class/func header
    target_block_name = re.search(rf"^{' ' * (indent - 4)}((class|def)\s+\S+)(\(|\:)", lines[start_index]).groups()[0]

    # from now on, the `block` means inner blocks unless explicitly specified
    indent_str = " " * indent
    block_without_name_idx = 0
    empty_block_idx = 0

    # Find the lines for the definition header
    index = start_index
    if "(" in lines[start_index] and "):" not in lines[start_index] in lines[start_index]:
        while index < end_index:
            if _is_definition_header_ending_line(lines[index]):
                break
            index += 1

    # the first line outside the definition header
    index += 1
    splits.append((target_block_name, start_index, index))

    block_start_index, prev_block_end_index = index, index
    while index < end_index:
        # if found, it will be an inner block
        block_found = re.search(rf"^{indent_str}((class|def)\s+\S+)(\(|\:)", lines[index])
        if block_found:
            name = block_found.groups()[0]

            block_end_index = find_block_end(lines, index, indent + 4)

            # backtrace to include the lines before the found block's definition header (e.g. comments, decorators,
            # etc.) until an empty line is encountered.
            block_start_index = index
            if index > prev_block_end_index and backtrace:
                idx = index - 1
                for idx in range(index - 1, prev_block_end_index - 2, -1):
                    if not (len(lines[idx].strip()) > 0 and lines[idx].startswith(indent_str)):
                        break
                idx += 1
                if idx < index:
                    block_start_index = idx

            # between the current found block and the previous found block
            if block_start_index > prev_block_end_index:
                # give it a dummy name
                if len("".join(lines[prev_block_end_index:block_start_index]).strip()) == 0:
                    prev_block_name = f"_empty_block_{empty_block_idx}"
                    empty_block_idx += 1
                else:
                    prev_block_name = f"_block_without_name_{block_without_name_idx}"
                    block_without_name_idx += 1
                # Add it as a block
                splits.append((prev_block_name, prev_block_end_index, block_start_index))

            # Add the current found block
            splits.append((name, block_start_index, block_end_index))
            prev_block_end_index = block_end_index
            index = block_end_index - 1

        index += 1

    if index > prev_block_end_index:
        if len("".join(lines[prev_block_end_index:index]).strip()) == 0:
            prev_block_name = f"_empty_block_{empty_block_idx}"
        else:
            prev_block_name = f"_block_without_name_{block_without_name_idx}"
        splits.append((prev_block_name, prev_block_end_index, index))

    return splits


def find_code_in_transformers(
    object_name: str, base_path: str = None, return_indices: bool = False
) -> Union[str, Tuple[List[str], int, int]]:
Sylvain Gugger's avatar
Sylvain Gugger committed
336
337
338
339
    """
    Find and return the source code of an object.

    Args:
340
341
342
343
        object_name (`str`):
            The name of the object we want the source code of.
        base_path (`str`, *optional*):
            The path to the base folder where files are checked. If not set, it will be set to `TRANSFORMERS_PATH`.
Yih-Dar's avatar
Yih-Dar committed
344
345
346
347
        return_indices(`bool`, *optional*, defaults to `False`):
            If `False`, will only return the code (as a string), otherwise it will also return the whole lines of the
            file where the object specified by `object_name` is defined, together the start/end indices of the block in
            the file that defines the object.
Sylvain Gugger's avatar
Sylvain Gugger committed
348
349

    Returns:
Yih-Dar's avatar
Yih-Dar committed
350
351
352
        `Union[str, Tuple[List[str], int, int]]`: If `return_indices=False`, only the source code of the object will be
        returned. Otherwise, it also returns the whole lines of the file where the object specified by `object_name` is
        defined, together the start/end indices of the block in the file that defines the object.
Sylvain Gugger's avatar
Sylvain Gugger committed
353
    """
354
355
356
    parts = object_name.split(".")
    i = 0

357
358
359
360
361
362
363
364
365
366
367
368
    # We can't set this as the default value in the argument, otherwise `CopyCheckTester` will fail, as it uses a
    # patched temp directory.
    if base_path is None:
        base_path = TRANSFORMERS_PATH

    # Detail: the `Copied from` statement is originally designed to work with the last part of `TRANSFORMERS_PATH`,
    # (which is `transformers`). The same should be applied for `MODEL_TEST_PATH`. However, its last part is `models`
    # (to only check and search in it) which is a bit confusing. So we keep the copied statement staring with
    # `tests.models.` and change it to `tests` here.
    if base_path == MODEL_TEST_PATH:
        base_path = "tests"

369
370
    # First let's find the module where our object lives.
    module = parts[i]
371
    while i < len(parts) and not os.path.isfile(os.path.join(base_path, f"{module}.py")):
372
        i += 1
373
374
        if i < len(parts):
            module = os.path.join(module, parts[i])
375
376
377
378
379
    if i >= len(parts):
        raise ValueError(
            f"`object_name` should begin with the name of a module of transformers but got {object_name}."
        )

380
    with open(os.path.join(base_path, f"{module}.py"), "r", encoding="utf-8", newline="\n") as f:
381
382
383
384
385
386
        lines = f.readlines()

    # Now let's find the class / func in the code!
    indent = ""
    line_index = 0
    for name in parts[i + 1 :]:
387
        while (
388
            line_index < len(lines) and re.search(rf"^{indent}(class|def)\s+{name}(\(|\:)", lines[line_index]) is None
389
        ):
390
            line_index += 1
Yih-Dar's avatar
Yih-Dar committed
391
        # find the target specified in the current level in `parts` -> increase `indent` so we can search the next
392
        indent += "    "
Yih-Dar's avatar
Yih-Dar committed
393
        # the index of the first line in the (currently found) block *body*
394
395
396
397
398
        line_index += 1

    if line_index >= len(lines):
        raise ValueError(f" {object_name} does not match any function or class in {module}.")

Yih-Dar's avatar
Yih-Dar committed
399
400
    # `indent` is already one level deeper than the (found) class/func block's definition header

401
    # We found the beginning of the class / func, now let's find the end (when the indent diminishes).
Yih-Dar's avatar
Yih-Dar committed
402
    # `start_index` is the index of the class/func block's definition header
403
    start_index = line_index - 1
Yih-Dar's avatar
Yih-Dar committed
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
    end_index = find_block_end(lines, start_index, len(indent))

    code = "".join(lines[start_index:end_index])
    return (code, (lines, start_index, end_index)) if return_indices else code


def replace_code(code: str, replace_pattern: str) -> str:
    """Replace `code` by a pattern of the form `with X1->X2,Y1->Y2,Z1->Z2`.

    Args:
        code (`str`): The code to be modified.
        replace_pattern (`str`): The pattern used to modify `code`.

    Returns:
        `str`: The modified code.
    """
    if len(replace_pattern) > 0:
        patterns = replace_pattern.replace("with", "").split(",")
        patterns = [_re_replace_pattern.search(p) for p in patterns]
        for pattern in patterns:
            if pattern is None:
                continue
            obj1, obj2, option = pattern.groups()
            code = re.sub(obj1, obj2, code)
            if option.strip() == "all-casing":
                code = re.sub(obj1.lower(), obj2.lower(), code)
                code = re.sub(obj1.upper(), obj2.upper(), code)

    return code

434

Yih-Dar's avatar
Yih-Dar committed
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
def find_code_and_splits(object_name: str, base_path: str, buffer: dict = None):
    """Find the code of an object (specified by `object_name`) and split it into blocks.

    Args:
        object_name (`str`):
            The name of the object, e.g. `transformers.models.bert.modeling_bert.BertAttention` or
            `tests.models.llama.test_modeling_llama.LlamaModelTest.test_config`.
        base_path (`str`):
            The path to the base directory within which the search will be performed. It could be either
            `TRANSFORMERS_PATH` or `MODEL_TEST_PATH`.
        buffer (`dict`, *optional*):
            The buffer used to store the previous results in order to speed up the process.

    Returns:
        lines (`List[str]`):
            The lines of the whole file where the object is defined.
        code (`str`):
            The object's code.
        code_splits (`List[Tuple[str, int, int]]`):
            `code` splitted into blocks. See `split_code_into_blocks`.
    """
    if buffer is None:
        buffer = {}

    if (object_name, base_path) in buffer:
        lines, code, code_splits = buffer[(object_name, base_path)]
    else:
        code, (lines, target_start_index, target_end_index) = find_code_in_transformers(
            object_name, base_path=base_path, return_indices=True
        )
        indent = get_indent(code)

        # Split the code into blocks
        # `indent` is the indent of the class/func definition header, but `code_splits` expects the indent level of the
        # block body.
        code_splits = split_code_into_blocks(
            lines, target_start_index, target_end_index, len(indent) + 4, backtrace=True
        )
        buffer[(object_name, base_path)] = lines, code, code_splits

    return lines, code, code_splits
476
477
478


_re_copy_warning = re.compile(r"^(\s*)#\s*Copied from\s+transformers\.(\S+\.\S+)\s*($|\S.*$)")
479
_re_copy_warning_for_test_file = re.compile(r"^(\s*)#\s*Copied from\s+tests\.(\S+\.\S+)\s*($|\S.*$)")
480
_re_replace_pattern = re.compile(r"^\s*(\S+)->(\S+)(\s+.*|$)")
481
_re_fill_pattern = re.compile(r"<FILL\s+[^>]*>")
482
483


Sylvain Gugger's avatar
Sylvain Gugger committed
484
485
486
487
488
489
490
491
492
493
def get_indent(code: str) -> str:
    """
    Find the indent in the first non empty line in a code sample.

    Args:
        code (`str`): The code to inspect.

    Returns:
        `str`: The indent looked at (as string).
    """
494
495
496
497
498
499
    lines = code.split("\n")
    idx = 0
    while idx < len(lines) and len(lines[idx]) == 0:
        idx += 1
    if idx < len(lines):
        return re.search(r"^(\s*)\S", lines[idx]).groups()[0]
500
501
502
    return ""


503
504
505
506
507
508
509
510
def run_ruff(code):
    command = ["ruff", "format", "-", "--config", "pyproject.toml", "--silent"]
    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
    stdout, _ = process.communicate(input=code.encode())
    return stdout.decode()


def stylify(code: str) -> str:
511
    """
512
513
    Applies the ruff part of our `make style` command to some code. This formats the code using `ruff format`.
    As `ruff` does not provide a python api this cannot be done on the fly.
Sylvain Gugger's avatar
Sylvain Gugger committed
514
515
516
517
518
519

    Args:
        code (`str`): The code to format.

    Returns:
        `str`: The formatted code.
520
521
522
523
    """
    has_indent = len(get_indent(code)) > 0
    if has_indent:
        code = f"class Bla:\n{code}"
524
525
    formatted_code = run_ruff(code)
    return formatted_code[len("class Bla:\n") :] if has_indent else formatted_code
526
527


Sylvain Gugger's avatar
Sylvain Gugger committed
528
def check_codes_match(observed_code: str, theoretical_code: str) -> Optional[int]:
529
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
530
531
532
533
534
535
536
537
538
    Checks if two version of a code match with the exception of the class/function name.

    Args:
        observed_code (`str`): The code found.
        theoretical_code (`str`): The code to match.

    Returns:
        `Optional[int]`: The index of the first line where there is a difference (if any) and `None` if the codes
        match.
539
540
541
542
    """
    observed_code_header = observed_code.split("\n")[0]
    theoretical_code_header = theoretical_code.split("\n")[0]

Sylvain Gugger's avatar
Sylvain Gugger committed
543
    # Catch the function/class name: it is expected that those do not match.
544
545
546
547
548
549
550
551
    _re_class_match = re.compile(r"class\s+([^\(:]+)(?:\(|:)")
    _re_func_match = re.compile(r"def\s+([^\(]+)\(")
    for re_pattern in [_re_class_match, _re_func_match]:
        if re_pattern.match(observed_code_header) is not None:
            observed_obj_name = re_pattern.search(observed_code_header).groups()[0]
            theoretical_name = re_pattern.search(theoretical_code_header).groups()[0]
            theoretical_code_header = theoretical_code_header.replace(theoretical_name, observed_obj_name)

Sylvain Gugger's avatar
Sylvain Gugger committed
552
    # Find the first diff. Line 0 is special since we need to compare with the function/class names ignored.
553
554
555
556
557
558
559
560
561
562
563
    diff_index = 0
    if theoretical_code_header != observed_code_header:
        return 0

    diff_index = 1
    for observed_line, theoretical_line in zip(observed_code.split("\n")[1:], theoretical_code.split("\n")[1:]):
        if observed_line != theoretical_line:
            return diff_index
        diff_index += 1


Yih-Dar's avatar
Yih-Dar committed
564
def is_copy_consistent(filename: str, overwrite: bool = False, buffer: dict = None) -> Optional[List[Tuple[str, int]]]:
565
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
566
    Check if the code commented as a copy in a file matches the original.
567

Sylvain Gugger's avatar
Sylvain Gugger committed
568
569
570
571
572
    Args:
        filename (`str`):
            The name of the file to check.
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
Yih-Dar's avatar
Yih-Dar committed
573
574
        buffer (`dict`, *optional*):
            The buffer used to store the previous results in order to speed up the process.
Sylvain Gugger's avatar
Sylvain Gugger committed
575
576
577
578

    Returns:
        `Optional[List[Tuple[str, int]]]`: If `overwrite=False`, returns the list of differences as tuples `(str, int)`
        with the name of the object having a diff and the line number where theere is the first diff.
579
    """
Yih-Dar's avatar
Yih-Dar committed
580
581
    base_path = TRANSFORMERS_PATH if not filename.startswith("tests") else MODEL_TEST_PATH

582
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
583
        lines = f.readlines()
584
    diffs = []
585
    line_index = 0
586
    # Not a for loop cause `lines` is going to change (if `overwrite=True`).
587
    while line_index < len(lines):
588
589
590
591
592
        search_re = _re_copy_warning
        if filename.startswith("tests"):
            search_re = _re_copy_warning_for_test_file

        search = search_re.search(lines[line_index])
593
594
595
596
597
598
        if search is None:
            line_index += 1
            continue

        # There is some copied code here, let's retrieve the original.
        indent, object_name, replace_pattern = search.groups()
599

Yih-Dar's avatar
Yih-Dar committed
600
601
602
603
604
605
606
607
608
609
610
611
612
        # Find the file lines, the object's code, and its blocks
        target_lines, theoretical_code, theoretical_code_splits = find_code_and_splits(
            object_name, base_path, buffer=buffer
        )

        # code replaced by the patterns
        theoretical_code_blocks = OrderedDict()
        for name, start, end in theoretical_code_splits:
            name = replace_code(name, replace_pattern)
            code = "".join(target_lines[start:end])
            code = replace_code(code, replace_pattern)
            theoretical_code_blocks[name] = code

613
        theoretical_indent = get_indent(theoretical_code)
614

Yih-Dar's avatar
Yih-Dar committed
615
616
        # `start_index` is the index of the first line (the definition header) after `# Copied from`.
        # (`indent != theoretical_indent` doesn't seem to occur so far, not sure what this case is for.)
617
        start_index = line_index + 1 if indent == theoretical_indent else line_index
Yih-Dar's avatar
Yih-Dar committed
618
        # enter the block body
619
        line_index = start_index + 1
620

621
622
        subcode = "\n".join(theoretical_code.split("\n")[1:])
        indent = get_indent(subcode)
623
        # Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
Yih-Dar's avatar
Yih-Dar committed
624
        # We can't call `find_block_end` directly as there is sth. special `# End copy"` here.
625
626
627
628
629
630
        should_continue = True
        while line_index < len(lines) and should_continue:
            line_index += 1
            if line_index >= len(lines):
                break
            line = lines[line_index]
631
632
            # There is a special pattern `# End copy` to stop early. It's not documented cause it shouldn't really be
            # used.
633
            should_continue = _should_continue(line, indent) and re.search(f"^{indent}# End copy", line) is None
Yih-Dar's avatar
Yih-Dar committed
634
        # `line_index` is outside the block
635
636
637
638
        # Clean up empty lines at the end (if any).
        while len(lines[line_index - 1]) <= 1:
            line_index -= 1

Yih-Dar's avatar
Yih-Dar committed
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
        # Split the observed code into blocks
        observed_code_splits = split_code_into_blocks(lines, start_index, line_index, len(indent), backtrace=True)

        is_class = lines[start_index].startswith(f"{' ' * (len(indent) - 4)}class ")
        # sanity check
        _sanity_check_splits(theoretical_code_splits, observed_code_splits, is_class=is_class)

        # observed code in a structured way (a dict mapping block names to blocks' code)
        observed_code_blocks = OrderedDict()
        for name, start, end in observed_code_splits:
            code = "".join(lines[start:end])
            observed_code_blocks[name] = code

        # Below, we change some names in `theoretical_code_blocks` and `observed_code_blocks`. These mappings map the
        # original names to the modified names: this is used to restore the original order of the code blocks.
        name_mappings_1 = {k: k for k in theoretical_code_blocks.keys()}
        name_mappings_2 = {k: k for k in observed_code_blocks.keys()}

        # Update code blocks' name and content:
        #   If `"# Ignore copy"` is found in a block of the observed code:
        #     1. if it's a block only in the observed code --> add it to the theoretical code.
        #     2. if it's also in the theoretical code () --> put its content (body) to the corresponding block under the
        #        same name in the theoretical code.
        #   In both cases, we change the name to have a prefix `_ignored_` so we know if we can discard them during the
        #   comparison.
        ignored_existing_block_index = 0
        ignored_new_block_index = 0
        for name in list(observed_code_blocks.keys()):
            code = observed_code_blocks[name]
            if "# Ignore copy" in code:
                if name in theoretical_code_blocks:
                    # in the target --> just copy the content
                    del theoretical_code_blocks[name]
                    theoretical_code_blocks[f"_ignored_existing_block_{ignored_existing_block_index}"] = code
                    name_mappings_1[name] = f"_ignored_existing_block_{ignored_existing_block_index}"

                    del observed_code_blocks[name]
                    observed_code_blocks[f"_ignored_existing_block_{ignored_existing_block_index}"] = code
                    name_mappings_2[name] = f"_ignored_existing_block_{ignored_existing_block_index}"
                    ignored_existing_block_index += 1
                else:
                    # not in the target --> add it
                    theoretical_code_blocks[f"_ignored_new_block_{ignored_new_block_index}"] = code
                    name_mappings_1[
                        f"_ignored_new_block_{ignored_new_block_index}"
                    ] = f"_ignored_new_block_{ignored_new_block_index}"

                    del observed_code_blocks[name]
                    observed_code_blocks[f"_ignored_new_block_{ignored_new_block_index}"] = code
                    name_mappings_2[name] = f"_ignored_new_block_{ignored_new_block_index}"
                    ignored_new_block_index += 1

        # Respect the original block order:
        #   1. in `theoretical_code_blocks`: the new blocks will follow the existing ones
        #   2. in `observed_code_blocks`: the original order are kept with names modified potentially. This is necessary
        #      to compute the correct `diff_index` if `overwrite=True` and there is a diff.
        theoretical_code_blocks = {
            name_mappings_1[orig_name]: theoretical_code_blocks[name_mappings_1[orig_name]]
            for orig_name in name_mappings_1
        }
        observed_code_blocks = {
            name_mappings_2[orig_name]: observed_code_blocks[name_mappings_2[orig_name]]
            for orig_name in name_mappings_2
        }

        # Ignore the blocks specified to be ignored. This is the version used to check if there is a mismatch
        theoretical_code_blocks_clean = {
            k: v
            for k, v in theoretical_code_blocks.items()
            if not (k.startswith(("_ignored_existing_block_", "_ignored_new_block_")))
        }
        theoretical_code = "".join(list(theoretical_code_blocks_clean.values()))

        # stylify `theoretical_code` before compare (this is needed only when `replace_pattern` is not empty)
        if replace_pattern:
714
            theoretical_code = stylify(theoretical_code)
Yih-Dar's avatar
Yih-Dar committed
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
        # Remove `\n\n` in `theoretical_code` before compare (so no empty line)
        while "\n\n" in theoretical_code:
            theoretical_code = theoretical_code.replace("\n\n", "\n")

        # Compute `observed_code` where we don't include any empty line + keep track the line index between the
        # original/processed `observed_code` so we can have the correct `diff_index`.
        idx_to_orig_idx_mapping_for_observed_code_lines = {}
        idx = -1
        orig_idx = -1
        observed_code = ""
        for name, code in observed_code_blocks.items():
            if code.endswith("\n"):
                code = code[:-1]
            for code_line in code.split("\n"):
                orig_idx += 1
                if code_line.strip() and not name.startswith(("_ignored_existing_block_", "_ignored_new_block_")):
                    idx += 1
                    observed_code += code_line + "\n"
                    idx_to_orig_idx_mapping_for_observed_code_lines[idx] = orig_idx
734

735
        # Test for a diff and act accordingly.
736
737
        diff_index = check_codes_match(observed_code, theoretical_code)
        if diff_index is not None:
Yih-Dar's avatar
Yih-Dar committed
738
739
            # switch to the index in the original `observed_code` (i.e. before removing empty lines)
            diff_index = idx_to_orig_idx_mapping_for_observed_code_lines[diff_index]
740
            diffs.append([object_name, diff_index + start_index + 1])
741
            if overwrite:
Yih-Dar's avatar
Yih-Dar committed
742
743
744
745
                # `theoretical_code_to_write` is a single string but may have several lines.
                theoretical_code_to_write = stylify("".join(list(theoretical_code_blocks.values())))
                lines = lines[:start_index] + [theoretical_code_to_write] + lines[line_index:]
                # Here we treat it as a single entry in `lines`.
746
747
                line_index = start_index + 1

748
    if overwrite and len(diffs) > 0:
749
750
        # Warn the user a file has been modified.
        print(f"Detected changes, rewriting {filename}.")
751
        with open(filename, "w", encoding="utf-8", newline="\n") as f:
752
            f.writelines(lines)
753
    return diffs
754
755


Yih-Dar's avatar
Yih-Dar committed
756
def check_copies(overwrite: bool = False, file: str = None):
757
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
758
    Check every file is copy-consistent with the original. Also check the model list in the main README and other
759
    READMEs are consistent.
Sylvain Gugger's avatar
Sylvain Gugger committed
760
761
762
763

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
Yih-Dar's avatar
Yih-Dar committed
764
765
        file (`bool`, *optional*):
            The path to a specific file to check and/or fix.
766
    """
Yih-Dar's avatar
Yih-Dar committed
767
768
769
770
771
772
773
774
    buffer = {}

    if file is None:
        all_files = glob.glob(os.path.join(TRANSFORMERS_PATH, "**/*.py"), recursive=True)
        all_test_files = glob.glob(os.path.join(MODEL_TEST_PATH, "**/*.py"), recursive=True)
        all_files = list(all_files) + list(all_test_files)
    else:
        all_files = [file]
775

776
777
    diffs = []
    for filename in all_files:
Yih-Dar's avatar
Yih-Dar committed
778
        new_diffs = is_copy_consistent(filename, overwrite, buffer)
779
        diffs += [f"- {filename}: copy does not match {d[0]} at line {d[1]}" for d in new_diffs]
780
781
782
    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
783
            "Found the following copy inconsistencies:\n"
784
            + diff
785
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."
786
        )
787
788
789
    check_model_list_copy(overwrite=overwrite)


790
def check_full_copies(overwrite: bool = False):
791
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
792
793
794
795
796
    Check the files that are full copies of others (as indicated in `FULL_COPIES`) are copy-consistent.

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
797
    """
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
    diffs = []
    for target, source in FULL_COPIES.items():
        with open(source, "r", encoding="utf-8") as f:
            source_code = f.read()
        with open(target, "r", encoding="utf-8") as f:
            target_code = f.read()
        if source_code != target_code:
            if overwrite:
                with open(target, "w", encoding="utf-8") as f:
                    print(f"Replacing the content of {target} by the one of {source}.")
                    f.write(source_code)
            else:
                diffs.append(f"- {target}: copy does not match {source}.")

    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
            "Found the following copy inconsistencies:\n"
            + diff
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."
        )


Sylvain Gugger's avatar
Sylvain Gugger committed
821
822
823
824
825
826
827
828
829
830
831
832
def get_model_list(filename: str, start_prompt: str, end_prompt: str) -> str:
    """
    Extracts the model list from a README.

    Args:
        filename (`str`): The name of the README file to check.
        start_prompt (`str`): The string to look for that introduces the model list.
        end_prompt (`str`): The string to look for that ends the model list.

    Returns:
        `str`: The model list.
    """
833
    with open(os.path.join(REPO_PATH, filename), "r", encoding="utf-8", newline="\n") as f:
834
835
836
        lines = f.readlines()
    # Find the start of the list.
    start_index = 0
837
    while not lines[start_index].startswith(start_prompt):
838
839
840
841
842
843
844
        start_index += 1
    start_index += 1

    result = []
    current_line = ""
    end_index = start_index

Sylvain Gugger's avatar
Sylvain Gugger committed
845
    # Keep going until the end of the list.
846
    while not lines[end_index].startswith(end_prompt):
847
848
849
850
851
852
853
854
855
856
857
858
859
        if lines[end_index].startswith("1."):
            if len(current_line) > 1:
                result.append(current_line)
            current_line = lines[end_index]
        elif len(lines[end_index]) > 1:
            current_line = f"{current_line[:-1]} {lines[end_index].lstrip()}"
        end_index += 1
    if len(current_line) > 1:
        result.append(current_line)

    return "".join(result)


Sylvain Gugger's avatar
Sylvain Gugger committed
860
def convert_to_localized_md(model_list: str, localized_model_list: str, format_str: str) -> Tuple[bool, str]:
861
862
863
864
865
866
867
868
869
870
871
872
873
874
    """
    Compare the model list from the main README to the one in a localized README.

    Args:
        model_list (`str`): The model list in the main README.
        localized_model_list (`str`): The model list in one of the localized README.
        format_str (`str`):
            The template for a model entry in the localized README (look at the `format_model_list` in the entries of
            `LOCALIZED_READMES` for examples).

    Returns:
        `Tuple[bool, str]`: A tuple where the first value indicates if the READMEs match or not, and the second value
        is the correct localized README.
    """
875
876
877
878
879
880
881
882
883
884
885
886
887

    def _rep(match):
        title, model_link, paper_affiliations, paper_title_link, paper_authors, supplements = match.groups()
        return format_str.format(
            title=title,
            model_link=model_link,
            paper_affiliations=paper_affiliations,
            paper_title_link=paper_title_link,
            paper_authors=paper_authors,
            supplements=" " + supplements.strip() if len(supplements) != 0 else "",
        )

    # This regex captures metadata from an English model description, including model title, model link,
888
889
    # affiliations of the paper, title of the paper, authors of the paper, and supplemental data (see DistilBERT for
    # example).
890
891
892
    _re_capture_meta = re.compile(
        r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\* \(from ([^)]*)\)[^\[]*([^\)]*\)).*?by (.*?[A-Za-z\*]{2,}?)\. (.*)$"
    )
893
894
    # This regex is used to synchronize link.
    _re_capture_title_link = re.compile(r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\*")
895
896
897
898
899
900
901
902
903
904
905
906

    if len(localized_model_list) == 0:
        localized_model_index = {}
    else:
        try:
            localized_model_index = {
                re.search(r"\*\*\[([^\]]*)", line).groups()[0]: line
                for line in localized_model_list.strip().split("\n")
            }
        except AttributeError:
            raise AttributeError("A model name in localized READMEs cannot be recognized.")

907
908
909
    model_keys = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in model_list.strip().split("\n")]

    # We exclude keys in localized README not in the main one.
910
    readmes_match = not any(k not in model_keys for k in localized_model_index)
911
912
    localized_model_index = {k: v for k, v in localized_model_index.items() if k in model_keys}

913
    for model in model_list.strip().split("\n"):
914
        title, model_link = _re_capture_title_link.search(model).groups()
915
        if title not in localized_model_index:
916
            readmes_match = False
917
918
            # Add an anchor white space behind a model description string for regex.
            # If metadata cannot be captured, the English version will be directly copied.
919
            localized_model_index[title] = _re_capture_meta.sub(_rep, model + " ")
920
921
922
923
924
        elif _re_fill_pattern.search(localized_model_index[title]) is not None:
            update = _re_capture_meta.sub(_rep, model + " ")
            if update != localized_model_index[title]:
                readmes_match = False
                localized_model_index[title] = update
925
926
927
928
929
        else:
            # Synchronize link
            localized_model_index[title] = _re_capture_title_link.sub(
                f"**[{title}]({model_link})**", localized_model_index[title], count=1
            )
930
931
932

    sorted_index = sorted(localized_model_index.items(), key=lambda x: x[0].lower())

933
    return readmes_match, "\n".join((x[1] for x in sorted_index)) + "\n"
934
935


Sylvain Gugger's avatar
Sylvain Gugger committed
936
def _find_text_in_file(filename: str, start_prompt: str, end_prompt: str) -> Tuple[str, int, int, List[str]]:
Sylvain Gugger's avatar
Sylvain Gugger committed
937
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
938
939
940
941
942
943
944
945
946
947
    Find the text in a file between two prompts.

    Args:
        filename (`str`): The name of the file to look into.
        start_prompt (`str`): The string to look for that introduces the content looked for.
        end_prompt (`str`): The string to look for that ends the content looked for.

    Returns:
        Tuple[str, int, int, List[str]]: The content between the two prompts, the index of the start line in the
        original file, the index of the end line in the original file and the list of lines of that file.
Sylvain Gugger's avatar
Sylvain Gugger committed
948
949
    """
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
950
        lines = f.readlines()
Sylvain Gugger's avatar
Sylvain Gugger committed
951
    # Find the start prompt.
952
    start_index = 0
Sylvain Gugger's avatar
Sylvain Gugger committed
953
    while not lines[start_index].startswith(start_prompt):
954
955
956
957
        start_index += 1
    start_index += 1

    end_index = start_index
Sylvain Gugger's avatar
Sylvain Gugger committed
958
    while not lines[end_index].startswith(end_prompt):
959
960
961
962
963
964
965
966
        end_index += 1
    end_index -= 1

    while len(lines[start_index]) <= 1:
        start_index += 1
    while len(lines[end_index]) <= 1:
        end_index -= 1
    end_index += 1
Sylvain Gugger's avatar
Sylvain Gugger committed
967
968
    return "".join(lines[start_index:end_index]), start_index, end_index, lines

969

Sylvain Gugger's avatar
Sylvain Gugger committed
970
def check_model_list_copy(overwrite: bool = False):
971
972
    """
    Check the model lists in the README is consistent with the ones in the other READMES and also with `index.nmd`.
Sylvain Gugger's avatar
Sylvain Gugger committed
973
974
975
976

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
977
    """
978
979
980
981
    # Fix potential doc links in the README
    with open(os.path.join(REPO_PATH, "README.md"), "r", encoding="utf-8", newline="\n") as f:
        readme = f.read()
    new_readme = readme.replace("https://huggingface.co/transformers", "https://huggingface.co/docs/transformers")
982
    new_readme = new_readme.replace(
983
984
985
986
987
988
989
990
991
992
993
        "https://huggingface.co/docs/main/transformers", "https://huggingface.co/docs/transformers/main"
    )
    if new_readme != readme:
        if overwrite:
            with open(os.path.join(REPO_PATH, "README.md"), "w", encoding="utf-8", newline="\n") as f:
                f.write(new_readme)
        else:
            raise ValueError(
                "The main README contains wrong links to the documentation of Transformers. Run `make fix-copies` to "
                "automatically fix them."
            )
994
995
996
997
998
999
1000

    md_list = get_model_list(
        filename="README.md",
        start_prompt=LOCALIZED_READMES["README.md"]["start_prompt"],
        end_prompt=LOCALIZED_READMES["README.md"]["end_prompt"],
    )

1001
    # Build the converted Markdown.
1002
1003
1004
1005
1006
1007
1008
    converted_md_lists = []
    for filename, value in LOCALIZED_READMES.items():
        _start_prompt = value["start_prompt"]
        _end_prompt = value["end_prompt"]
        _format_model_list = value["format_model_list"]

        localized_md_list = get_model_list(filename, _start_prompt, _end_prompt)
1009
        readmes_match, converted_md_list = convert_to_localized_md(md_list, localized_md_list, _format_model_list)
1010

1011
        converted_md_lists.append((filename, readmes_match, converted_md_list, _start_prompt, _end_prompt))
1012

Sylvain Gugger's avatar
Sylvain Gugger committed
1013
    # Compare the converted Markdowns
1014
    for converted_md_list in converted_md_lists:
1015
        filename, readmes_match, converted_md, _start_prompt, _end_prompt = converted_md_list
1016

1017
1018
1019
1020
1021
1022
1023
1024
        if filename == "README.md":
            continue
        if overwrite:
            _, start_index, end_index, lines = _find_text_in_file(
                filename=os.path.join(REPO_PATH, filename), start_prompt=_start_prompt, end_prompt=_end_prompt
            )
            with open(os.path.join(REPO_PATH, filename), "w", encoding="utf-8", newline="\n") as f:
                f.writelines(lines[:start_index] + [converted_md] + lines[end_index:])
1025
        elif not readmes_match:
1026
1027
1028
1029
            raise ValueError(
                f"The model list in the README changed and the list in `{filename}` has not been updated. Run "
                "`make fix-copies` to fix this."
            )
1030

Sylvain Gugger's avatar
Sylvain Gugger committed
1031

1032
# Map a model name with the name it has in the README for the check_readme check
1033
1034
1035
1036
1037
1038
SPECIAL_MODEL_NAMES = {
    "Bert Generation": "BERT For Sequence Generation",
    "BigBird": "BigBird-RoBERTa",
    "Data2VecAudio": "Data2Vec",
    "Data2VecText": "Data2Vec",
    "Data2VecVision": "Data2Vec",
1039
    "DonutSwin": "Swin Transformer",
1040
    "Marian": "MarianMT",
1041
    "MaskFormerSwin": "Swin Transformer",
1042
1043
1044
    "OpenAI GPT-2": "GPT-2",
    "OpenAI GPT": "GPT",
    "Perceiver": "Perceiver IO",
1045
    "SAM": "Segment Anything",
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
    "ViT": "Vision Transformer (ViT)",
}

# Update this list with the models that shouldn't be in the README. This only concerns modular models or those who do
# not have an associated paper.
MODELS_NOT_IN_README = [
    "BertJapanese",
    "Encoder decoder",
    "FairSeq Machine-Translation",
    "HerBERT",
    "RetriBERT",
    "Speech Encoder decoder",
    "Speech2Text",
    "Speech2Text2",
amyeroberts's avatar
amyeroberts committed
1060
    "TimmBackbone",
1061
1062
    "Vision Encoder decoder",
    "VisionTextDualEncoder",
1063
    "CLIPVisionModel",
1064
1065
]

1066
# Template for new entries to add in the main README when we have missing models.
1067
README_TEMPLATE = (
1068
1069
    "1. **[{model_name}](https://huggingface.co/docs/main/transformers/model_doc/{model_type})** (from "
    "<FILL INSTITUTION>) released with the paper [<FILL PAPER TITLE>](<FILL ARKIV LINK>) by <FILL AUTHORS>."
1070
1071
1072
)


Sylvain Gugger's avatar
Sylvain Gugger committed
1073
def check_readme(overwrite: bool = False):
1074
    """
Sylvain Gugger's avatar
Sylvain Gugger committed
1075
1076
1077
1078
1079
    Check if the main README contains all the models in the library or not.

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to add an entry for the missing models using `README_TEMPLATE`.
1080
    """
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
    info = LOCALIZED_READMES["README.md"]
    models, start_index, end_index, lines = _find_text_in_file(
        os.path.join(REPO_PATH, "README.md"),
        info["start_prompt"],
        info["end_prompt"],
    )
    models_in_readme = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in models.strip().split("\n")]

    model_names_mapping = transformers_module.models.auto.configuration_auto.MODEL_NAMES_MAPPING
    absents = [
        (key, name)
        for key, name in model_names_mapping.items()
        if SPECIAL_MODEL_NAMES.get(name, name) not in models_in_readme
    ]
    # Remove exceptions
    absents = [(key, name) for key, name in absents if name not in MODELS_NOT_IN_README]
    if len(absents) > 0 and not overwrite:
        print(absents)
        raise ValueError(
            "The main README doesn't contain all models, run `make fix-copies` to fill it with the missing model(s)"
            " then complete the generated entries.\nIf the model is not supposed to be in the main README, add it to"
            " the list `MODELS_NOT_IN_README` in utils/check_copies.py.\nIf it has a different name in the repo than"
            " in the README, map the correspondence in `SPECIAL_MODEL_NAMES` in utils/check_copies.py."
        )

    new_models = [README_TEMPLATE.format(model_name=name, model_type=key) for key, name in absents]

    all_models = models.strip().split("\n") + new_models
    all_models = sorted(all_models, key=lambda x: re.search(r"\*\*\[([^\]]*)", x).groups()[0].lower())
    all_models = "\n".join(all_models) + "\n"

    if all_models != models:
        if overwrite:
            print("Fixing the main README.")
            with open(os.path.join(REPO_PATH, "README.md"), "w", encoding="utf-8", newline="\n") as f:
                f.writelines(lines[:start_index] + [all_models] + lines[end_index:])
        else:
            raise ValueError("The main README model list is not properly sorted. Run `make fix-copies` to fix this.")


1121
1122
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
Yih-Dar's avatar
Yih-Dar committed
1123
    parser.add_argument("--file", type=str, default=None, help="A specific file to check and/or fix")
1124
1125
1126
    parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
    args = parser.parse_args()

1127
    check_readme(args.fix_and_overwrite)
Yih-Dar's avatar
Yih-Dar committed
1128
    check_copies(args.fix_and_overwrite, args.file)
1129
    check_full_copies(args.fix_and_overwrite)