check_repo.py 45.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
"""
Utility that performs several consistency checks on the repo. This includes:
- checking all models are properly defined in the __init__ of models/
- checking all models are in the main __init__
- checking all models are properly tested
- checking all object in the main __init__ are documented
- checking all models are in at least one auto class
- checking all the auto mapping are properly defined (no typos, importable)
- checking the list of deprecated models is up to date

Use from the root of the repo with (as used in `make repo-consistency`):

```bash
python utils/check_repo.py
```

It has no auto-fix mode.
"""
33
34
35
import inspect
import os
import re
36
import sys
Sylvain Gugger's avatar
Sylvain Gugger committed
37
import types
38
import warnings
39
from collections import OrderedDict
40
from difflib import get_close_matches
41
from pathlib import Path
Sylvain Gugger's avatar
Sylvain Gugger committed
42
from typing import List, Tuple
43

44
from transformers import is_flax_available, is_tf_available, is_torch_available
45
from transformers.models.auto import get_values
Yih-Dar's avatar
Yih-Dar committed
46
from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
47
48
49
50
from transformers.models.auto.feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING_NAMES
from transformers.models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING_NAMES
from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES
from transformers.models.auto.tokenization_auto import TOKENIZER_MAPPING_NAMES
51
from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import
52

53
54
55
56
57

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_repo.py
PATH_TO_TRANSFORMERS = "src/transformers"
PATH_TO_TESTS = "tests"
58
PATH_TO_DOC = "docs/source/en"
59

60
61
# Update this list with models that are supposed to be private.
PRIVATE_MODELS = [
Jongjyh's avatar
Jongjyh committed
62
    "AltRobertaModel",
63
    "DPRSpanPredictor",
Daniel Stancl's avatar
Daniel Stancl committed
64
    "LongT5Stack",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
65
    "RealmBertModel",
66
    "T5Stack",
67
    "MT5Stack",
68
    "UMT5Stack",
69
    "SwitchTransformersStack",
70
    "TFDPRSpanPredictor",
71
72
    "MaskFormerSwinModel",
    "MaskFormerSwinPreTrainedModel",
73
74
    "BridgeTowerTextModel",
    "BridgeTowerVisionModel",
75
76
]

77
78
# Update this list for models that are not tested with a comment explaining the reason it should not be.
# Being in this list is an exception and should **not** be the rule.
79
IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [
80
    # models to ignore for not tested
NielsRogge's avatar
NielsRogge committed
81
    "InstructBlipQFormerModel",  # Building part of bigger (tested) model.
82
    "UMT5EncoderModel",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
83
    "Blip2QFormerModel",  # Building part of bigger (tested) model.
84
    "ErnieMForInformationExtraction",
85
    "GraphormerDecoderHead",  # Building part of bigger (tested) model.
86
87
    "JukeboxVQVAE",  # Building part of bigger (tested) model.
    "JukeboxPrior",  # Building part of bigger (tested) model.
88
    "DecisionTransformerGPT2Model",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
89
    "SegformerDecodeHead",  # Building part of bigger (tested) model.
wangpeng's avatar
wangpeng committed
90
    "MgpstrModel",  # Building part of bigger (tested) model.
91
    "BertLMHeadModel",  # Needs to be setup as decoder.
92
    "MegatronBertLMHeadModel",  # Building part of bigger (tested) model.
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
93
94
95
96
    "RealmBertModel",  # Building part of bigger (tested) model.
    "RealmReader",  # Not regular model.
    "RealmScorer",  # Not regular model.
    "RealmForOpenQA",  # Not regular model.
97
98
99
    "ReformerForMaskedLM",  # Needs to be setup as decoder.
    "TFElectraMainLayer",  # Building part of bigger (tested) model (should it be a TFPreTrainedModel ?)
    "TFRobertaForMultipleChoice",  # TODO: fix
100
    "TFRobertaPreLayerNormForMultipleChoice",  # TODO: fix
abhishek thakur's avatar
abhishek thakur committed
101
    "SeparableConv1D",  # Building part of bigger (tested) model.
102
    "FlaxBartForCausalLM",  # Building part of bigger (tested) model.
103
    "FlaxBertForCausalLM",  # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM.
Younes Belkada's avatar
Younes Belkada committed
104
    "OPTDecoderWrapper",
105
    "TFSegformerDecodeHead",  # Not a regular model.
Jongjyh's avatar
Jongjyh committed
106
    "AltRobertaModel",  # Building part of bigger (tested) model.
Younes Belkada's avatar
Younes Belkada committed
107
    "BlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
Matt's avatar
Matt committed
108
    "TFBlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
109
110
    "BridgeTowerTextModel",  # No need to test it as it is tested by BridgeTowerModel model.
    "BridgeTowerVisionModel",  # No need to test it as it is tested by BridgeTowerModel model.
Yoach Lacombe's avatar
Yoach Lacombe committed
111
112
    "BarkCausalModel",  # Building part of bigger (tested) model.
    "BarkModel",  # Does not have a forward signature - generation tested with integration tests
113
114
115
116
117
]

# Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't
# trigger the common tests.
TEST_FILES_WITH_NO_COMMON_TESTS = [
Yih-Dar's avatar
Yih-Dar committed
118
119
120
121
122
123
124
125
126
127
128
129
130
    "models/decision_transformer/test_modeling_decision_transformer.py",
    "models/camembert/test_modeling_camembert.py",
    "models/mt5/test_modeling_flax_mt5.py",
    "models/mbart/test_modeling_mbart.py",
    "models/mt5/test_modeling_mt5.py",
    "models/pegasus/test_modeling_pegasus.py",
    "models/camembert/test_modeling_tf_camembert.py",
    "models/mt5/test_modeling_tf_mt5.py",
    "models/xlm_roberta/test_modeling_tf_xlm_roberta.py",
    "models/xlm_roberta/test_modeling_flax_xlm_roberta.py",
    "models/xlm_prophetnet/test_modeling_xlm_prophetnet.py",
    "models/xlm_roberta/test_modeling_xlm_roberta.py",
    "models/vision_text_dual_encoder/test_modeling_vision_text_dual_encoder.py",
Matt's avatar
Matt committed
131
    "models/vision_text_dual_encoder/test_modeling_tf_vision_text_dual_encoder.py",
Yih-Dar's avatar
Yih-Dar committed
132
133
    "models/vision_text_dual_encoder/test_modeling_flax_vision_text_dual_encoder.py",
    "models/decision_transformer/test_modeling_decision_transformer.py",
Yoach Lacombe's avatar
Yoach Lacombe committed
134
    "models/bark/test_modeling_bark.py",
135
136
]

137
138
# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and
# should **not** be the rule.
139
IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [
140
    # models to ignore for model xxx mapping
141
142
    "AlignTextModel",
    "AlignVisionModel",
143
144
145
146
    "ClapTextModel",
    "ClapTextModelWithProjection",
    "ClapAudioModel",
    "ClapAudioModelWithProjection",
NielsRogge's avatar
NielsRogge committed
147
148
149
    "Blip2ForConditionalGeneration",
    "Blip2QFormerModel",
    "Blip2VisionModel",
150
    "ErnieMForInformationExtraction",
151
    "GitVisionModel",
152
153
    "GraphormerModel",
    "GraphormerForGraphClassification",
Younes Belkada's avatar
Younes Belkada committed
154
155
156
157
158
159
    "BlipForConditionalGeneration",
    "BlipForImageTextRetrieval",
    "BlipForQuestionAnswering",
    "BlipVisionModel",
    "BlipTextLMHeadModel",
    "BlipTextModel",
Matt's avatar
Matt committed
160
161
162
163
164
165
    "TFBlipForConditionalGeneration",
    "TFBlipForImageTextRetrieval",
    "TFBlipForQuestionAnswering",
    "TFBlipVisionModel",
    "TFBlipTextLMHeadModel",
    "TFBlipTextModel",
NielsRogge's avatar
NielsRogge committed
166
    "Swin2SRForImageSuperResolution",
167
168
    "BridgeTowerForImageAndTextRetrieval",
    "BridgeTowerForMaskedLM",
169
    "BridgeTowerForContrastiveLearning",
NielsRogge's avatar
NielsRogge committed
170
171
172
    "CLIPSegForImageSegmentation",
    "CLIPSegVisionModel",
    "CLIPSegTextModel",
Matt's avatar
Matt committed
173
    "EsmForProteinFolding",
174
    "GPTSanJapaneseModel",
175
    "TimeSeriesTransformerForPrediction",
176
    "InformerForPrediction",
177
    "AutoformerForPrediction",
178
179
    "JukeboxVQVAE",
    "JukeboxPrior",
180
    "SamModel",
NielsRogge's avatar
NielsRogge committed
181
    "DPTForDepthEstimation",
182
    "DecisionTransformerGPT2Model",
NielsRogge's avatar
NielsRogge committed
183
    "GLPNForDepthEstimation",
NielsRogge's avatar
NielsRogge committed
184
185
    "ViltForImagesAndTextClassification",
    "ViltForImageAndTextRetrieval",
186
    "ViltForTokenClassification",
NielsRogge's avatar
NielsRogge committed
187
    "ViltForMaskedLM",
NielsRogge's avatar
NielsRogge committed
188
189
    "PerceiverForMultimodalAutoencoding",
    "PerceiverForOpticalFlow",
NielsRogge's avatar
NielsRogge committed
190
    "SegformerDecodeHead",
191
    "TFSegformerDecodeHead",
Kamal Raj's avatar
Kamal Raj committed
192
    "FlaxBeitForMaskedImageModeling",
NielsRogge's avatar
NielsRogge committed
193
    "BeitForMaskedImageModeling",
194
195
    "ChineseCLIPTextModel",
    "ChineseCLIPVisionModel",
Suraj Patil's avatar
Suraj Patil committed
196
    "CLIPTextModel",
197
    "CLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
198
    "CLIPVisionModel",
199
    "CLIPVisionModelWithProjection",
200
201
    "GroupViTTextModel",
    "GroupViTVisionModel",
Yih-Dar's avatar
Yih-Dar committed
202
203
    "TFCLIPTextModel",
    "TFCLIPVisionModel",
204
205
    "TFGroupViTTextModel",
    "TFGroupViTVisionModel",
Suraj Patil's avatar
Suraj Patil committed
206
207
    "FlaxCLIPTextModel",
    "FlaxCLIPVisionModel",
208
    "FlaxWav2Vec2ForCTC",
NielsRogge's avatar
NielsRogge committed
209
    "DetrForSegmentation",
Younes Belkada's avatar
Younes Belkada committed
210
211
212
    "Pix2StructVisionModel",
    "Pix2StructTextModel",
    "Pix2StructForConditionalGeneration",
213
    "ConditionalDetrForSegmentation",
214
215
    "DPRReader",
    "FlaubertForQuestionAnswering",
216
217
218
219
    "FlavaImageCodebook",
    "FlavaTextModel",
    "FlavaImageModel",
    "FlavaMultimodalModel",
220
    "GPT2DoubleHeadsModel",
221
    "GPTSw3DoubleHeadsModel",
NielsRogge's avatar
NielsRogge committed
222
223
    "InstructBlipVisionModel",
    "InstructBlipQFormerModel",
224
    "LayoutLMForQuestionAnswering",
Ryokan RI's avatar
Ryokan RI committed
225
    "LukeForMaskedLM",
NielsRogge's avatar
NielsRogge committed
226
227
228
    "LukeForEntityClassification",
    "LukeForEntityPairClassification",
    "LukeForEntitySpanClassification",
wangpeng's avatar
wangpeng committed
229
    "MgpstrModel",
230
    "OpenAIGPTDoubleHeadsModel",
231
232
233
    "OwlViTTextModel",
    "OwlViTVisionModel",
    "OwlViTForObjectDetection",
234
235
236
    "RagModel",
    "RagSequenceForGeneration",
    "RagTokenForGeneration",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
237
238
239
240
    "RealmEmbedder",
    "RealmForOpenQA",
    "RealmScorer",
    "RealmReader",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
241
    "TFDPRReader",
242
    "TFGPT2DoubleHeadsModel",
243
    "TFLayoutLMForQuestionAnswering",
244
    "TFOpenAIGPTDoubleHeadsModel",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
245
246
247
    "TFRagModel",
    "TFRagSequenceForGeneration",
    "TFRagTokenForGeneration",
248
    "Wav2Vec2ForCTC",
Patrick von Platen's avatar
Patrick von Platen committed
249
    "HubertForCTC",
250
251
    "SEWForCTC",
    "SEWDForCTC",
252
253
    "XLMForQuestionAnswering",
    "XLNetForQuestionAnswering",
abhishek thakur's avatar
abhishek thakur committed
254
    "SeparableConv1D",
Gunjan Chhablani's avatar
Gunjan Chhablani committed
255
256
257
258
    "VisualBertForRegionToPhraseAlignment",
    "VisualBertForVisualReasoning",
    "VisualBertForQuestionAnswering",
    "VisualBertForMultipleChoice",
Will Rice's avatar
Will Rice committed
259
    "TFWav2Vec2ForCTC",
Will Rice's avatar
Will Rice committed
260
    "TFHubertForCTC",
NielsRogge's avatar
NielsRogge committed
261
262
    "XCLIPVisionModel",
    "XCLIPTextModel",
Jongjyh's avatar
Jongjyh committed
263
264
265
    "AltCLIPTextModel",
    "AltCLIPVisionModel",
    "AltRobertaModel",
Zineng Tang's avatar
Zineng Tang committed
266
    "TvltForAudioVisualClassification",
Yoach Lacombe's avatar
Yoach Lacombe committed
267
268
269
270
271
272
    "BarkCausalModel",
    "BarkCoarseModel",
    "BarkFineModel",
    "BarkSemanticModel",
    "MusicgenModel",
    "MusicgenForConditionalGeneration",
273
274
275
    "SpeechT5ForSpeechToSpeech",
    "SpeechT5ForTextToSpeech",
    "SpeechT5HifiGan",
276
277
]

278
# DO NOT edit this list!
Sylvain Gugger's avatar
Sylvain Gugger committed
279
# (The corresponding pytorch objects should never have been in the main `__init__`, but it's too late to remove)
280
281
282
283
284
285
286
287
288
289
290
291
292
OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK = [
    "FlaxBertLayer",
    "FlaxBigBirdLayer",
    "FlaxRoFormerLayer",
    "TFBertLayer",
    "TFLxmertEncoder",
    "TFLxmertXLayer",
    "TFMPNetLayer",
    "TFMobileBertLayer",
    "TFSegformerLayer",
    "TFViTMAELayer",
]

Sylvain Gugger's avatar
Sylvain Gugger committed
293
# Update this list for models that have multiple model types for the same model doc.
294
295
296
297
MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(
    [
        ("data2vec-text", "data2vec"),
        ("data2vec-audio", "data2vec"),
298
        ("data2vec-vision", "data2vec"),
NielsRogge's avatar
NielsRogge committed
299
        ("donut-swin", "donut"),
300
301
302
303
    ]
)


304
# This is to make sure the transformers module imported is the one in the repo.
305
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
306
307


308
def check_missing_backends():
Sylvain Gugger's avatar
Sylvain Gugger committed
309
310
311
312
    """
    Checks if all backends are installed (otherwise the check of this script is incomplete). Will error in the CI if
    that's not the case but only throw a warning for users running this.
    """
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
    missing_backends = []
    if not is_torch_available():
        missing_backends.append("PyTorch")
    if not is_tf_available():
        missing_backends.append("TensorFlow")
    if not is_flax_available():
        missing_backends.append("Flax")
    if len(missing_backends) > 0:
        missing = ", ".join(missing_backends)
        if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES:
            raise Exception(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}."
            )
        else:
            warnings.warn(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you "
                "didn't make any change in one of those backends modeling files, you should probably execute the "
                "command above to be on the safe side."
            )


336
def check_model_list():
Sylvain Gugger's avatar
Sylvain Gugger committed
337
338
339
    """
    Checks the model listed as subfolders of `models` match the models available in `transformers.models`.
    """
340
341
342
343
    # Get the models from the directory structure of `src/transformers/models/`
    models_dir = os.path.join(PATH_TO_TRANSFORMERS, "models")
    _models = []
    for model in os.listdir(models_dir):
Sylvain Gugger's avatar
Sylvain Gugger committed
344
345
        if model == "deprecated":
            continue
346
347
348
349
        model_dir = os.path.join(models_dir, model)
        if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir):
            _models.append(model)

Sylvain Gugger's avatar
Sylvain Gugger committed
350
    # Get the models in the submodule `transformers.models`
351
352
    models = [model for model in dir(transformers.models) if not model.startswith("__")]

353
    missing_models = sorted(set(_models).difference(models))
354
355
356
357
358
359
    if missing_models:
        raise Exception(
            f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}."
        )


360
361
# If some modeling modules should be ignored for all checks, they should be added in the nested list
# _ignore_modules of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
362
363
def get_model_modules() -> List[str]:
    """Get all the model modules inside the transformers library (except deprecated models)."""
364
365
366
367
368
369
370
371
    _ignore_modules = [
        "modeling_auto",
        "modeling_encoder_decoder",
        "modeling_marian",
        "modeling_mmbt",
        "modeling_outputs",
        "modeling_retribert",
        "modeling_utils",
Sylvain Gugger's avatar
Sylvain Gugger committed
372
        "modeling_flax_auto",
373
        "modeling_flax_encoder_decoder",
Stas Bekman's avatar
Stas Bekman committed
374
        "modeling_flax_utils",
375
        "modeling_speech_encoder_decoder",
376
        "modeling_flax_speech_encoder_decoder",
377
        "modeling_flax_vision_encoder_decoder",
amyeroberts's avatar
amyeroberts committed
378
        "modeling_timm_backbone",
379
380
        "modeling_transfo_xl_utilities",
        "modeling_tf_auto",
381
        "modeling_tf_encoder_decoder",
382
383
384
385
        "modeling_tf_outputs",
        "modeling_tf_pytorch_utils",
        "modeling_tf_utils",
        "modeling_tf_transfo_xl_utilities",
386
        "modeling_tf_vision_encoder_decoder",
387
        "modeling_vision_encoder_decoder",
388
389
    ]
    modules = []
Sylvain Gugger's avatar
Sylvain Gugger committed
390
391
    for model in dir(transformers.models):
        # There are some magic dunder attributes in the dir, we ignore them
Sylvain Gugger's avatar
Sylvain Gugger committed
392
393
394
395
396
397
398
399
400
        if model == "deprecated" or model.startswith("__"):
            continue

        model_module = getattr(transformers.models, model)
        for submodule in dir(model_module):
            if submodule.startswith("modeling") and submodule not in _ignore_modules:
                modeling_module = getattr(model_module, submodule)
                if inspect.ismodule(modeling_module):
                    modules.append(modeling_module)
401
402
403
    return modules


Sylvain Gugger's avatar
Sylvain Gugger committed
404
405
406
407
408
409
410
411
412
413
414
415
416
def get_models(module: types.ModuleType, include_pretrained: bool = False) -> List[Tuple[str, type]]:
    """
    Get the objects in a module that are models.

    Args:
        module (`types.ModuleType`):
            The module from which we are extracting models.
        include_pretrained (`bool`, *optional*, defaults to `False`):
            Whether or not to include the `PreTrainedModel` subclass (like `BertPreTrainedModel`) or not.

    Returns:
        List[Tuple[str, type]]: List of models as tuples (class name, actual class).
    """
417
    models = []
418
    model_classes = (transformers.PreTrainedModel, transformers.TFPreTrainedModel, transformers.FlaxPreTrainedModel)
419
    for attr_name in dir(module):
420
        if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name):
421
422
423
424
425
426
427
            continue
        attr = getattr(module, attr_name)
        if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__:
            models.append((attr_name, attr))
    return models


Sylvain Gugger's avatar
Sylvain Gugger committed
428
429
430
431
def is_building_block(model: str) -> bool:
    """
    Returns `True` if a model is a building block part of a bigger model.
    """
432
433
434
435
436
437
    if model.endswith("Wrapper"):
        return True
    if model.endswith("Encoder"):
        return True
    if model.endswith("Decoder"):
        return True
438
439
    if model.endswith("Prenet"):
        return True
Sylvain Gugger's avatar
Sylvain Gugger committed
440
441
442
443
444
445
446


def is_a_private_model(model: str) -> bool:
    """Returns `True` if the model should not be in the main init."""
    if model in PRIVATE_MODELS:
        return True
    return is_building_block(model)
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463


def check_models_are_in_init():
    """Checks all models defined in the library are in the main init."""
    models_not_in_init = []
    dir_transformers = dir(transformers)
    for module in get_model_modules():
        models_not_in_init += [
            model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers
        ]

    # Remove private models
    models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)]
    if len(models_not_in_init) > 0:
        raise Exception(f"The following models should be in the main init: {','.join(models_not_in_init)}.")


464
465
# If some test_modeling files should be ignored when checking models are all tested, they should be added in the
# nested list _ignore_files of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
466
467
468
def get_model_test_files() -> List[str]:
    """
    Get the model test files.
Yih-Dar's avatar
Yih-Dar committed
469

Sylvain Gugger's avatar
Sylvain Gugger committed
470
471
472
473
    Returns:
        `List[str]`: The list of test files. The returned files will NOT contain the `tests` (i.e. `PATH_TO_TESTS`
        defined in this script). They will be considered as paths relative to `tests`. A caller has to use
        `os.path.join(PATH_TO_TESTS, ...)` to access the files.
Yih-Dar's avatar
Yih-Dar committed
474
475
    """

476
477
478
    _ignore_files = [
        "test_modeling_common",
        "test_modeling_encoder_decoder",
479
        "test_modeling_flax_encoder_decoder",
480
        "test_modeling_flax_speech_encoder_decoder",
481
482
        "test_modeling_marian",
        "test_modeling_tf_common",
483
        "test_modeling_tf_encoder_decoder",
484
485
    ]
    test_files = []
Yih-Dar's avatar
Yih-Dar committed
486
487
488
489
490
491
492
493
494
495
496
497
    model_test_root = os.path.join(PATH_TO_TESTS, "models")
    model_test_dirs = []
    for x in os.listdir(model_test_root):
        x = os.path.join(model_test_root, x)
        if os.path.isdir(x):
            model_test_dirs.append(x)

    for target_dir in [PATH_TO_TESTS] + model_test_dirs:
        for file_or_dir in os.listdir(target_dir):
            path = os.path.join(target_dir, file_or_dir)
            if os.path.isfile(path):
                filename = os.path.split(path)[-1]
498
                if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files:
Yih-Dar's avatar
Yih-Dar committed
499
500
501
                    file = os.path.join(*path.split(os.sep)[1:])
                    test_files.append(file)

502
503
504
505
506
    return test_files


# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class
# for the all_model_classes variable.
Sylvain Gugger's avatar
Sylvain Gugger committed
507
508
509
510
511
512
513
514
515
516
517
def find_tested_models(test_file: str) -> List[str]:
    """
    Parse the content of test_file to detect what's in `all_model_classes`. This detects the models that inherit from
    the common test class.

    Args:
        test_file (`str`): The path to the test file to check

    Returns:
        `List[str]`: The list of models tested in that file.
    """
518
    with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f:
519
        content = f.read()
Sylvain Gugger's avatar
Sylvain Gugger committed
520
    all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content)
521
522
    # Check with one less parenthesis as well
    all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
523
    if len(all_models) > 0:
524
        model_tested = []
Sylvain Gugger's avatar
Sylvain Gugger committed
525
526
527
528
529
        for entry in all_models:
            for line in entry.split(","):
                name = line.strip()
                if len(name) > 0:
                    model_tested.append(name)
530
531
532
        return model_tested


Sylvain Gugger's avatar
Sylvain Gugger committed
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
def should_be_tested(model_name: str) -> bool:
    """
    Whether or not a model should be tested.
    """
    if model_name in IGNORE_NON_TESTED:
        return False
    return not is_building_block(model_name)


def check_models_are_tested(module: types.ModuleType, test_file: str) -> List[str]:
    """Check models defined in a module are all tested in a given file.

    Args:
        module (`types.ModuleType`): The module in which we get the models.
        test_file (`str`): The path to the file where the module is tested.

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
552
    # XxxPreTrainedModel are not tested
553
554
555
    defined_models = get_models(module)
    tested_models = find_tested_models(test_file)
    if tested_models is None:
556
        if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS:
557
558
559
560
561
562
563
564
            return
        return [
            f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. "
            + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file "
            + "`utils/check_repo.py`."
        ]
    failures = []
    for model_name, _ in defined_models:
Sylvain Gugger's avatar
Sylvain Gugger committed
565
        if model_name not in tested_models and should_be_tested(model_name):
566
567
568
569
570
571
572
573
574
575
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not tested in "
                + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file."
                + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`"
                + "in the file `utils/check_repo.py`."
            )
    return failures


def check_all_models_are_tested():
Patrick von Platen's avatar
Patrick von Platen committed
576
    """Check all models are properly tested."""
577
578
579
580
    modules = get_model_modules()
    test_files = get_model_test_files()
    failures = []
    for module in modules:
Sylvain Gugger's avatar
Sylvain Gugger committed
581
        # Matches a module to its test file.
582
583
        test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file]
        if len(test_file) == 0:
584
            failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")
585
586
587
588
        elif len(test_file) > 1:
            failures.append(f"{module.__name__} has several test files: {test_file}.")
        else:
            test_file = test_file[0]
589
590
591
            new_failures = check_models_are_tested(module, test_file)
            if new_failures is not None:
                failures += new_failures
592
593
594
595
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
596
def get_all_auto_configured_models() -> List[str]:
Patrick von Platen's avatar
Patrick von Platen committed
597
    """Return the list of all models in at least one auto class."""
598
    result = set()  # To avoid duplicates we concatenate all model classes in a set.
599
600
    if is_torch_available():
        for attr_name in dir(transformers.models.auto.modeling_auto):
601
            if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"):
602
603
604
                result = result | set(get_values(getattr(transformers.models.auto.modeling_auto, attr_name)))
    if is_tf_available():
        for attr_name in dir(transformers.models.auto.modeling_tf_auto):
605
            if attr_name.startswith("TF_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
606
607
608
                result = result | set(get_values(getattr(transformers.models.auto.modeling_tf_auto, attr_name)))
    if is_flax_available():
        for attr_name in dir(transformers.models.auto.modeling_flax_auto):
609
            if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
610
                result = result | set(get_values(getattr(transformers.models.auto.modeling_flax_auto, attr_name)))
611
    return list(result)
612
613


Sylvain Gugger's avatar
Sylvain Gugger committed
614
615
def ignore_unautoclassed(model_name: str) -> bool:
    """Rules to determine if a model should be in an auto class."""
616
617
618
619
620
621
622
623
624
    # Special white list
    if model_name in IGNORE_NON_AUTO_CONFIGURED:
        return True
    # Encoder and Decoder should be ignored
    if "Encoder" in model_name or "Decoder" in model_name:
        return True
    return False


Sylvain Gugger's avatar
Sylvain Gugger committed
625
626
627
628
629
630
631
632
633
634
635
636
637
def check_models_are_auto_configured(module: types.ModuleType, all_auto_models: List[str]) -> List[str]:
    """
    Check models defined in module are each in an auto class.

    Args:
        module (`types.ModuleType`):
            The module in which we get the models.
        all_auto_models (`List[str]`):
            The list of all models in an auto class (as obtained with `get_all_auto_configured_models()`).

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
638
639
640
    defined_models = get_models(module)
    failures = []
    for model_name, _ in defined_models:
641
        if model_name not in all_auto_models and not ignore_unautoclassed(model_name):
642
643
644
645
646
647
648
649
650
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "
                "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "
                "`utils/check_repo.py`."
            )
    return failures


def check_all_models_are_auto_configured():
Patrick von Platen's avatar
Patrick von Platen committed
651
    """Check all models are each in an auto class."""
Sylvain Gugger's avatar
Sylvain Gugger committed
652
    # This is where we need to check we have all backends or the check is incomplete.
653
    check_missing_backends()
654
655
656
657
658
659
660
661
662
663
664
    modules = get_model_modules()
    all_auto_models = get_all_auto_configured_models()
    failures = []
    for module in modules:
        new_failures = check_models_are_auto_configured(module, all_auto_models)
        if new_failures is not None:
            failures += new_failures
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


665
666
def check_all_auto_object_names_being_defined():
    """Check all names defined in auto (name) mappings exist in the library."""
Sylvain Gugger's avatar
Sylvain Gugger committed
667
    # This is where we need to check we have all backends or the check is incomplete.
668
    check_missing_backends()
669

670
    failures = []
671
    mappings_to_check = {
672
673
674
675
676
677
        "TOKENIZER_MAPPING_NAMES": TOKENIZER_MAPPING_NAMES,
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

678
679
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
680
681
682
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
683
684
685
686
687
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
688
        for _, class_names in mapping.items():
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
            if not isinstance(class_names, tuple):
                class_names = (class_names,)
                for class_name in class_names:
                    if class_name is None:
                        continue
                    # dummy object is accepted
                    if not hasattr(transformers, class_name):
                        # If the class name is in a model name mapping, let's not check if there is a definition in any modeling
                        # module, if it's a private model defined in this file.
                        if name.endswith("MODEL_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
                        failures.append(
                            f"`{class_name}` appears in the mapping `{name}` but it is not defined in the library."
                        )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Yih-Dar's avatar
Yih-Dar committed
707
708
def check_all_auto_mapping_names_in_config_mapping_names():
    """Check all keys defined in auto mappings (mappings of names) appear in `CONFIG_MAPPING_NAMES`."""
Sylvain Gugger's avatar
Sylvain Gugger committed
709
    # This is where we need to check we have all backends or the check is incomplete.
710
    check_missing_backends()
Yih-Dar's avatar
Yih-Dar committed
711

712
    failures = []
Yih-Dar's avatar
Yih-Dar committed
713
    # `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer` is special, and don't need to follow the rule.
714
    mappings_to_check = {
Yih-Dar's avatar
Yih-Dar committed
715
716
717
718
719
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

720
721
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
722
723
724
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
725
726
727
728
729
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
730
        for model_type in mapping:
Yih-Dar's avatar
Yih-Dar committed
731
732
733
734
735
736
737
738
739
            if model_type not in CONFIG_MAPPING_NAMES:
                failures.append(
                    f"`{model_type}` appears in the mapping `{name}` but it is not defined in the keys of "
                    "`CONFIG_MAPPING_NAMES`."
                )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


740
def check_all_auto_mappings_importable():
Sylvain Gugger's avatar
Sylvain Gugger committed
741
742
    """Check all auto mappings can be imported."""
    # This is where we need to check we have all backends or the check is incomplete.
743
744
745
746
747
748
749
750
751
752
753
754
755
    check_missing_backends()

    failures = []
    mappings_to_check = {}
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

Sylvain Gugger's avatar
Sylvain Gugger committed
756
    for name in mappings_to_check:
757
758
        name = name.replace("_MAPPING_NAMES", "_MAPPING")
        if not hasattr(transformers, name):
759
760
761
762
763
764
            failures.append(f"`{name}`")
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


def check_objects_being_equally_in_main_init():
Sylvain Gugger's avatar
Sylvain Gugger committed
765
766
767
    """
    Check if a (TensorFlow or Flax) object is in the main __init__ iif its counterpart in PyTorch is.
    """
768
769
770
771
772
    attrs = dir(transformers)

    failures = []
    for attr in attrs:
        obj = getattr(transformers, attr)
Sylvain Gugger's avatar
Sylvain Gugger committed
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
        if not hasattr(obj, "__module__") or "models.deprecated" in obj.__module__:
            continue

        module_path = obj.__module__
        module_name = module_path.split(".")[-1]
        module_dir = ".".join(module_path.split(".")[:-1])
        if (
            module_name.startswith("modeling_")
            and not module_name.startswith("modeling_tf_")
            and not module_name.startswith("modeling_flax_")
        ):
            parent_module = sys.modules[module_dir]

            frameworks = []
            if is_tf_available():
                frameworks.append("TF")
            if is_flax_available():
                frameworks.append("Flax")

            for framework in frameworks:
                other_module_path = module_path.replace("modeling_", f"modeling_{framework.lower()}_")
                if os.path.isfile("src/" + other_module_path.replace(".", "/") + ".py"):
                    other_module_name = module_name.replace("modeling_", f"modeling_{framework.lower()}_")
                    other_module = getattr(parent_module, other_module_name)
                    if hasattr(other_module, f"{framework}{attr}"):
                        if not hasattr(transformers, f"{framework}{attr}"):
                            if f"{framework}{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}{attr}")
                    if hasattr(other_module, f"{framework}_{attr}"):
                        if not hasattr(transformers, f"{framework}_{attr}"):
                            if f"{framework}_{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}_{attr}")
805
806
807
808
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
809
810
811
_re_decorator = re.compile(r"^\s*@(\S+)\s+$")


Sylvain Gugger's avatar
Sylvain Gugger committed
812
813
814
815
816
817
818
819
820
821
def check_decorator_order(filename: str) -> List[int]:
    """
    Check that in a given test file, the slow decorator is always last.

    Args:
        filename (`str`): The path to a test file to check.

    Returns:
        `List[int]`: The list of failures as a list of indices where there are problems.
    """
822
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
Sylvain Gugger's avatar
Sylvain Gugger committed
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
        lines = f.readlines()
    decorator_before = None
    errors = []
    for i, line in enumerate(lines):
        search = _re_decorator.search(line)
        if search is not None:
            decorator_name = search.groups()[0]
            if decorator_before is not None and decorator_name.startswith("parameterized"):
                errors.append(i)
            decorator_before = decorator_name
        elif decorator_before is not None:
            decorator_before = None
    return errors


def check_all_decorator_order():
Patrick von Platen's avatar
Patrick von Platen committed
839
    """Check that in all test files, the slow decorator is always last."""
Sylvain Gugger's avatar
Sylvain Gugger committed
840
841
842
843
844
845
846
847
848
    errors = []
    for fname in os.listdir(PATH_TO_TESTS):
        if fname.endswith(".py"):
            filename = os.path.join(PATH_TO_TESTS, fname)
            new_errors = check_decorator_order(filename)
            errors += [f"- {filename}, line {i}" for i in new_errors]
    if len(errors) > 0:
        msg = "\n".join(errors)
        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
849
850
            "The parameterized decorator (and its variants) should always be first, but this is not the case in the"
            f" following files:\n{msg}"
Sylvain Gugger's avatar
Sylvain Gugger committed
851
852
853
        )


Sylvain Gugger's avatar
Sylvain Gugger committed
854
855
856
857
858
859
860
def find_all_documented_objects() -> List[str]:
    """
    Parse the content of all doc files to detect which classes and functions it documents.

    Returns:
        `List[str]`: The list of all object names being documented.
    """
861
862
    documented_obj = []
    for doc_file in Path(PATH_TO_DOC).glob("**/*.rst"):
Julien Plu's avatar
Julien Plu committed
863
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
864
865
866
            content = f.read()
        raw_doc_objs = re.findall(r"(?:autoclass|autofunction):: transformers.(\S+)\s+", content)
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
867
    for doc_file in Path(PATH_TO_DOC).glob("**/*.md"):
Sylvain Gugger's avatar
Sylvain Gugger committed
868
869
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
            content = f.read()
870
        raw_doc_objs = re.findall(r"\[\[autodoc\]\]\s+(\S+)\s+", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
871
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
872
873
874
875
876
877
    return documented_obj


# One good reason for not being documented is to be deprecated. Put in this list deprecated objects.
DEPRECATED_OBJECTS = [
    "AutoModelWithLMHead",
878
    "BartPretrainedModel",
879
880
    "DataCollator",
    "DataCollatorForSOP",
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
    "GlueDataset",
    "GlueDataTrainingArguments",
    "LineByLineTextDataset",
    "LineByLineWithRefDataset",
    "LineByLineWithSOPTextDataset",
    "PretrainedBartModel",
    "PretrainedFSMTModel",
    "SingleSentenceClassificationProcessor",
    "SquadDataTrainingArguments",
    "SquadDataset",
    "SquadExample",
    "SquadFeatures",
    "SquadV1Processor",
    "SquadV2Processor",
    "TFAutoModelWithLMHead",
896
    "TFBartPretrainedModel",
897
898
    "TextDataset",
    "TextDatasetForNextSentencePrediction",
899
    "Wav2Vec2ForMaskedLM",
900
    "Wav2Vec2Tokenizer",
901
902
903
904
905
906
907
908
909
910
    "glue_compute_metrics",
    "glue_convert_examples_to_features",
    "glue_output_modes",
    "glue_processors",
    "glue_tasks_num_labels",
    "squad_convert_examples_to_features",
    "xnli_compute_metrics",
    "xnli_output_modes",
    "xnli_processors",
    "xnli_tasks_num_labels",
911
912
    "TFTrainer",
    "TFTrainingArguments",
913
914
915
916
917
918
919
]

# Exceptionally, some objects should not be documented after all rules passed.
# ONLY PUT SOMETHING IN THIS LIST AS A LAST RESORT!
UNDOCUMENTED_OBJECTS = [
    "AddedToken",  # This is a tokenizers class.
    "BasicTokenizer",  # Internal, should never have been in the main init.
920
    "CharacterTokenizer",  # Internal, should never have been in the main init.
921
    "DPRPretrainedReader",  # Like an Encoder.
Sylvain Gugger's avatar
Sylvain Gugger committed
922
    "DummyObject",  # Just picked by mistake sometimes.
923
    "MecabTokenizer",  # Internal, should never have been in the main init.
924
925
926
927
928
929
930
931
932
933
934
    "ModelCard",  # Internal type.
    "SqueezeBertModule",  # Internal building block (should have been called SqueezeBertLayer)
    "TFDPRPretrainedReader",  # Like an Encoder.
    "TransfoXLCorpus",  # Internal type.
    "WordpieceTokenizer",  # Internal, should never have been in the main init.
    "absl",  # External module
    "add_end_docstrings",  # Internal, should never have been in the main init.
    "add_start_docstrings",  # Internal, should never have been in the main init.
    "convert_tf_weight_name_to_pt_weight_name",  # Internal used to convert model weights
    "logger",  # Internal logger
    "logging",  # External module
935
    "requires_backends",  # Internal function
Jongjyh's avatar
Jongjyh committed
936
    "AltRobertaModel",  # Internal module
937
938
939
940
941
942
    "FalconConfig",  # TODO Matt Remove this and re-add the docs once TGI is ready
    "FalconForCausalLM",
    "FalconForQuestionAnswering",
    "FalconForSequenceClassification",
    "FalconForTokenClassification",
    "FalconModel",
943
944
945
946
947
948
949
950
951
]

# This list should be empty. Objects in it should get their own doc page.
SHOULD_HAVE_THEIR_OWN_PAGE = [
    # Benchmarks
    "PyTorchBenchmark",
    "PyTorchBenchmarkArguments",
    "TensorFlowBenchmark",
    "TensorFlowBenchmarkArguments",
952
    "AutoBackbone",
NielsRogge's avatar
NielsRogge committed
953
954
    "BitBackbone",
    "ConvNextBackbone",
Alara Dirik's avatar
Alara Dirik committed
955
    "ConvNextV2Backbone",
956
    "DinatBackbone",
Alara Dirik's avatar
Alara Dirik committed
957
    "FocalNetBackbone",
NielsRogge's avatar
NielsRogge committed
958
    "MaskFormerSwinBackbone",
959
960
    "MaskFormerSwinConfig",
    "MaskFormerSwinModel",
NielsRogge's avatar
NielsRogge committed
961
962
    "NatBackbone",
    "ResNetBackbone",
NielsRogge's avatar
NielsRogge committed
963
    "SwinBackbone",
amyeroberts's avatar
amyeroberts committed
964
965
    "TimmBackbone",
    "TimmBackboneConfig",
966
967
968
]


Sylvain Gugger's avatar
Sylvain Gugger committed
969
970
def ignore_undocumented(name: str) -> bool:
    """Rules to determine if `name` should be undocumented (returns `True` if it should not be documented)."""
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
    # NOT DOCUMENTED ON PURPOSE.
    # Constants uppercase are not documented.
    if name.isupper():
        return True
    # PreTrainedModels / Encoders / Decoders / Layers / Embeddings / Attention are not documented.
    if (
        name.endswith("PreTrainedModel")
        or name.endswith("Decoder")
        or name.endswith("Encoder")
        or name.endswith("Layer")
        or name.endswith("Embeddings")
        or name.endswith("Attention")
    ):
        return True
    # Submodules are not documented.
    if os.path.isdir(os.path.join(PATH_TO_TRANSFORMERS, name)) or os.path.isfile(
        os.path.join(PATH_TO_TRANSFORMERS, f"{name}.py")
    ):
        return True
    # All load functions are not documented.
    if name.startswith("load_tf") or name.startswith("load_pytorch"):
        return True
    # is_xxx_available functions are not documented.
    if name.startswith("is_") and name.endswith("_available"):
        return True
    # Deprecated objects are not documented.
    if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS:
        return True
    # MMBT model does not really work.
    if name.startswith("MMBT"):
        return True
    if name in SHOULD_HAVE_THEIR_OWN_PAGE:
        return True
    return False


def check_all_objects_are_documented():
Patrick von Platen's avatar
Patrick von Platen committed
1008
    """Check all models are properly documented."""
1009
    documented_objs = find_all_documented_objects()
1010
1011
1012
    modules = transformers._modules
    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]
    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]
1013
1014
1015
1016
1017
    if len(undocumented_objs) > 0:
        raise Exception(
            "The following objects are in the public init so should be documented:\n - "
            + "\n - ".join(undocumented_objs)
        )
1018
    check_docstrings_are_in_md()
1019
1020
1021
1022
1023
1024
    check_model_type_doc_match()


def check_model_type_doc_match():
    """Check all doc pages have a corresponding model type."""
    model_doc_folder = Path(PATH_TO_DOC) / "model_doc"
1025
    model_docs = [m.stem for m in model_doc_folder.glob("*.md")]
1026
1027

    model_types = list(transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys())
1028
    model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types]
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046

    errors = []
    for m in model_docs:
        if m not in model_types and m != "auto":
            close_matches = get_close_matches(m, model_types)
            error_message = f"{m} is not a proper model identifier."
            if len(close_matches) > 0:
                close_matches = "/".join(close_matches)
                error_message += f" Did you mean {close_matches}?"
            errors.append(error_message)

    if len(errors) > 0:
        raise ValueError(
            "Some model doc pages do not match any existing model type:\n"
            + "\n".join(errors)
            + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in "
            "models/auto/configuration_auto.py."
        )
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056


# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`.
_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`")
# Re pattern to catch things between double backquotes.
_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")
# Re pattern to catch example introduction.
_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE)


Sylvain Gugger's avatar
Sylvain Gugger committed
1057
def is_rst_docstring(docstring: str) -> True:
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
    """
    Returns `True` if `docstring` is written in rst.
    """
    if _re_rst_special_words.search(docstring) is not None:
        return True
    if _re_double_backquotes.search(docstring) is not None:
        return True
    if _re_rst_example.search(docstring) is not None:
        return True
    return False


def check_docstrings_are_in_md():
Sylvain Gugger's avatar
Sylvain Gugger committed
1071
    """Check all docstrings are written in md and nor rst."""
1072
1073
    files_with_rst = []
    for file in Path(PATH_TO_TRANSFORMERS).glob("**/*.py"):
1074
        with open(file, encoding="utf-8") as f:
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
            code = f.read()
        docstrings = code.split('"""')

        for idx, docstring in enumerate(docstrings):
            if idx % 2 == 0 or not is_rst_docstring(docstring):
                continue
            files_with_rst.append(file)
            break

    if len(files_with_rst) > 0:
        raise ValueError(
            "The following files have docstrings written in rst:\n"
            + "\n".join([f"- {f}" for f in files_with_rst])
Kamal Raj's avatar
Kamal Raj committed
1088
            + "\nTo fix this run `doc-builder convert path_to_py_file` after installing `doc-builder`\n"
1089
1090
            "(`pip install git+https://github.com/huggingface/doc-builder`)"
        )
1091
1092


1093
def check_deprecated_constant_is_up_to_date():
Sylvain Gugger's avatar
Sylvain Gugger committed
1094
1095
1096
    """
    Check if the constant `DEPRECATED_MODELS` in `models/auto/configuration_auto.py` is up to date.
    """
1097
1098
1099
1100
1101
1102
1103
1104
1105
    deprecated_folder = os.path.join(PATH_TO_TRANSFORMERS, "models", "deprecated")
    deprecated_models = [m for m in os.listdir(deprecated_folder) if not m.startswith("_")]

    constant_to_check = transformers.models.auto.configuration_auto.DEPRECATED_MODELS
    message = []
    missing_models = sorted(set(deprecated_models) - set(constant_to_check))
    if len(missing_models) != 0:
        missing_models = ", ".join(missing_models)
        message.append(
1106
            "The following models are in the deprecated folder, make sure to add them to `DEPRECATED_MODELS` in "
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
            f"`models/auto/configuration_auto.py`: {missing_models}."
        )

    extra_models = sorted(set(constant_to_check) - set(deprecated_models))
    if len(extra_models) != 0:
        extra_models = ", ".join(extra_models)
        message.append(
            "The following models are in the `DEPRECATED_MODELS` constant but not in the deprecated folder. Either "
            f"remove them from the constant or move to the deprecated folder: {extra_models}."
        )

    if len(message) > 0:
        raise Exception("\n".join(message))


1122
def check_repo_quality():
Patrick von Platen's avatar
Patrick von Platen committed
1123
    """Check all models are properly tested and documented."""
1124
1125
    print("Checking all models are included.")
    check_model_list()
1126
1127
    print("Checking all models are public.")
    check_models_are_in_init()
1128
    print("Checking all models are properly tested.")
Sylvain Gugger's avatar
Sylvain Gugger committed
1129
    check_all_decorator_order()
1130
    check_all_models_are_tested()
1131
    print("Checking all objects are properly documented.")
1132
    check_all_objects_are_documented()
1133
1134
    print("Checking all models are in at least one auto class.")
    check_all_models_are_auto_configured()
1135
1136
    print("Checking all names in auto name mappings are defined.")
    check_all_auto_object_names_being_defined()
Yih-Dar's avatar
Yih-Dar committed
1137
1138
    print("Checking all keys in auto name mappings are defined in `CONFIG_MAPPING_NAMES`.")
    check_all_auto_mapping_names_in_config_mapping_names()
1139
1140
    print("Checking all auto mappings could be imported.")
    check_all_auto_mappings_importable()
1141
1142
    print("Checking all objects are equally (across frameworks) in the main __init__.")
    check_objects_being_equally_in_main_init()
1143
1144
    print("Checking the DEPRECATED_MODELS constant is up to date.")
    check_deprecated_constant_is_up_to_date()
1145
1146
1147
1148


if __name__ == "__main__":
    check_repo_quality()