check_repo.py 45.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
"""
Utility that performs several consistency checks on the repo. This includes:
- checking all models are properly defined in the __init__ of models/
- checking all models are in the main __init__
- checking all models are properly tested
- checking all object in the main __init__ are documented
- checking all models are in at least one auto class
- checking all the auto mapping are properly defined (no typos, importable)
- checking the list of deprecated models is up to date

Use from the root of the repo with (as used in `make repo-consistency`):

```bash
python utils/check_repo.py
```

It has no auto-fix mode.
"""
33
34
35
import inspect
import os
import re
36
import sys
Sylvain Gugger's avatar
Sylvain Gugger committed
37
import types
38
import warnings
39
from collections import OrderedDict
40
from difflib import get_close_matches
41
from pathlib import Path
Sylvain Gugger's avatar
Sylvain Gugger committed
42
from typing import List, Tuple
43

44
from transformers import is_flax_available, is_tf_available, is_torch_available
45
from transformers.models.auto import get_values
Yih-Dar's avatar
Yih-Dar committed
46
from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
47
48
49
50
from transformers.models.auto.feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING_NAMES
from transformers.models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING_NAMES
from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES
from transformers.models.auto.tokenization_auto import TOKENIZER_MAPPING_NAMES
51
from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import
52

53
54
55
56
57

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_repo.py
PATH_TO_TRANSFORMERS = "src/transformers"
PATH_TO_TESTS = "tests"
58
PATH_TO_DOC = "docs/source/en"
59

60
61
# Update this list with models that are supposed to be private.
PRIVATE_MODELS = [
Jongjyh's avatar
Jongjyh committed
62
    "AltRobertaModel",
63
    "DPRSpanPredictor",
Daniel Stancl's avatar
Daniel Stancl committed
64
    "LongT5Stack",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
65
    "RealmBertModel",
66
    "T5Stack",
67
    "MT5Stack",
68
    "UMT5Stack",
Susnato Dhar's avatar
Susnato Dhar committed
69
    "Pop2PianoStack",
70
    "SwitchTransformersStack",
71
    "TFDPRSpanPredictor",
72
73
    "MaskFormerSwinModel",
    "MaskFormerSwinPreTrainedModel",
74
75
    "BridgeTowerTextModel",
    "BridgeTowerVisionModel",
Yih-Dar's avatar
Yih-Dar committed
76
77
78
    "Kosmos2TextModel",
    "Kosmos2TextForCausalLM",
    "Kosmos2VisionModel",
79
80
]

81
82
# Update this list for models that are not tested with a comment explaining the reason it should not be.
# Being in this list is an exception and should **not** be the rule.
83
IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [
84
    # models to ignore for not tested
Pablo Montalvo's avatar
Pablo Montalvo committed
85
    "FuyuForCausalLM",  # Not tested fort now
NielsRogge's avatar
NielsRogge committed
86
    "InstructBlipQFormerModel",  # Building part of bigger (tested) model.
87
    "UMT5EncoderModel",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
88
    "Blip2QFormerModel",  # Building part of bigger (tested) model.
89
    "ErnieMForInformationExtraction",
90
    "GraphormerDecoderHead",  # Building part of bigger (tested) model.
91
92
    "JukeboxVQVAE",  # Building part of bigger (tested) model.
    "JukeboxPrior",  # Building part of bigger (tested) model.
93
    "DecisionTransformerGPT2Model",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
94
    "SegformerDecodeHead",  # Building part of bigger (tested) model.
wangpeng's avatar
wangpeng committed
95
    "MgpstrModel",  # Building part of bigger (tested) model.
96
    "BertLMHeadModel",  # Needs to be setup as decoder.
97
    "MegatronBertLMHeadModel",  # Building part of bigger (tested) model.
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
98
99
100
101
    "RealmBertModel",  # Building part of bigger (tested) model.
    "RealmReader",  # Not regular model.
    "RealmScorer",  # Not regular model.
    "RealmForOpenQA",  # Not regular model.
102
103
104
    "ReformerForMaskedLM",  # Needs to be setup as decoder.
    "TFElectraMainLayer",  # Building part of bigger (tested) model (should it be a TFPreTrainedModel ?)
    "TFRobertaForMultipleChoice",  # TODO: fix
105
    "TFRobertaPreLayerNormForMultipleChoice",  # TODO: fix
abhishek thakur's avatar
abhishek thakur committed
106
    "SeparableConv1D",  # Building part of bigger (tested) model.
107
    "FlaxBartForCausalLM",  # Building part of bigger (tested) model.
108
    "FlaxBertForCausalLM",  # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM.
Younes Belkada's avatar
Younes Belkada committed
109
    "OPTDecoderWrapper",
110
    "TFSegformerDecodeHead",  # Not a regular model.
Jongjyh's avatar
Jongjyh committed
111
    "AltRobertaModel",  # Building part of bigger (tested) model.
Younes Belkada's avatar
Younes Belkada committed
112
    "BlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
Matt's avatar
Matt committed
113
    "TFBlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
114
115
    "BridgeTowerTextModel",  # No need to test it as it is tested by BridgeTowerModel model.
    "BridgeTowerVisionModel",  # No need to test it as it is tested by BridgeTowerModel model.
Yoach Lacombe's avatar
Yoach Lacombe committed
116
117
    "BarkCausalModel",  # Building part of bigger (tested) model.
    "BarkModel",  # Does not have a forward signature - generation tested with integration tests
118
119
120
    "SeamlessM4TTextToUnitModel",  # Building part of bigger (tested) model.
    "SeamlessM4TCodeHifiGan",  # Building part of bigger (tested) model.
    "SeamlessM4TTextToUnitForConditionalGeneration",  # Building part of bigger (tested) model.
121
122
123
124
125
]

# Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't
# trigger the common tests.
TEST_FILES_WITH_NO_COMMON_TESTS = [
Yih-Dar's avatar
Yih-Dar committed
126
127
128
129
130
131
132
133
134
135
136
137
138
    "models/decision_transformer/test_modeling_decision_transformer.py",
    "models/camembert/test_modeling_camembert.py",
    "models/mt5/test_modeling_flax_mt5.py",
    "models/mbart/test_modeling_mbart.py",
    "models/mt5/test_modeling_mt5.py",
    "models/pegasus/test_modeling_pegasus.py",
    "models/camembert/test_modeling_tf_camembert.py",
    "models/mt5/test_modeling_tf_mt5.py",
    "models/xlm_roberta/test_modeling_tf_xlm_roberta.py",
    "models/xlm_roberta/test_modeling_flax_xlm_roberta.py",
    "models/xlm_prophetnet/test_modeling_xlm_prophetnet.py",
    "models/xlm_roberta/test_modeling_xlm_roberta.py",
    "models/vision_text_dual_encoder/test_modeling_vision_text_dual_encoder.py",
Matt's avatar
Matt committed
139
    "models/vision_text_dual_encoder/test_modeling_tf_vision_text_dual_encoder.py",
Yih-Dar's avatar
Yih-Dar committed
140
141
    "models/vision_text_dual_encoder/test_modeling_flax_vision_text_dual_encoder.py",
    "models/decision_transformer/test_modeling_decision_transformer.py",
Yoach Lacombe's avatar
Yoach Lacombe committed
142
    "models/bark/test_modeling_bark.py",
143
144
]

145
146
# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and
# should **not** be the rule.
147
IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [
148
    # models to ignore for model xxx mapping
149
150
    "AlignTextModel",
    "AlignVisionModel",
151
152
153
154
    "ClapTextModel",
    "ClapTextModelWithProjection",
    "ClapAudioModel",
    "ClapAudioModelWithProjection",
NielsRogge's avatar
NielsRogge committed
155
156
157
    "Blip2ForConditionalGeneration",
    "Blip2QFormerModel",
    "Blip2VisionModel",
158
    "ErnieMForInformationExtraction",
159
    "GitVisionModel",
160
161
    "GraphormerModel",
    "GraphormerForGraphClassification",
Younes Belkada's avatar
Younes Belkada committed
162
163
164
165
166
167
    "BlipForConditionalGeneration",
    "BlipForImageTextRetrieval",
    "BlipForQuestionAnswering",
    "BlipVisionModel",
    "BlipTextLMHeadModel",
    "BlipTextModel",
Jinho Park's avatar
Jinho Park committed
168
169
    "BrosSpadeEEForTokenClassification",
    "BrosSpadeELForTokenClassification",
Matt's avatar
Matt committed
170
171
172
173
174
175
    "TFBlipForConditionalGeneration",
    "TFBlipForImageTextRetrieval",
    "TFBlipForQuestionAnswering",
    "TFBlipVisionModel",
    "TFBlipTextLMHeadModel",
    "TFBlipTextModel",
NielsRogge's avatar
NielsRogge committed
176
    "Swin2SRForImageSuperResolution",
177
178
    "BridgeTowerForImageAndTextRetrieval",
    "BridgeTowerForMaskedLM",
179
    "BridgeTowerForContrastiveLearning",
NielsRogge's avatar
NielsRogge committed
180
181
182
    "CLIPSegForImageSegmentation",
    "CLIPSegVisionModel",
    "CLIPSegTextModel",
Matt's avatar
Matt committed
183
    "EsmForProteinFolding",
184
    "GPTSanJapaneseModel",
185
    "TimeSeriesTransformerForPrediction",
186
    "InformerForPrediction",
187
    "AutoformerForPrediction",
188
189
    "PatchTSTForPretraining",
    "PatchTSTForPrediction",
190
191
    "JukeboxVQVAE",
    "JukeboxPrior",
192
    "SamModel",
NielsRogge's avatar
NielsRogge committed
193
    "DPTForDepthEstimation",
194
    "DecisionTransformerGPT2Model",
NielsRogge's avatar
NielsRogge committed
195
    "GLPNForDepthEstimation",
NielsRogge's avatar
NielsRogge committed
196
197
    "ViltForImagesAndTextClassification",
    "ViltForImageAndTextRetrieval",
198
    "ViltForTokenClassification",
NielsRogge's avatar
NielsRogge committed
199
    "ViltForMaskedLM",
NielsRogge's avatar
NielsRogge committed
200
201
    "PerceiverForMultimodalAutoencoding",
    "PerceiverForOpticalFlow",
NielsRogge's avatar
NielsRogge committed
202
    "SegformerDecodeHead",
203
    "TFSegformerDecodeHead",
Kamal Raj's avatar
Kamal Raj committed
204
    "FlaxBeitForMaskedImageModeling",
NielsRogge's avatar
NielsRogge committed
205
    "BeitForMaskedImageModeling",
206
207
    "ChineseCLIPTextModel",
    "ChineseCLIPVisionModel",
Suraj Patil's avatar
Suraj Patil committed
208
    "CLIPTextModel",
209
    "CLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
210
    "CLIPVisionModel",
211
    "CLIPVisionModelWithProjection",
Susnato Dhar's avatar
Susnato Dhar committed
212
213
    "ClvpForCausalLM",
    "ClvpModel",
214
215
    "GroupViTTextModel",
    "GroupViTVisionModel",
Yih-Dar's avatar
Yih-Dar committed
216
217
    "TFCLIPTextModel",
    "TFCLIPVisionModel",
218
219
    "TFGroupViTTextModel",
    "TFGroupViTVisionModel",
Suraj Patil's avatar
Suraj Patil committed
220
    "FlaxCLIPTextModel",
221
    "FlaxCLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
222
    "FlaxCLIPVisionModel",
223
    "FlaxWav2Vec2ForCTC",
NielsRogge's avatar
NielsRogge committed
224
    "DetrForSegmentation",
Younes Belkada's avatar
Younes Belkada committed
225
226
227
    "Pix2StructVisionModel",
    "Pix2StructTextModel",
    "Pix2StructForConditionalGeneration",
228
    "ConditionalDetrForSegmentation",
229
230
    "DPRReader",
    "FlaubertForQuestionAnswering",
231
232
233
234
    "FlavaImageCodebook",
    "FlavaTextModel",
    "FlavaImageModel",
    "FlavaMultimodalModel",
235
    "GPT2DoubleHeadsModel",
236
    "GPTSw3DoubleHeadsModel",
NielsRogge's avatar
NielsRogge committed
237
238
    "InstructBlipVisionModel",
    "InstructBlipQFormerModel",
239
    "LayoutLMForQuestionAnswering",
Ryokan RI's avatar
Ryokan RI committed
240
    "LukeForMaskedLM",
NielsRogge's avatar
NielsRogge committed
241
242
243
    "LukeForEntityClassification",
    "LukeForEntityPairClassification",
    "LukeForEntitySpanClassification",
wangpeng's avatar
wangpeng committed
244
    "MgpstrModel",
245
    "OpenAIGPTDoubleHeadsModel",
246
247
    "OwlViTTextModel",
    "OwlViTVisionModel",
NielsRogge's avatar
NielsRogge committed
248
249
    "Owlv2TextModel",
    "Owlv2VisionModel",
250
    "OwlViTForObjectDetection",
251
252
253
    "RagModel",
    "RagSequenceForGeneration",
    "RagTokenForGeneration",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
254
255
256
257
    "RealmEmbedder",
    "RealmForOpenQA",
    "RealmScorer",
    "RealmReader",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
258
    "TFDPRReader",
259
    "TFGPT2DoubleHeadsModel",
260
    "TFLayoutLMForQuestionAnswering",
261
    "TFOpenAIGPTDoubleHeadsModel",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
262
263
264
    "TFRagModel",
    "TFRagSequenceForGeneration",
    "TFRagTokenForGeneration",
265
    "Wav2Vec2ForCTC",
Patrick von Platen's avatar
Patrick von Platen committed
266
    "HubertForCTC",
267
268
    "SEWForCTC",
    "SEWDForCTC",
269
270
    "XLMForQuestionAnswering",
    "XLNetForQuestionAnswering",
abhishek thakur's avatar
abhishek thakur committed
271
    "SeparableConv1D",
Gunjan Chhablani's avatar
Gunjan Chhablani committed
272
273
274
275
    "VisualBertForRegionToPhraseAlignment",
    "VisualBertForVisualReasoning",
    "VisualBertForQuestionAnswering",
    "VisualBertForMultipleChoice",
Will Rice's avatar
Will Rice committed
276
    "TFWav2Vec2ForCTC",
Will Rice's avatar
Will Rice committed
277
    "TFHubertForCTC",
NielsRogge's avatar
NielsRogge committed
278
279
    "XCLIPVisionModel",
    "XCLIPTextModel",
Jongjyh's avatar
Jongjyh committed
280
281
282
    "AltCLIPTextModel",
    "AltCLIPVisionModel",
    "AltRobertaModel",
Zineng Tang's avatar
Zineng Tang committed
283
    "TvltForAudioVisualClassification",
Yoach Lacombe's avatar
Yoach Lacombe committed
284
285
286
287
288
289
    "BarkCausalModel",
    "BarkCoarseModel",
    "BarkFineModel",
    "BarkSemanticModel",
    "MusicgenModel",
    "MusicgenForConditionalGeneration",
290
291
292
    "SpeechT5ForSpeechToSpeech",
    "SpeechT5ForTextToSpeech",
    "SpeechT5HifiGan",
NielsRogge's avatar
NielsRogge committed
293
    "VitMatteForImageMatting",
294
295
296
297
    "SeamlessM4TTextToUnitModel",
    "SeamlessM4TTextToUnitForConditionalGeneration",
    "SeamlessM4TCodeHifiGan",
    "SeamlessM4TForSpeechToSpeech",  # no auto class for speech-to-speech
298
299
]

300
# DO NOT edit this list!
Sylvain Gugger's avatar
Sylvain Gugger committed
301
# (The corresponding pytorch objects should never have been in the main `__init__`, but it's too late to remove)
302
303
304
305
306
307
308
309
310
311
312
313
314
OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK = [
    "FlaxBertLayer",
    "FlaxBigBirdLayer",
    "FlaxRoFormerLayer",
    "TFBertLayer",
    "TFLxmertEncoder",
    "TFLxmertXLayer",
    "TFMPNetLayer",
    "TFMobileBertLayer",
    "TFSegformerLayer",
    "TFViTMAELayer",
]

Sylvain Gugger's avatar
Sylvain Gugger committed
315
# Update this list for models that have multiple model types for the same model doc.
316
317
318
319
MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(
    [
        ("data2vec-text", "data2vec"),
        ("data2vec-audio", "data2vec"),
320
        ("data2vec-vision", "data2vec"),
NielsRogge's avatar
NielsRogge committed
321
        ("donut-swin", "donut"),
322
323
324
325
    ]
)


326
# This is to make sure the transformers module imported is the one in the repo.
327
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
328
329


330
def check_missing_backends():
Sylvain Gugger's avatar
Sylvain Gugger committed
331
332
333
334
    """
    Checks if all backends are installed (otherwise the check of this script is incomplete). Will error in the CI if
    that's not the case but only throw a warning for users running this.
    """
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
    missing_backends = []
    if not is_torch_available():
        missing_backends.append("PyTorch")
    if not is_tf_available():
        missing_backends.append("TensorFlow")
    if not is_flax_available():
        missing_backends.append("Flax")
    if len(missing_backends) > 0:
        missing = ", ".join(missing_backends)
        if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES:
            raise Exception(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}."
            )
        else:
            warnings.warn(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you "
                "didn't make any change in one of those backends modeling files, you should probably execute the "
                "command above to be on the safe side."
            )


358
def check_model_list():
Sylvain Gugger's avatar
Sylvain Gugger committed
359
360
361
    """
    Checks the model listed as subfolders of `models` match the models available in `transformers.models`.
    """
362
363
364
365
    # Get the models from the directory structure of `src/transformers/models/`
    models_dir = os.path.join(PATH_TO_TRANSFORMERS, "models")
    _models = []
    for model in os.listdir(models_dir):
Sylvain Gugger's avatar
Sylvain Gugger committed
366
367
        if model == "deprecated":
            continue
368
369
370
371
        model_dir = os.path.join(models_dir, model)
        if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir):
            _models.append(model)

Sylvain Gugger's avatar
Sylvain Gugger committed
372
    # Get the models in the submodule `transformers.models`
373
374
    models = [model for model in dir(transformers.models) if not model.startswith("__")]

375
    missing_models = sorted(set(_models).difference(models))
376
377
378
379
380
381
    if missing_models:
        raise Exception(
            f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}."
        )


382
383
# If some modeling modules should be ignored for all checks, they should be added in the nested list
# _ignore_modules of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
384
385
def get_model_modules() -> List[str]:
    """Get all the model modules inside the transformers library (except deprecated models)."""
386
387
388
389
390
391
392
393
    _ignore_modules = [
        "modeling_auto",
        "modeling_encoder_decoder",
        "modeling_marian",
        "modeling_mmbt",
        "modeling_outputs",
        "modeling_retribert",
        "modeling_utils",
Sylvain Gugger's avatar
Sylvain Gugger committed
394
        "modeling_flax_auto",
395
        "modeling_flax_encoder_decoder",
Stas Bekman's avatar
Stas Bekman committed
396
        "modeling_flax_utils",
397
        "modeling_speech_encoder_decoder",
398
        "modeling_flax_speech_encoder_decoder",
399
        "modeling_flax_vision_encoder_decoder",
amyeroberts's avatar
amyeroberts committed
400
        "modeling_timm_backbone",
401
402
        "modeling_transfo_xl_utilities",
        "modeling_tf_auto",
403
        "modeling_tf_encoder_decoder",
404
405
406
407
        "modeling_tf_outputs",
        "modeling_tf_pytorch_utils",
        "modeling_tf_utils",
        "modeling_tf_transfo_xl_utilities",
408
        "modeling_tf_vision_encoder_decoder",
409
        "modeling_vision_encoder_decoder",
410
411
    ]
    modules = []
Sylvain Gugger's avatar
Sylvain Gugger committed
412
413
    for model in dir(transformers.models):
        # There are some magic dunder attributes in the dir, we ignore them
Sylvain Gugger's avatar
Sylvain Gugger committed
414
415
416
417
418
419
420
421
422
        if model == "deprecated" or model.startswith("__"):
            continue

        model_module = getattr(transformers.models, model)
        for submodule in dir(model_module):
            if submodule.startswith("modeling") and submodule not in _ignore_modules:
                modeling_module = getattr(model_module, submodule)
                if inspect.ismodule(modeling_module):
                    modules.append(modeling_module)
423
424
425
    return modules


Sylvain Gugger's avatar
Sylvain Gugger committed
426
427
428
429
430
431
432
433
434
435
436
437
438
def get_models(module: types.ModuleType, include_pretrained: bool = False) -> List[Tuple[str, type]]:
    """
    Get the objects in a module that are models.

    Args:
        module (`types.ModuleType`):
            The module from which we are extracting models.
        include_pretrained (`bool`, *optional*, defaults to `False`):
            Whether or not to include the `PreTrainedModel` subclass (like `BertPreTrainedModel`) or not.

    Returns:
        List[Tuple[str, type]]: List of models as tuples (class name, actual class).
    """
439
    models = []
440
    model_classes = (transformers.PreTrainedModel, transformers.TFPreTrainedModel, transformers.FlaxPreTrainedModel)
441
    for attr_name in dir(module):
442
        if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name):
443
444
445
446
447
448
449
            continue
        attr = getattr(module, attr_name)
        if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__:
            models.append((attr_name, attr))
    return models


Sylvain Gugger's avatar
Sylvain Gugger committed
450
451
452
453
def is_building_block(model: str) -> bool:
    """
    Returns `True` if a model is a building block part of a bigger model.
    """
454
455
456
457
458
459
    if model.endswith("Wrapper"):
        return True
    if model.endswith("Encoder"):
        return True
    if model.endswith("Decoder"):
        return True
460
461
    if model.endswith("Prenet"):
        return True
Sylvain Gugger's avatar
Sylvain Gugger committed
462
463
464
465
466
467
468


def is_a_private_model(model: str) -> bool:
    """Returns `True` if the model should not be in the main init."""
    if model in PRIVATE_MODELS:
        return True
    return is_building_block(model)
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485


def check_models_are_in_init():
    """Checks all models defined in the library are in the main init."""
    models_not_in_init = []
    dir_transformers = dir(transformers)
    for module in get_model_modules():
        models_not_in_init += [
            model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers
        ]

    # Remove private models
    models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)]
    if len(models_not_in_init) > 0:
        raise Exception(f"The following models should be in the main init: {','.join(models_not_in_init)}.")


486
487
# If some test_modeling files should be ignored when checking models are all tested, they should be added in the
# nested list _ignore_files of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
488
489
490
def get_model_test_files() -> List[str]:
    """
    Get the model test files.
Yih-Dar's avatar
Yih-Dar committed
491

Sylvain Gugger's avatar
Sylvain Gugger committed
492
493
494
495
    Returns:
        `List[str]`: The list of test files. The returned files will NOT contain the `tests` (i.e. `PATH_TO_TESTS`
        defined in this script). They will be considered as paths relative to `tests`. A caller has to use
        `os.path.join(PATH_TO_TESTS, ...)` to access the files.
Yih-Dar's avatar
Yih-Dar committed
496
497
    """

498
499
500
    _ignore_files = [
        "test_modeling_common",
        "test_modeling_encoder_decoder",
501
        "test_modeling_flax_encoder_decoder",
502
        "test_modeling_flax_speech_encoder_decoder",
503
504
        "test_modeling_marian",
        "test_modeling_tf_common",
505
        "test_modeling_tf_encoder_decoder",
506
507
    ]
    test_files = []
Yih-Dar's avatar
Yih-Dar committed
508
509
510
511
512
513
514
515
516
517
518
519
    model_test_root = os.path.join(PATH_TO_TESTS, "models")
    model_test_dirs = []
    for x in os.listdir(model_test_root):
        x = os.path.join(model_test_root, x)
        if os.path.isdir(x):
            model_test_dirs.append(x)

    for target_dir in [PATH_TO_TESTS] + model_test_dirs:
        for file_or_dir in os.listdir(target_dir):
            path = os.path.join(target_dir, file_or_dir)
            if os.path.isfile(path):
                filename = os.path.split(path)[-1]
520
                if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files:
Yih-Dar's avatar
Yih-Dar committed
521
522
523
                    file = os.path.join(*path.split(os.sep)[1:])
                    test_files.append(file)

524
525
526
527
528
    return test_files


# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class
# for the all_model_classes variable.
Sylvain Gugger's avatar
Sylvain Gugger committed
529
530
531
532
533
534
535
536
537
538
539
def find_tested_models(test_file: str) -> List[str]:
    """
    Parse the content of test_file to detect what's in `all_model_classes`. This detects the models that inherit from
    the common test class.

    Args:
        test_file (`str`): The path to the test file to check

    Returns:
        `List[str]`: The list of models tested in that file.
    """
540
    with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f:
541
        content = f.read()
Sylvain Gugger's avatar
Sylvain Gugger committed
542
    all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content)
543
544
    # Check with one less parenthesis as well
    all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
545
    if len(all_models) > 0:
546
        model_tested = []
Sylvain Gugger's avatar
Sylvain Gugger committed
547
548
549
550
551
        for entry in all_models:
            for line in entry.split(","):
                name = line.strip()
                if len(name) > 0:
                    model_tested.append(name)
552
553
554
        return model_tested


Sylvain Gugger's avatar
Sylvain Gugger committed
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
def should_be_tested(model_name: str) -> bool:
    """
    Whether or not a model should be tested.
    """
    if model_name in IGNORE_NON_TESTED:
        return False
    return not is_building_block(model_name)


def check_models_are_tested(module: types.ModuleType, test_file: str) -> List[str]:
    """Check models defined in a module are all tested in a given file.

    Args:
        module (`types.ModuleType`): The module in which we get the models.
        test_file (`str`): The path to the file where the module is tested.

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
574
    # XxxPreTrainedModel are not tested
575
576
577
    defined_models = get_models(module)
    tested_models = find_tested_models(test_file)
    if tested_models is None:
578
        if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS:
579
580
581
582
583
584
585
586
            return
        return [
            f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. "
            + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file "
            + "`utils/check_repo.py`."
        ]
    failures = []
    for model_name, _ in defined_models:
Sylvain Gugger's avatar
Sylvain Gugger committed
587
        if model_name not in tested_models and should_be_tested(model_name):
588
589
590
591
592
593
594
595
596
597
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not tested in "
                + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file."
                + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`"
                + "in the file `utils/check_repo.py`."
            )
    return failures


def check_all_models_are_tested():
Patrick von Platen's avatar
Patrick von Platen committed
598
    """Check all models are properly tested."""
599
600
601
602
    modules = get_model_modules()
    test_files = get_model_test_files()
    failures = []
    for module in modules:
Sylvain Gugger's avatar
Sylvain Gugger committed
603
        # Matches a module to its test file.
604
605
        test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file]
        if len(test_file) == 0:
606
            failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")
607
608
609
610
        elif len(test_file) > 1:
            failures.append(f"{module.__name__} has several test files: {test_file}.")
        else:
            test_file = test_file[0]
611
612
613
            new_failures = check_models_are_tested(module, test_file)
            if new_failures is not None:
                failures += new_failures
614
615
616
617
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
618
def get_all_auto_configured_models() -> List[str]:
Patrick von Platen's avatar
Patrick von Platen committed
619
    """Return the list of all models in at least one auto class."""
620
    result = set()  # To avoid duplicates we concatenate all model classes in a set.
621
622
    if is_torch_available():
        for attr_name in dir(transformers.models.auto.modeling_auto):
623
            if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"):
624
625
626
                result = result | set(get_values(getattr(transformers.models.auto.modeling_auto, attr_name)))
    if is_tf_available():
        for attr_name in dir(transformers.models.auto.modeling_tf_auto):
627
            if attr_name.startswith("TF_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
628
629
630
                result = result | set(get_values(getattr(transformers.models.auto.modeling_tf_auto, attr_name)))
    if is_flax_available():
        for attr_name in dir(transformers.models.auto.modeling_flax_auto):
631
            if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
632
                result = result | set(get_values(getattr(transformers.models.auto.modeling_flax_auto, attr_name)))
633
    return list(result)
634
635


Sylvain Gugger's avatar
Sylvain Gugger committed
636
637
def ignore_unautoclassed(model_name: str) -> bool:
    """Rules to determine if a model should be in an auto class."""
638
639
640
641
642
643
644
645
646
    # Special white list
    if model_name in IGNORE_NON_AUTO_CONFIGURED:
        return True
    # Encoder and Decoder should be ignored
    if "Encoder" in model_name or "Decoder" in model_name:
        return True
    return False


Sylvain Gugger's avatar
Sylvain Gugger committed
647
648
649
650
651
652
653
654
655
656
657
658
659
def check_models_are_auto_configured(module: types.ModuleType, all_auto_models: List[str]) -> List[str]:
    """
    Check models defined in module are each in an auto class.

    Args:
        module (`types.ModuleType`):
            The module in which we get the models.
        all_auto_models (`List[str]`):
            The list of all models in an auto class (as obtained with `get_all_auto_configured_models()`).

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
660
661
662
    defined_models = get_models(module)
    failures = []
    for model_name, _ in defined_models:
663
        if model_name not in all_auto_models and not ignore_unautoclassed(model_name):
664
665
666
667
668
669
670
671
672
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "
                "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "
                "`utils/check_repo.py`."
            )
    return failures


def check_all_models_are_auto_configured():
Patrick von Platen's avatar
Patrick von Platen committed
673
    """Check all models are each in an auto class."""
Sylvain Gugger's avatar
Sylvain Gugger committed
674
    # This is where we need to check we have all backends or the check is incomplete.
675
    check_missing_backends()
676
677
678
679
680
681
682
683
684
685
686
    modules = get_model_modules()
    all_auto_models = get_all_auto_configured_models()
    failures = []
    for module in modules:
        new_failures = check_models_are_auto_configured(module, all_auto_models)
        if new_failures is not None:
            failures += new_failures
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


687
688
def check_all_auto_object_names_being_defined():
    """Check all names defined in auto (name) mappings exist in the library."""
Sylvain Gugger's avatar
Sylvain Gugger committed
689
    # This is where we need to check we have all backends or the check is incomplete.
690
    check_missing_backends()
691

692
    failures = []
693
    mappings_to_check = {
694
695
696
697
698
699
        "TOKENIZER_MAPPING_NAMES": TOKENIZER_MAPPING_NAMES,
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

700
701
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
702
703
704
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
705
706
707
708
709
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
710
        for _, class_names in mapping.items():
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
            if not isinstance(class_names, tuple):
                class_names = (class_names,)
                for class_name in class_names:
                    if class_name is None:
                        continue
                    # dummy object is accepted
                    if not hasattr(transformers, class_name):
                        # If the class name is in a model name mapping, let's not check if there is a definition in any modeling
                        # module, if it's a private model defined in this file.
                        if name.endswith("MODEL_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
                        failures.append(
                            f"`{class_name}` appears in the mapping `{name}` but it is not defined in the library."
                        )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Yih-Dar's avatar
Yih-Dar committed
729
730
def check_all_auto_mapping_names_in_config_mapping_names():
    """Check all keys defined in auto mappings (mappings of names) appear in `CONFIG_MAPPING_NAMES`."""
Sylvain Gugger's avatar
Sylvain Gugger committed
731
    # This is where we need to check we have all backends or the check is incomplete.
732
    check_missing_backends()
Yih-Dar's avatar
Yih-Dar committed
733

734
    failures = []
Yih-Dar's avatar
Yih-Dar committed
735
    # `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer` is special, and don't need to follow the rule.
736
    mappings_to_check = {
Yih-Dar's avatar
Yih-Dar committed
737
738
739
740
741
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

742
743
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
744
745
746
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
747
748
749
750
751
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
752
        for model_type in mapping:
Yih-Dar's avatar
Yih-Dar committed
753
754
755
756
757
758
759
760
761
            if model_type not in CONFIG_MAPPING_NAMES:
                failures.append(
                    f"`{model_type}` appears in the mapping `{name}` but it is not defined in the keys of "
                    "`CONFIG_MAPPING_NAMES`."
                )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


762
def check_all_auto_mappings_importable():
Sylvain Gugger's avatar
Sylvain Gugger committed
763
764
    """Check all auto mappings can be imported."""
    # This is where we need to check we have all backends or the check is incomplete.
765
766
767
768
769
770
771
772
773
774
775
776
777
    check_missing_backends()

    failures = []
    mappings_to_check = {}
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

Sylvain Gugger's avatar
Sylvain Gugger committed
778
    for name in mappings_to_check:
779
780
        name = name.replace("_MAPPING_NAMES", "_MAPPING")
        if not hasattr(transformers, name):
781
782
783
784
785
786
            failures.append(f"`{name}`")
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


def check_objects_being_equally_in_main_init():
Sylvain Gugger's avatar
Sylvain Gugger committed
787
788
789
    """
    Check if a (TensorFlow or Flax) object is in the main __init__ iif its counterpart in PyTorch is.
    """
790
791
792
793
794
    attrs = dir(transformers)

    failures = []
    for attr in attrs:
        obj = getattr(transformers, attr)
Sylvain Gugger's avatar
Sylvain Gugger committed
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
        if not hasattr(obj, "__module__") or "models.deprecated" in obj.__module__:
            continue

        module_path = obj.__module__
        module_name = module_path.split(".")[-1]
        module_dir = ".".join(module_path.split(".")[:-1])
        if (
            module_name.startswith("modeling_")
            and not module_name.startswith("modeling_tf_")
            and not module_name.startswith("modeling_flax_")
        ):
            parent_module = sys.modules[module_dir]

            frameworks = []
            if is_tf_available():
                frameworks.append("TF")
            if is_flax_available():
                frameworks.append("Flax")

            for framework in frameworks:
                other_module_path = module_path.replace("modeling_", f"modeling_{framework.lower()}_")
                if os.path.isfile("src/" + other_module_path.replace(".", "/") + ".py"):
                    other_module_name = module_name.replace("modeling_", f"modeling_{framework.lower()}_")
                    other_module = getattr(parent_module, other_module_name)
                    if hasattr(other_module, f"{framework}{attr}"):
                        if not hasattr(transformers, f"{framework}{attr}"):
                            if f"{framework}{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}{attr}")
                    if hasattr(other_module, f"{framework}_{attr}"):
                        if not hasattr(transformers, f"{framework}_{attr}"):
                            if f"{framework}_{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}_{attr}")
827
828
829
830
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
831
832
833
_re_decorator = re.compile(r"^\s*@(\S+)\s+$")


Sylvain Gugger's avatar
Sylvain Gugger committed
834
835
836
837
838
839
840
841
842
843
def check_decorator_order(filename: str) -> List[int]:
    """
    Check that in a given test file, the slow decorator is always last.

    Args:
        filename (`str`): The path to a test file to check.

    Returns:
        `List[int]`: The list of failures as a list of indices where there are problems.
    """
844
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
Sylvain Gugger's avatar
Sylvain Gugger committed
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
        lines = f.readlines()
    decorator_before = None
    errors = []
    for i, line in enumerate(lines):
        search = _re_decorator.search(line)
        if search is not None:
            decorator_name = search.groups()[0]
            if decorator_before is not None and decorator_name.startswith("parameterized"):
                errors.append(i)
            decorator_before = decorator_name
        elif decorator_before is not None:
            decorator_before = None
    return errors


def check_all_decorator_order():
Patrick von Platen's avatar
Patrick von Platen committed
861
    """Check that in all test files, the slow decorator is always last."""
Sylvain Gugger's avatar
Sylvain Gugger committed
862
863
864
865
866
867
868
869
870
    errors = []
    for fname in os.listdir(PATH_TO_TESTS):
        if fname.endswith(".py"):
            filename = os.path.join(PATH_TO_TESTS, fname)
            new_errors = check_decorator_order(filename)
            errors += [f"- {filename}, line {i}" for i in new_errors]
    if len(errors) > 0:
        msg = "\n".join(errors)
        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
871
872
            "The parameterized decorator (and its variants) should always be first, but this is not the case in the"
            f" following files:\n{msg}"
Sylvain Gugger's avatar
Sylvain Gugger committed
873
874
875
        )


Sylvain Gugger's avatar
Sylvain Gugger committed
876
877
878
879
880
881
882
def find_all_documented_objects() -> List[str]:
    """
    Parse the content of all doc files to detect which classes and functions it documents.

    Returns:
        `List[str]`: The list of all object names being documented.
    """
883
884
    documented_obj = []
    for doc_file in Path(PATH_TO_DOC).glob("**/*.rst"):
Julien Plu's avatar
Julien Plu committed
885
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
886
887
888
            content = f.read()
        raw_doc_objs = re.findall(r"(?:autoclass|autofunction):: transformers.(\S+)\s+", content)
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
889
    for doc_file in Path(PATH_TO_DOC).glob("**/*.md"):
Sylvain Gugger's avatar
Sylvain Gugger committed
890
891
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
            content = f.read()
892
        raw_doc_objs = re.findall(r"\[\[autodoc\]\]\s+(\S+)\s+", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
893
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
894
895
896
897
898
899
    return documented_obj


# One good reason for not being documented is to be deprecated. Put in this list deprecated objects.
DEPRECATED_OBJECTS = [
    "AutoModelWithLMHead",
900
    "BartPretrainedModel",
901
902
    "DataCollator",
    "DataCollatorForSOP",
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
    "GlueDataset",
    "GlueDataTrainingArguments",
    "LineByLineTextDataset",
    "LineByLineWithRefDataset",
    "LineByLineWithSOPTextDataset",
    "PretrainedBartModel",
    "PretrainedFSMTModel",
    "SingleSentenceClassificationProcessor",
    "SquadDataTrainingArguments",
    "SquadDataset",
    "SquadExample",
    "SquadFeatures",
    "SquadV1Processor",
    "SquadV2Processor",
    "TFAutoModelWithLMHead",
918
    "TFBartPretrainedModel",
919
920
    "TextDataset",
    "TextDatasetForNextSentencePrediction",
921
    "Wav2Vec2ForMaskedLM",
922
    "Wav2Vec2Tokenizer",
923
924
925
926
927
928
929
930
931
932
    "glue_compute_metrics",
    "glue_convert_examples_to_features",
    "glue_output_modes",
    "glue_processors",
    "glue_tasks_num_labels",
    "squad_convert_examples_to_features",
    "xnli_compute_metrics",
    "xnli_output_modes",
    "xnli_processors",
    "xnli_tasks_num_labels",
933
934
    "TFTrainer",
    "TFTrainingArguments",
935
936
937
938
939
940
941
]

# Exceptionally, some objects should not be documented after all rules passed.
# ONLY PUT SOMETHING IN THIS LIST AS A LAST RESORT!
UNDOCUMENTED_OBJECTS = [
    "AddedToken",  # This is a tokenizers class.
    "BasicTokenizer",  # Internal, should never have been in the main init.
942
    "CharacterTokenizer",  # Internal, should never have been in the main init.
943
    "DPRPretrainedReader",  # Like an Encoder.
Sylvain Gugger's avatar
Sylvain Gugger committed
944
    "DummyObject",  # Just picked by mistake sometimes.
945
    "MecabTokenizer",  # Internal, should never have been in the main init.
946
947
948
949
950
951
952
953
954
955
956
    "ModelCard",  # Internal type.
    "SqueezeBertModule",  # Internal building block (should have been called SqueezeBertLayer)
    "TFDPRPretrainedReader",  # Like an Encoder.
    "TransfoXLCorpus",  # Internal type.
    "WordpieceTokenizer",  # Internal, should never have been in the main init.
    "absl",  # External module
    "add_end_docstrings",  # Internal, should never have been in the main init.
    "add_start_docstrings",  # Internal, should never have been in the main init.
    "convert_tf_weight_name_to_pt_weight_name",  # Internal used to convert model weights
    "logger",  # Internal logger
    "logging",  # External module
957
    "requires_backends",  # Internal function
Jongjyh's avatar
Jongjyh committed
958
    "AltRobertaModel",  # Internal module
959
960
961
962
963
964
965
966
967
]

# This list should be empty. Objects in it should get their own doc page.
SHOULD_HAVE_THEIR_OWN_PAGE = [
    # Benchmarks
    "PyTorchBenchmark",
    "PyTorchBenchmarkArguments",
    "TensorFlowBenchmark",
    "TensorFlowBenchmarkArguments",
968
    "AutoBackbone",
NielsRogge's avatar
NielsRogge committed
969
970
    "BitBackbone",
    "ConvNextBackbone",
Alara Dirik's avatar
Alara Dirik committed
971
    "ConvNextV2Backbone",
972
    "DinatBackbone",
973
    "Dinov2Backbone",
Alara Dirik's avatar
Alara Dirik committed
974
    "FocalNetBackbone",
NielsRogge's avatar
NielsRogge committed
975
    "MaskFormerSwinBackbone",
976
977
    "MaskFormerSwinConfig",
    "MaskFormerSwinModel",
NielsRogge's avatar
NielsRogge committed
978
979
    "NatBackbone",
    "ResNetBackbone",
NielsRogge's avatar
NielsRogge committed
980
    "SwinBackbone",
amyeroberts's avatar
amyeroberts committed
981
982
    "TimmBackbone",
    "TimmBackboneConfig",
NielsRogge's avatar
NielsRogge committed
983
    "VitDetBackbone",
984
985
986
]


Sylvain Gugger's avatar
Sylvain Gugger committed
987
988
def ignore_undocumented(name: str) -> bool:
    """Rules to determine if `name` should be undocumented (returns `True` if it should not be documented)."""
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
    # NOT DOCUMENTED ON PURPOSE.
    # Constants uppercase are not documented.
    if name.isupper():
        return True
    # PreTrainedModels / Encoders / Decoders / Layers / Embeddings / Attention are not documented.
    if (
        name.endswith("PreTrainedModel")
        or name.endswith("Decoder")
        or name.endswith("Encoder")
        or name.endswith("Layer")
        or name.endswith("Embeddings")
        or name.endswith("Attention")
    ):
        return True
    # Submodules are not documented.
    if os.path.isdir(os.path.join(PATH_TO_TRANSFORMERS, name)) or os.path.isfile(
        os.path.join(PATH_TO_TRANSFORMERS, f"{name}.py")
    ):
        return True
    # All load functions are not documented.
    if name.startswith("load_tf") or name.startswith("load_pytorch"):
        return True
    # is_xxx_available functions are not documented.
    if name.startswith("is_") and name.endswith("_available"):
        return True
    # Deprecated objects are not documented.
    if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS:
        return True
    # MMBT model does not really work.
    if name.startswith("MMBT"):
        return True
    if name in SHOULD_HAVE_THEIR_OWN_PAGE:
        return True
    return False


def check_all_objects_are_documented():
Patrick von Platen's avatar
Patrick von Platen committed
1026
    """Check all models are properly documented."""
1027
    documented_objs = find_all_documented_objects()
1028
1029
1030
    modules = transformers._modules
    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]
    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]
1031
1032
1033
1034
1035
    if len(undocumented_objs) > 0:
        raise Exception(
            "The following objects are in the public init so should be documented:\n - "
            + "\n - ".join(undocumented_objs)
        )
1036
    check_docstrings_are_in_md()
1037
1038
1039
1040
1041
1042
    check_model_type_doc_match()


def check_model_type_doc_match():
    """Check all doc pages have a corresponding model type."""
    model_doc_folder = Path(PATH_TO_DOC) / "model_doc"
1043
    model_docs = [m.stem for m in model_doc_folder.glob("*.md")]
1044
1045

    model_types = list(transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys())
1046
    model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types]
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064

    errors = []
    for m in model_docs:
        if m not in model_types and m != "auto":
            close_matches = get_close_matches(m, model_types)
            error_message = f"{m} is not a proper model identifier."
            if len(close_matches) > 0:
                close_matches = "/".join(close_matches)
                error_message += f" Did you mean {close_matches}?"
            errors.append(error_message)

    if len(errors) > 0:
        raise ValueError(
            "Some model doc pages do not match any existing model type:\n"
            + "\n".join(errors)
            + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in "
            "models/auto/configuration_auto.py."
        )
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074


# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`.
_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`")
# Re pattern to catch things between double backquotes.
_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")
# Re pattern to catch example introduction.
_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE)


Sylvain Gugger's avatar
Sylvain Gugger committed
1075
def is_rst_docstring(docstring: str) -> True:
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
    """
    Returns `True` if `docstring` is written in rst.
    """
    if _re_rst_special_words.search(docstring) is not None:
        return True
    if _re_double_backquotes.search(docstring) is not None:
        return True
    if _re_rst_example.search(docstring) is not None:
        return True
    return False


def check_docstrings_are_in_md():
Sylvain Gugger's avatar
Sylvain Gugger committed
1089
    """Check all docstrings are written in md and nor rst."""
1090
1091
    files_with_rst = []
    for file in Path(PATH_TO_TRANSFORMERS).glob("**/*.py"):
1092
        with open(file, encoding="utf-8") as f:
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
            code = f.read()
        docstrings = code.split('"""')

        for idx, docstring in enumerate(docstrings):
            if idx % 2 == 0 or not is_rst_docstring(docstring):
                continue
            files_with_rst.append(file)
            break

    if len(files_with_rst) > 0:
        raise ValueError(
            "The following files have docstrings written in rst:\n"
            + "\n".join([f"- {f}" for f in files_with_rst])
Kamal Raj's avatar
Kamal Raj committed
1106
            + "\nTo fix this run `doc-builder convert path_to_py_file` after installing `doc-builder`\n"
1107
1108
            "(`pip install git+https://github.com/huggingface/doc-builder`)"
        )
1109
1110


1111
def check_deprecated_constant_is_up_to_date():
Sylvain Gugger's avatar
Sylvain Gugger committed
1112
1113
1114
    """
    Check if the constant `DEPRECATED_MODELS` in `models/auto/configuration_auto.py` is up to date.
    """
1115
1116
1117
1118
1119
1120
1121
1122
1123
    deprecated_folder = os.path.join(PATH_TO_TRANSFORMERS, "models", "deprecated")
    deprecated_models = [m for m in os.listdir(deprecated_folder) if not m.startswith("_")]

    constant_to_check = transformers.models.auto.configuration_auto.DEPRECATED_MODELS
    message = []
    missing_models = sorted(set(deprecated_models) - set(constant_to_check))
    if len(missing_models) != 0:
        missing_models = ", ".join(missing_models)
        message.append(
1124
            "The following models are in the deprecated folder, make sure to add them to `DEPRECATED_MODELS` in "
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
            f"`models/auto/configuration_auto.py`: {missing_models}."
        )

    extra_models = sorted(set(constant_to_check) - set(deprecated_models))
    if len(extra_models) != 0:
        extra_models = ", ".join(extra_models)
        message.append(
            "The following models are in the `DEPRECATED_MODELS` constant but not in the deprecated folder. Either "
            f"remove them from the constant or move to the deprecated folder: {extra_models}."
        )

    if len(message) > 0:
        raise Exception("\n".join(message))


1140
def check_repo_quality():
Patrick von Platen's avatar
Patrick von Platen committed
1141
    """Check all models are properly tested and documented."""
1142
1143
    print("Checking all models are included.")
    check_model_list()
1144
1145
    print("Checking all models are public.")
    check_models_are_in_init()
1146
    print("Checking all models are properly tested.")
Sylvain Gugger's avatar
Sylvain Gugger committed
1147
    check_all_decorator_order()
1148
    check_all_models_are_tested()
1149
    print("Checking all objects are properly documented.")
1150
    check_all_objects_are_documented()
1151
1152
    print("Checking all models are in at least one auto class.")
    check_all_models_are_auto_configured()
1153
1154
    print("Checking all names in auto name mappings are defined.")
    check_all_auto_object_names_being_defined()
Yih-Dar's avatar
Yih-Dar committed
1155
1156
    print("Checking all keys in auto name mappings are defined in `CONFIG_MAPPING_NAMES`.")
    check_all_auto_mapping_names_in_config_mapping_names()
1157
1158
    print("Checking all auto mappings could be imported.")
    check_all_auto_mappings_importable()
1159
1160
    print("Checking all objects are equally (across frameworks) in the main __init__.")
    check_objects_being_equally_in_main_init()
1161
1162
    print("Checking the DEPRECATED_MODELS constant is up to date.")
    check_deprecated_constant_is_up_to_date()
1163
1164
1165
1166


if __name__ == "__main__":
    check_repo_quality()