check_repo.py 46.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
"""
Utility that performs several consistency checks on the repo. This includes:
- checking all models are properly defined in the __init__ of models/
- checking all models are in the main __init__
- checking all models are properly tested
- checking all object in the main __init__ are documented
- checking all models are in at least one auto class
- checking all the auto mapping are properly defined (no typos, importable)
- checking the list of deprecated models is up to date

Use from the root of the repo with (as used in `make repo-consistency`):

```bash
python utils/check_repo.py
```

It has no auto-fix mode.
"""
33
34
35
import inspect
import os
import re
36
import sys
Sylvain Gugger's avatar
Sylvain Gugger committed
37
import types
38
import warnings
39
from collections import OrderedDict
40
from difflib import get_close_matches
41
from pathlib import Path
Sylvain Gugger's avatar
Sylvain Gugger committed
42
from typing import List, Tuple
43

44
from transformers import is_flax_available, is_tf_available, is_torch_available
45
from transformers.models.auto import get_values
Yih-Dar's avatar
Yih-Dar committed
46
from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
47
48
49
50
from transformers.models.auto.feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING_NAMES
from transformers.models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING_NAMES
from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES
from transformers.models.auto.tokenization_auto import TOKENIZER_MAPPING_NAMES
51
from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import
52

53
54
55
56
57

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_repo.py
PATH_TO_TRANSFORMERS = "src/transformers"
PATH_TO_TESTS = "tests"
58
PATH_TO_DOC = "docs/source/en"
59

60
61
# Update this list with models that are supposed to be private.
PRIVATE_MODELS = [
Jongjyh's avatar
Jongjyh committed
62
    "AltRobertaModel",
63
    "DPRSpanPredictor",
Daniel Stancl's avatar
Daniel Stancl committed
64
    "LongT5Stack",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
65
    "RealmBertModel",
66
    "T5Stack",
67
    "MT5Stack",
68
    "UMT5Stack",
Susnato Dhar's avatar
Susnato Dhar committed
69
    "Pop2PianoStack",
70
    "SwitchTransformersStack",
71
    "TFDPRSpanPredictor",
72
73
    "MaskFormerSwinModel",
    "MaskFormerSwinPreTrainedModel",
74
75
    "BridgeTowerTextModel",
    "BridgeTowerVisionModel",
Yih-Dar's avatar
Yih-Dar committed
76
77
78
    "Kosmos2TextModel",
    "Kosmos2TextForCausalLM",
    "Kosmos2VisionModel",
Yoach Lacombe's avatar
Yoach Lacombe committed
79
80
81
    "SeamlessM4Tv2TextToUnitModel",
    "SeamlessM4Tv2CodeHifiGan",
    "SeamlessM4Tv2TextToUnitForConditionalGeneration",
82
83
]

84
85
# Update this list for models that are not tested with a comment explaining the reason it should not be.
# Being in this list is an exception and should **not** be the rule.
86
IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [
87
    # models to ignore for not tested
Pablo Montalvo's avatar
Pablo Montalvo committed
88
    "FuyuForCausalLM",  # Not tested fort now
NielsRogge's avatar
NielsRogge committed
89
    "InstructBlipQFormerModel",  # Building part of bigger (tested) model.
90
    "UMT5EncoderModel",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
91
    "Blip2QFormerModel",  # Building part of bigger (tested) model.
92
    "ErnieMForInformationExtraction",
93
    "GraphormerDecoderHead",  # Building part of bigger (tested) model.
94
95
    "JukeboxVQVAE",  # Building part of bigger (tested) model.
    "JukeboxPrior",  # Building part of bigger (tested) model.
96
    "DecisionTransformerGPT2Model",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
97
    "SegformerDecodeHead",  # Building part of bigger (tested) model.
wangpeng's avatar
wangpeng committed
98
    "MgpstrModel",  # Building part of bigger (tested) model.
99
    "BertLMHeadModel",  # Needs to be setup as decoder.
100
    "MegatronBertLMHeadModel",  # Building part of bigger (tested) model.
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
101
102
103
104
    "RealmBertModel",  # Building part of bigger (tested) model.
    "RealmReader",  # Not regular model.
    "RealmScorer",  # Not regular model.
    "RealmForOpenQA",  # Not regular model.
105
106
107
    "ReformerForMaskedLM",  # Needs to be setup as decoder.
    "TFElectraMainLayer",  # Building part of bigger (tested) model (should it be a TFPreTrainedModel ?)
    "TFRobertaForMultipleChoice",  # TODO: fix
108
    "TFRobertaPreLayerNormForMultipleChoice",  # TODO: fix
abhishek thakur's avatar
abhishek thakur committed
109
    "SeparableConv1D",  # Building part of bigger (tested) model.
110
    "FlaxBartForCausalLM",  # Building part of bigger (tested) model.
111
    "FlaxBertForCausalLM",  # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM.
Younes Belkada's avatar
Younes Belkada committed
112
    "OPTDecoderWrapper",
113
    "TFSegformerDecodeHead",  # Not a regular model.
Jongjyh's avatar
Jongjyh committed
114
    "AltRobertaModel",  # Building part of bigger (tested) model.
Younes Belkada's avatar
Younes Belkada committed
115
    "BlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
Matt's avatar
Matt committed
116
    "TFBlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
117
118
    "BridgeTowerTextModel",  # No need to test it as it is tested by BridgeTowerModel model.
    "BridgeTowerVisionModel",  # No need to test it as it is tested by BridgeTowerModel model.
Yoach Lacombe's avatar
Yoach Lacombe committed
119
    "BarkCausalModel",  # Building part of bigger (tested) model.
jiqing-feng's avatar
jiqing-feng committed
120
    "BarkModel",  # Does not have a forward signature - generation tested with integration tests.
121
122
123
    "SeamlessM4TTextToUnitModel",  # Building part of bigger (tested) model.
    "SeamlessM4TCodeHifiGan",  # Building part of bigger (tested) model.
    "SeamlessM4TTextToUnitForConditionalGeneration",  # Building part of bigger (tested) model.
124
125
126
127
128
]

# Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't
# trigger the common tests.
TEST_FILES_WITH_NO_COMMON_TESTS = [
Yih-Dar's avatar
Yih-Dar committed
129
130
131
132
133
134
135
136
137
138
139
140
141
    "models/decision_transformer/test_modeling_decision_transformer.py",
    "models/camembert/test_modeling_camembert.py",
    "models/mt5/test_modeling_flax_mt5.py",
    "models/mbart/test_modeling_mbart.py",
    "models/mt5/test_modeling_mt5.py",
    "models/pegasus/test_modeling_pegasus.py",
    "models/camembert/test_modeling_tf_camembert.py",
    "models/mt5/test_modeling_tf_mt5.py",
    "models/xlm_roberta/test_modeling_tf_xlm_roberta.py",
    "models/xlm_roberta/test_modeling_flax_xlm_roberta.py",
    "models/xlm_prophetnet/test_modeling_xlm_prophetnet.py",
    "models/xlm_roberta/test_modeling_xlm_roberta.py",
    "models/vision_text_dual_encoder/test_modeling_vision_text_dual_encoder.py",
Matt's avatar
Matt committed
142
    "models/vision_text_dual_encoder/test_modeling_tf_vision_text_dual_encoder.py",
Yih-Dar's avatar
Yih-Dar committed
143
144
    "models/vision_text_dual_encoder/test_modeling_flax_vision_text_dual_encoder.py",
    "models/decision_transformer/test_modeling_decision_transformer.py",
Yoach Lacombe's avatar
Yoach Lacombe committed
145
    "models/bark/test_modeling_bark.py",
146
147
]

148
149
# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and
# should **not** be the rule.
150
IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [
151
    # models to ignore for model xxx mapping
152
153
    "AlignTextModel",
    "AlignVisionModel",
154
155
156
157
    "ClapTextModel",
    "ClapTextModelWithProjection",
    "ClapAudioModel",
    "ClapAudioModelWithProjection",
NielsRogge's avatar
NielsRogge committed
158
159
160
    "Blip2ForConditionalGeneration",
    "Blip2QFormerModel",
    "Blip2VisionModel",
161
    "ErnieMForInformationExtraction",
162
    "GitVisionModel",
163
164
    "GraphormerModel",
    "GraphormerForGraphClassification",
Younes Belkada's avatar
Younes Belkada committed
165
166
167
168
169
170
    "BlipForConditionalGeneration",
    "BlipForImageTextRetrieval",
    "BlipForQuestionAnswering",
    "BlipVisionModel",
    "BlipTextLMHeadModel",
    "BlipTextModel",
Jinho Park's avatar
Jinho Park committed
171
172
    "BrosSpadeEEForTokenClassification",
    "BrosSpadeELForTokenClassification",
Matt's avatar
Matt committed
173
174
175
176
177
178
    "TFBlipForConditionalGeneration",
    "TFBlipForImageTextRetrieval",
    "TFBlipForQuestionAnswering",
    "TFBlipVisionModel",
    "TFBlipTextLMHeadModel",
    "TFBlipTextModel",
NielsRogge's avatar
NielsRogge committed
179
    "Swin2SRForImageSuperResolution",
180
181
    "BridgeTowerForImageAndTextRetrieval",
    "BridgeTowerForMaskedLM",
182
    "BridgeTowerForContrastiveLearning",
NielsRogge's avatar
NielsRogge committed
183
184
185
    "CLIPSegForImageSegmentation",
    "CLIPSegVisionModel",
    "CLIPSegTextModel",
Matt's avatar
Matt committed
186
    "EsmForProteinFolding",
187
    "GPTSanJapaneseModel",
188
    "TimeSeriesTransformerForPrediction",
189
    "InformerForPrediction",
190
    "AutoformerForPrediction",
191
192
    "PatchTSTForPretraining",
    "PatchTSTForPrediction",
193
194
    "JukeboxVQVAE",
    "JukeboxPrior",
195
    "SamModel",
NielsRogge's avatar
NielsRogge committed
196
    "DPTForDepthEstimation",
197
    "DecisionTransformerGPT2Model",
NielsRogge's avatar
NielsRogge committed
198
    "GLPNForDepthEstimation",
NielsRogge's avatar
NielsRogge committed
199
200
    "ViltForImagesAndTextClassification",
    "ViltForImageAndTextRetrieval",
201
    "ViltForTokenClassification",
NielsRogge's avatar
NielsRogge committed
202
    "ViltForMaskedLM",
NielsRogge's avatar
NielsRogge committed
203
204
    "PerceiverForMultimodalAutoencoding",
    "PerceiverForOpticalFlow",
NielsRogge's avatar
NielsRogge committed
205
    "SegformerDecodeHead",
206
    "TFSegformerDecodeHead",
Kamal Raj's avatar
Kamal Raj committed
207
    "FlaxBeitForMaskedImageModeling",
NielsRogge's avatar
NielsRogge committed
208
    "BeitForMaskedImageModeling",
209
210
    "ChineseCLIPTextModel",
    "ChineseCLIPVisionModel",
Suraj Patil's avatar
Suraj Patil committed
211
    "CLIPTextModel",
212
    "CLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
213
    "CLIPVisionModel",
214
    "CLIPVisionModelWithProjection",
Susnato Dhar's avatar
Susnato Dhar committed
215
216
    "ClvpForCausalLM",
    "ClvpModel",
217
218
    "GroupViTTextModel",
    "GroupViTVisionModel",
Yih-Dar's avatar
Yih-Dar committed
219
220
    "TFCLIPTextModel",
    "TFCLIPVisionModel",
221
222
    "TFGroupViTTextModel",
    "TFGroupViTVisionModel",
Suraj Patil's avatar
Suraj Patil committed
223
    "FlaxCLIPTextModel",
224
    "FlaxCLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
225
    "FlaxCLIPVisionModel",
226
    "FlaxWav2Vec2ForCTC",
NielsRogge's avatar
NielsRogge committed
227
    "DetrForSegmentation",
Younes Belkada's avatar
Younes Belkada committed
228
229
230
    "Pix2StructVisionModel",
    "Pix2StructTextModel",
    "Pix2StructForConditionalGeneration",
231
    "ConditionalDetrForSegmentation",
232
233
    "DPRReader",
    "FlaubertForQuestionAnswering",
234
235
236
237
    "FlavaImageCodebook",
    "FlavaTextModel",
    "FlavaImageModel",
    "FlavaMultimodalModel",
238
    "GPT2DoubleHeadsModel",
239
    "GPTSw3DoubleHeadsModel",
NielsRogge's avatar
NielsRogge committed
240
241
    "InstructBlipVisionModel",
    "InstructBlipQFormerModel",
242
    "LayoutLMForQuestionAnswering",
Ryokan RI's avatar
Ryokan RI committed
243
    "LukeForMaskedLM",
NielsRogge's avatar
NielsRogge committed
244
245
246
    "LukeForEntityClassification",
    "LukeForEntityPairClassification",
    "LukeForEntitySpanClassification",
wangpeng's avatar
wangpeng committed
247
    "MgpstrModel",
248
    "OpenAIGPTDoubleHeadsModel",
249
250
    "OwlViTTextModel",
    "OwlViTVisionModel",
NielsRogge's avatar
NielsRogge committed
251
252
    "Owlv2TextModel",
    "Owlv2VisionModel",
253
    "OwlViTForObjectDetection",
254
255
    "PatchTSMixerForPrediction",
    "PatchTSMixerForPretraining",
256
257
258
    "RagModel",
    "RagSequenceForGeneration",
    "RagTokenForGeneration",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
259
260
261
262
    "RealmEmbedder",
    "RealmForOpenQA",
    "RealmScorer",
    "RealmReader",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
263
    "TFDPRReader",
264
    "TFGPT2DoubleHeadsModel",
265
    "TFLayoutLMForQuestionAnswering",
266
    "TFOpenAIGPTDoubleHeadsModel",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
267
268
269
    "TFRagModel",
    "TFRagSequenceForGeneration",
    "TFRagTokenForGeneration",
270
    "Wav2Vec2ForCTC",
Patrick von Platen's avatar
Patrick von Platen committed
271
    "HubertForCTC",
272
273
    "SEWForCTC",
    "SEWDForCTC",
274
275
    "XLMForQuestionAnswering",
    "XLNetForQuestionAnswering",
abhishek thakur's avatar
abhishek thakur committed
276
    "SeparableConv1D",
Gunjan Chhablani's avatar
Gunjan Chhablani committed
277
278
279
280
    "VisualBertForRegionToPhraseAlignment",
    "VisualBertForVisualReasoning",
    "VisualBertForQuestionAnswering",
    "VisualBertForMultipleChoice",
Will Rice's avatar
Will Rice committed
281
    "TFWav2Vec2ForCTC",
Will Rice's avatar
Will Rice committed
282
    "TFHubertForCTC",
NielsRogge's avatar
NielsRogge committed
283
284
    "XCLIPVisionModel",
    "XCLIPTextModel",
Jongjyh's avatar
Jongjyh committed
285
286
287
    "AltCLIPTextModel",
    "AltCLIPVisionModel",
    "AltRobertaModel",
Zineng Tang's avatar
Zineng Tang committed
288
    "TvltForAudioVisualClassification",
Yoach Lacombe's avatar
Yoach Lacombe committed
289
290
291
292
293
294
    "BarkCausalModel",
    "BarkCoarseModel",
    "BarkFineModel",
    "BarkSemanticModel",
    "MusicgenModel",
    "MusicgenForConditionalGeneration",
295
296
297
    "SpeechT5ForSpeechToSpeech",
    "SpeechT5ForTextToSpeech",
    "SpeechT5HifiGan",
NielsRogge's avatar
NielsRogge committed
298
    "VitMatteForImageMatting",
299
300
301
302
    "SeamlessM4TTextToUnitModel",
    "SeamlessM4TTextToUnitForConditionalGeneration",
    "SeamlessM4TCodeHifiGan",
    "SeamlessM4TForSpeechToSpeech",  # no auto class for speech-to-speech
jiqing-feng's avatar
jiqing-feng committed
303
    "TvpForVideoGrounding",
Yoach Lacombe's avatar
Yoach Lacombe committed
304
305
306
307
    "SeamlessM4Tv2NARTextToUnitModel",
    "SeamlessM4Tv2NARTextToUnitForConditionalGeneration",
    "SeamlessM4Tv2CodeHifiGan",
    "SeamlessM4Tv2ForSpeechToSpeech",  # no auto class for speech-to-speech
308
309
]

310
# DO NOT edit this list!
Sylvain Gugger's avatar
Sylvain Gugger committed
311
# (The corresponding pytorch objects should never have been in the main `__init__`, but it's too late to remove)
312
313
314
315
316
317
318
319
320
321
322
323
324
OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK = [
    "FlaxBertLayer",
    "FlaxBigBirdLayer",
    "FlaxRoFormerLayer",
    "TFBertLayer",
    "TFLxmertEncoder",
    "TFLxmertXLayer",
    "TFMPNetLayer",
    "TFMobileBertLayer",
    "TFSegformerLayer",
    "TFViTMAELayer",
]

Sylvain Gugger's avatar
Sylvain Gugger committed
325
# Update this list for models that have multiple model types for the same model doc.
326
327
328
329
MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(
    [
        ("data2vec-text", "data2vec"),
        ("data2vec-audio", "data2vec"),
330
        ("data2vec-vision", "data2vec"),
NielsRogge's avatar
NielsRogge committed
331
        ("donut-swin", "donut"),
332
333
334
335
    ]
)


336
# This is to make sure the transformers module imported is the one in the repo.
337
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
338
339


340
def check_missing_backends():
Sylvain Gugger's avatar
Sylvain Gugger committed
341
342
343
344
    """
    Checks if all backends are installed (otherwise the check of this script is incomplete). Will error in the CI if
    that's not the case but only throw a warning for users running this.
    """
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
    missing_backends = []
    if not is_torch_available():
        missing_backends.append("PyTorch")
    if not is_tf_available():
        missing_backends.append("TensorFlow")
    if not is_flax_available():
        missing_backends.append("Flax")
    if len(missing_backends) > 0:
        missing = ", ".join(missing_backends)
        if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES:
            raise Exception(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}."
            )
        else:
            warnings.warn(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you "
                "didn't make any change in one of those backends modeling files, you should probably execute the "
                "command above to be on the safe side."
            )


368
def check_model_list():
Sylvain Gugger's avatar
Sylvain Gugger committed
369
370
371
    """
    Checks the model listed as subfolders of `models` match the models available in `transformers.models`.
    """
372
373
374
375
    # Get the models from the directory structure of `src/transformers/models/`
    models_dir = os.path.join(PATH_TO_TRANSFORMERS, "models")
    _models = []
    for model in os.listdir(models_dir):
Sylvain Gugger's avatar
Sylvain Gugger committed
376
377
        if model == "deprecated":
            continue
378
379
380
381
        model_dir = os.path.join(models_dir, model)
        if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir):
            _models.append(model)

Sylvain Gugger's avatar
Sylvain Gugger committed
382
    # Get the models in the submodule `transformers.models`
383
384
    models = [model for model in dir(transformers.models) if not model.startswith("__")]

385
    missing_models = sorted(set(_models).difference(models))
386
387
388
389
390
391
    if missing_models:
        raise Exception(
            f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}."
        )


392
393
# If some modeling modules should be ignored for all checks, they should be added in the nested list
# _ignore_modules of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
394
395
def get_model_modules() -> List[str]:
    """Get all the model modules inside the transformers library (except deprecated models)."""
396
397
398
399
400
401
402
403
    _ignore_modules = [
        "modeling_auto",
        "modeling_encoder_decoder",
        "modeling_marian",
        "modeling_mmbt",
        "modeling_outputs",
        "modeling_retribert",
        "modeling_utils",
Sylvain Gugger's avatar
Sylvain Gugger committed
404
        "modeling_flax_auto",
405
        "modeling_flax_encoder_decoder",
Stas Bekman's avatar
Stas Bekman committed
406
        "modeling_flax_utils",
407
        "modeling_speech_encoder_decoder",
408
        "modeling_flax_speech_encoder_decoder",
409
        "modeling_flax_vision_encoder_decoder",
amyeroberts's avatar
amyeroberts committed
410
        "modeling_timm_backbone",
411
        "modeling_tf_auto",
412
        "modeling_tf_encoder_decoder",
413
414
415
        "modeling_tf_outputs",
        "modeling_tf_pytorch_utils",
        "modeling_tf_utils",
416
        "modeling_tf_vision_encoder_decoder",
417
        "modeling_vision_encoder_decoder",
418
419
    ]
    modules = []
Sylvain Gugger's avatar
Sylvain Gugger committed
420
421
    for model in dir(transformers.models):
        # There are some magic dunder attributes in the dir, we ignore them
Sylvain Gugger's avatar
Sylvain Gugger committed
422
423
424
425
426
427
428
429
430
        if model == "deprecated" or model.startswith("__"):
            continue

        model_module = getattr(transformers.models, model)
        for submodule in dir(model_module):
            if submodule.startswith("modeling") and submodule not in _ignore_modules:
                modeling_module = getattr(model_module, submodule)
                if inspect.ismodule(modeling_module):
                    modules.append(modeling_module)
431
432
433
    return modules


Sylvain Gugger's avatar
Sylvain Gugger committed
434
435
436
437
438
439
440
441
442
443
444
445
446
def get_models(module: types.ModuleType, include_pretrained: bool = False) -> List[Tuple[str, type]]:
    """
    Get the objects in a module that are models.

    Args:
        module (`types.ModuleType`):
            The module from which we are extracting models.
        include_pretrained (`bool`, *optional*, defaults to `False`):
            Whether or not to include the `PreTrainedModel` subclass (like `BertPreTrainedModel`) or not.

    Returns:
        List[Tuple[str, type]]: List of models as tuples (class name, actual class).
    """
447
    models = []
448
    model_classes = (transformers.PreTrainedModel, transformers.TFPreTrainedModel, transformers.FlaxPreTrainedModel)
449
    for attr_name in dir(module):
450
        if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name):
451
452
453
454
455
456
457
            continue
        attr = getattr(module, attr_name)
        if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__:
            models.append((attr_name, attr))
    return models


Sylvain Gugger's avatar
Sylvain Gugger committed
458
459
460
461
def is_building_block(model: str) -> bool:
    """
    Returns `True` if a model is a building block part of a bigger model.
    """
462
463
464
465
466
467
    if model.endswith("Wrapper"):
        return True
    if model.endswith("Encoder"):
        return True
    if model.endswith("Decoder"):
        return True
468
469
    if model.endswith("Prenet"):
        return True
Sylvain Gugger's avatar
Sylvain Gugger committed
470
471
472
473
474
475
476


def is_a_private_model(model: str) -> bool:
    """Returns `True` if the model should not be in the main init."""
    if model in PRIVATE_MODELS:
        return True
    return is_building_block(model)
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493


def check_models_are_in_init():
    """Checks all models defined in the library are in the main init."""
    models_not_in_init = []
    dir_transformers = dir(transformers)
    for module in get_model_modules():
        models_not_in_init += [
            model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers
        ]

    # Remove private models
    models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)]
    if len(models_not_in_init) > 0:
        raise Exception(f"The following models should be in the main init: {','.join(models_not_in_init)}.")


494
495
# If some test_modeling files should be ignored when checking models are all tested, they should be added in the
# nested list _ignore_files of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
496
497
498
def get_model_test_files() -> List[str]:
    """
    Get the model test files.
Yih-Dar's avatar
Yih-Dar committed
499

Sylvain Gugger's avatar
Sylvain Gugger committed
500
501
502
503
    Returns:
        `List[str]`: The list of test files. The returned files will NOT contain the `tests` (i.e. `PATH_TO_TESTS`
        defined in this script). They will be considered as paths relative to `tests`. A caller has to use
        `os.path.join(PATH_TO_TESTS, ...)` to access the files.
Yih-Dar's avatar
Yih-Dar committed
504
505
    """

506
507
508
    _ignore_files = [
        "test_modeling_common",
        "test_modeling_encoder_decoder",
509
        "test_modeling_flax_encoder_decoder",
510
        "test_modeling_flax_speech_encoder_decoder",
511
512
        "test_modeling_marian",
        "test_modeling_tf_common",
513
        "test_modeling_tf_encoder_decoder",
514
515
    ]
    test_files = []
Yih-Dar's avatar
Yih-Dar committed
516
517
518
519
520
521
522
523
524
525
526
527
    model_test_root = os.path.join(PATH_TO_TESTS, "models")
    model_test_dirs = []
    for x in os.listdir(model_test_root):
        x = os.path.join(model_test_root, x)
        if os.path.isdir(x):
            model_test_dirs.append(x)

    for target_dir in [PATH_TO_TESTS] + model_test_dirs:
        for file_or_dir in os.listdir(target_dir):
            path = os.path.join(target_dir, file_or_dir)
            if os.path.isfile(path):
                filename = os.path.split(path)[-1]
528
                if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files:
Yih-Dar's avatar
Yih-Dar committed
529
530
531
                    file = os.path.join(*path.split(os.sep)[1:])
                    test_files.append(file)

532
533
534
535
536
    return test_files


# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class
# for the all_model_classes variable.
Sylvain Gugger's avatar
Sylvain Gugger committed
537
538
539
540
541
542
543
544
545
546
547
def find_tested_models(test_file: str) -> List[str]:
    """
    Parse the content of test_file to detect what's in `all_model_classes`. This detects the models that inherit from
    the common test class.

    Args:
        test_file (`str`): The path to the test file to check

    Returns:
        `List[str]`: The list of models tested in that file.
    """
548
    with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f:
549
        content = f.read()
Sylvain Gugger's avatar
Sylvain Gugger committed
550
    all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content)
551
552
    # Check with one less parenthesis as well
    all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
553
    if len(all_models) > 0:
554
        model_tested = []
Sylvain Gugger's avatar
Sylvain Gugger committed
555
556
557
558
559
        for entry in all_models:
            for line in entry.split(","):
                name = line.strip()
                if len(name) > 0:
                    model_tested.append(name)
560
561
562
        return model_tested


Sylvain Gugger's avatar
Sylvain Gugger committed
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
def should_be_tested(model_name: str) -> bool:
    """
    Whether or not a model should be tested.
    """
    if model_name in IGNORE_NON_TESTED:
        return False
    return not is_building_block(model_name)


def check_models_are_tested(module: types.ModuleType, test_file: str) -> List[str]:
    """Check models defined in a module are all tested in a given file.

    Args:
        module (`types.ModuleType`): The module in which we get the models.
        test_file (`str`): The path to the file where the module is tested.

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
582
    # XxxPreTrainedModel are not tested
583
584
585
    defined_models = get_models(module)
    tested_models = find_tested_models(test_file)
    if tested_models is None:
586
        if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS:
587
588
589
590
591
592
593
594
            return
        return [
            f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. "
            + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file "
            + "`utils/check_repo.py`."
        ]
    failures = []
    for model_name, _ in defined_models:
Sylvain Gugger's avatar
Sylvain Gugger committed
595
        if model_name not in tested_models and should_be_tested(model_name):
596
597
598
599
600
601
602
603
604
605
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not tested in "
                + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file."
                + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`"
                + "in the file `utils/check_repo.py`."
            )
    return failures


def check_all_models_are_tested():
Patrick von Platen's avatar
Patrick von Platen committed
606
    """Check all models are properly tested."""
607
608
609
610
    modules = get_model_modules()
    test_files = get_model_test_files()
    failures = []
    for module in modules:
Sylvain Gugger's avatar
Sylvain Gugger committed
611
        # Matches a module to its test file.
612
613
        test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file]
        if len(test_file) == 0:
614
            failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")
615
616
617
618
        elif len(test_file) > 1:
            failures.append(f"{module.__name__} has several test files: {test_file}.")
        else:
            test_file = test_file[0]
619
620
621
            new_failures = check_models_are_tested(module, test_file)
            if new_failures is not None:
                failures += new_failures
622
623
624
625
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
626
def get_all_auto_configured_models() -> List[str]:
Patrick von Platen's avatar
Patrick von Platen committed
627
    """Return the list of all models in at least one auto class."""
628
    result = set()  # To avoid duplicates we concatenate all model classes in a set.
629
630
    if is_torch_available():
        for attr_name in dir(transformers.models.auto.modeling_auto):
631
            if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"):
632
633
634
                result = result | set(get_values(getattr(transformers.models.auto.modeling_auto, attr_name)))
    if is_tf_available():
        for attr_name in dir(transformers.models.auto.modeling_tf_auto):
635
            if attr_name.startswith("TF_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
636
637
638
                result = result | set(get_values(getattr(transformers.models.auto.modeling_tf_auto, attr_name)))
    if is_flax_available():
        for attr_name in dir(transformers.models.auto.modeling_flax_auto):
639
            if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
640
                result = result | set(get_values(getattr(transformers.models.auto.modeling_flax_auto, attr_name)))
641
    return list(result)
642
643


Sylvain Gugger's avatar
Sylvain Gugger committed
644
645
def ignore_unautoclassed(model_name: str) -> bool:
    """Rules to determine if a model should be in an auto class."""
646
647
648
649
650
651
652
653
654
    # Special white list
    if model_name in IGNORE_NON_AUTO_CONFIGURED:
        return True
    # Encoder and Decoder should be ignored
    if "Encoder" in model_name or "Decoder" in model_name:
        return True
    return False


Sylvain Gugger's avatar
Sylvain Gugger committed
655
656
657
658
659
660
661
662
663
664
665
666
667
def check_models_are_auto_configured(module: types.ModuleType, all_auto_models: List[str]) -> List[str]:
    """
    Check models defined in module are each in an auto class.

    Args:
        module (`types.ModuleType`):
            The module in which we get the models.
        all_auto_models (`List[str]`):
            The list of all models in an auto class (as obtained with `get_all_auto_configured_models()`).

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
668
669
670
    defined_models = get_models(module)
    failures = []
    for model_name, _ in defined_models:
671
        if model_name not in all_auto_models and not ignore_unautoclassed(model_name):
672
673
674
675
676
677
678
679
680
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "
                "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "
                "`utils/check_repo.py`."
            )
    return failures


def check_all_models_are_auto_configured():
Patrick von Platen's avatar
Patrick von Platen committed
681
    """Check all models are each in an auto class."""
Sylvain Gugger's avatar
Sylvain Gugger committed
682
    # This is where we need to check we have all backends or the check is incomplete.
683
    check_missing_backends()
684
685
686
687
688
689
690
691
692
693
694
    modules = get_model_modules()
    all_auto_models = get_all_auto_configured_models()
    failures = []
    for module in modules:
        new_failures = check_models_are_auto_configured(module, all_auto_models)
        if new_failures is not None:
            failures += new_failures
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


695
696
def check_all_auto_object_names_being_defined():
    """Check all names defined in auto (name) mappings exist in the library."""
Sylvain Gugger's avatar
Sylvain Gugger committed
697
    # This is where we need to check we have all backends or the check is incomplete.
698
    check_missing_backends()
699

700
    failures = []
701
    mappings_to_check = {
702
703
704
705
706
707
        "TOKENIZER_MAPPING_NAMES": TOKENIZER_MAPPING_NAMES,
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

708
709
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
710
711
712
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
713
714
715
716
717
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
718
        for _, class_names in mapping.items():
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
            if not isinstance(class_names, tuple):
                class_names = (class_names,)
                for class_name in class_names:
                    if class_name is None:
                        continue
                    # dummy object is accepted
                    if not hasattr(transformers, class_name):
                        # If the class name is in a model name mapping, let's not check if there is a definition in any modeling
                        # module, if it's a private model defined in this file.
                        if name.endswith("MODEL_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
                        failures.append(
                            f"`{class_name}` appears in the mapping `{name}` but it is not defined in the library."
                        )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Yih-Dar's avatar
Yih-Dar committed
737
738
def check_all_auto_mapping_names_in_config_mapping_names():
    """Check all keys defined in auto mappings (mappings of names) appear in `CONFIG_MAPPING_NAMES`."""
Sylvain Gugger's avatar
Sylvain Gugger committed
739
    # This is where we need to check we have all backends or the check is incomplete.
740
    check_missing_backends()
Yih-Dar's avatar
Yih-Dar committed
741

742
    failures = []
Yih-Dar's avatar
Yih-Dar committed
743
    # `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer` is special, and don't need to follow the rule.
744
    mappings_to_check = {
Yih-Dar's avatar
Yih-Dar committed
745
746
747
748
749
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

750
751
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
752
753
754
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
755
756
757
758
759
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
760
        for model_type in mapping:
Yih-Dar's avatar
Yih-Dar committed
761
762
763
764
765
766
767
768
769
            if model_type not in CONFIG_MAPPING_NAMES:
                failures.append(
                    f"`{model_type}` appears in the mapping `{name}` but it is not defined in the keys of "
                    "`CONFIG_MAPPING_NAMES`."
                )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


770
def check_all_auto_mappings_importable():
Sylvain Gugger's avatar
Sylvain Gugger committed
771
772
    """Check all auto mappings can be imported."""
    # This is where we need to check we have all backends or the check is incomplete.
773
774
775
776
777
778
779
780
781
782
783
784
785
    check_missing_backends()

    failures = []
    mappings_to_check = {}
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

Sylvain Gugger's avatar
Sylvain Gugger committed
786
    for name in mappings_to_check:
787
788
        name = name.replace("_MAPPING_NAMES", "_MAPPING")
        if not hasattr(transformers, name):
789
790
791
792
793
794
            failures.append(f"`{name}`")
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


def check_objects_being_equally_in_main_init():
Sylvain Gugger's avatar
Sylvain Gugger committed
795
796
797
    """
    Check if a (TensorFlow or Flax) object is in the main __init__ iif its counterpart in PyTorch is.
    """
798
799
800
801
802
    attrs = dir(transformers)

    failures = []
    for attr in attrs:
        obj = getattr(transformers, attr)
Sylvain Gugger's avatar
Sylvain Gugger committed
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
        if not hasattr(obj, "__module__") or "models.deprecated" in obj.__module__:
            continue

        module_path = obj.__module__
        module_name = module_path.split(".")[-1]
        module_dir = ".".join(module_path.split(".")[:-1])
        if (
            module_name.startswith("modeling_")
            and not module_name.startswith("modeling_tf_")
            and not module_name.startswith("modeling_flax_")
        ):
            parent_module = sys.modules[module_dir]

            frameworks = []
            if is_tf_available():
                frameworks.append("TF")
            if is_flax_available():
                frameworks.append("Flax")

            for framework in frameworks:
                other_module_path = module_path.replace("modeling_", f"modeling_{framework.lower()}_")
                if os.path.isfile("src/" + other_module_path.replace(".", "/") + ".py"):
                    other_module_name = module_name.replace("modeling_", f"modeling_{framework.lower()}_")
                    other_module = getattr(parent_module, other_module_name)
                    if hasattr(other_module, f"{framework}{attr}"):
                        if not hasattr(transformers, f"{framework}{attr}"):
                            if f"{framework}{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}{attr}")
                    if hasattr(other_module, f"{framework}_{attr}"):
                        if not hasattr(transformers, f"{framework}_{attr}"):
                            if f"{framework}_{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}_{attr}")
835
836
837
838
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
839
840
841
_re_decorator = re.compile(r"^\s*@(\S+)\s+$")


Sylvain Gugger's avatar
Sylvain Gugger committed
842
843
844
845
846
847
848
849
850
851
def check_decorator_order(filename: str) -> List[int]:
    """
    Check that in a given test file, the slow decorator is always last.

    Args:
        filename (`str`): The path to a test file to check.

    Returns:
        `List[int]`: The list of failures as a list of indices where there are problems.
    """
852
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
Sylvain Gugger's avatar
Sylvain Gugger committed
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
        lines = f.readlines()
    decorator_before = None
    errors = []
    for i, line in enumerate(lines):
        search = _re_decorator.search(line)
        if search is not None:
            decorator_name = search.groups()[0]
            if decorator_before is not None and decorator_name.startswith("parameterized"):
                errors.append(i)
            decorator_before = decorator_name
        elif decorator_before is not None:
            decorator_before = None
    return errors


def check_all_decorator_order():
Patrick von Platen's avatar
Patrick von Platen committed
869
    """Check that in all test files, the slow decorator is always last."""
Sylvain Gugger's avatar
Sylvain Gugger committed
870
871
872
873
874
875
876
877
878
    errors = []
    for fname in os.listdir(PATH_TO_TESTS):
        if fname.endswith(".py"):
            filename = os.path.join(PATH_TO_TESTS, fname)
            new_errors = check_decorator_order(filename)
            errors += [f"- {filename}, line {i}" for i in new_errors]
    if len(errors) > 0:
        msg = "\n".join(errors)
        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
879
880
            "The parameterized decorator (and its variants) should always be first, but this is not the case in the"
            f" following files:\n{msg}"
Sylvain Gugger's avatar
Sylvain Gugger committed
881
882
883
        )


Sylvain Gugger's avatar
Sylvain Gugger committed
884
885
886
887
888
889
890
def find_all_documented_objects() -> List[str]:
    """
    Parse the content of all doc files to detect which classes and functions it documents.

    Returns:
        `List[str]`: The list of all object names being documented.
    """
891
892
    documented_obj = []
    for doc_file in Path(PATH_TO_DOC).glob("**/*.rst"):
Julien Plu's avatar
Julien Plu committed
893
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
894
895
896
            content = f.read()
        raw_doc_objs = re.findall(r"(?:autoclass|autofunction):: transformers.(\S+)\s+", content)
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
897
    for doc_file in Path(PATH_TO_DOC).glob("**/*.md"):
Sylvain Gugger's avatar
Sylvain Gugger committed
898
899
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
            content = f.read()
900
        raw_doc_objs = re.findall(r"\[\[autodoc\]\]\s+(\S+)\s+", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
901
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
902
903
904
905
906
907
    return documented_obj


# One good reason for not being documented is to be deprecated. Put in this list deprecated objects.
DEPRECATED_OBJECTS = [
    "AutoModelWithLMHead",
908
    "BartPretrainedModel",
909
910
    "DataCollator",
    "DataCollatorForSOP",
911
912
913
914
915
    "GlueDataset",
    "GlueDataTrainingArguments",
    "LineByLineTextDataset",
    "LineByLineWithRefDataset",
    "LineByLineWithSOPTextDataset",
Steven Liu's avatar
Steven Liu committed
916
    "NerPipeline",
917
918
919
920
921
922
923
924
925
926
    "PretrainedBartModel",
    "PretrainedFSMTModel",
    "SingleSentenceClassificationProcessor",
    "SquadDataTrainingArguments",
    "SquadDataset",
    "SquadExample",
    "SquadFeatures",
    "SquadV1Processor",
    "SquadV2Processor",
    "TFAutoModelWithLMHead",
927
    "TFBartPretrainedModel",
928
929
    "TextDataset",
    "TextDatasetForNextSentencePrediction",
930
    "Wav2Vec2ForMaskedLM",
931
    "Wav2Vec2Tokenizer",
932
933
934
935
936
937
938
939
940
941
    "glue_compute_metrics",
    "glue_convert_examples_to_features",
    "glue_output_modes",
    "glue_processors",
    "glue_tasks_num_labels",
    "squad_convert_examples_to_features",
    "xnli_compute_metrics",
    "xnli_output_modes",
    "xnli_processors",
    "xnli_tasks_num_labels",
942
943
    "TFTrainer",
    "TFTrainingArguments",
944
945
946
947
948
949
950
]

# Exceptionally, some objects should not be documented after all rules passed.
# ONLY PUT SOMETHING IN THIS LIST AS A LAST RESORT!
UNDOCUMENTED_OBJECTS = [
    "AddedToken",  # This is a tokenizers class.
    "BasicTokenizer",  # Internal, should never have been in the main init.
951
    "CharacterTokenizer",  # Internal, should never have been in the main init.
952
    "DPRPretrainedReader",  # Like an Encoder.
Sylvain Gugger's avatar
Sylvain Gugger committed
953
    "DummyObject",  # Just picked by mistake sometimes.
954
    "MecabTokenizer",  # Internal, should never have been in the main init.
955
956
957
958
959
960
961
962
963
964
965
    "ModelCard",  # Internal type.
    "SqueezeBertModule",  # Internal building block (should have been called SqueezeBertLayer)
    "TFDPRPretrainedReader",  # Like an Encoder.
    "TransfoXLCorpus",  # Internal type.
    "WordpieceTokenizer",  # Internal, should never have been in the main init.
    "absl",  # External module
    "add_end_docstrings",  # Internal, should never have been in the main init.
    "add_start_docstrings",  # Internal, should never have been in the main init.
    "convert_tf_weight_name_to_pt_weight_name",  # Internal used to convert model weights
    "logger",  # Internal logger
    "logging",  # External module
966
    "requires_backends",  # Internal function
Jongjyh's avatar
Jongjyh committed
967
    "AltRobertaModel",  # Internal module
968
969
970
971
972
973
974
975
976
]

# This list should be empty. Objects in it should get their own doc page.
SHOULD_HAVE_THEIR_OWN_PAGE = [
    # Benchmarks
    "PyTorchBenchmark",
    "PyTorchBenchmarkArguments",
    "TensorFlowBenchmark",
    "TensorFlowBenchmarkArguments",
977
    "AutoBackbone",
NielsRogge's avatar
NielsRogge committed
978
    "BeitBackbone",
NielsRogge's avatar
NielsRogge committed
979
980
    "BitBackbone",
    "ConvNextBackbone",
Alara Dirik's avatar
Alara Dirik committed
981
    "ConvNextV2Backbone",
982
    "DinatBackbone",
983
    "Dinov2Backbone",
Alara Dirik's avatar
Alara Dirik committed
984
    "FocalNetBackbone",
NielsRogge's avatar
NielsRogge committed
985
    "MaskFormerSwinBackbone",
986
987
    "MaskFormerSwinConfig",
    "MaskFormerSwinModel",
NielsRogge's avatar
NielsRogge committed
988
989
    "NatBackbone",
    "ResNetBackbone",
NielsRogge's avatar
NielsRogge committed
990
    "SwinBackbone",
amyeroberts's avatar
amyeroberts committed
991
992
    "TimmBackbone",
    "TimmBackboneConfig",
NielsRogge's avatar
NielsRogge committed
993
    "VitDetBackbone",
994
995
996
]


Sylvain Gugger's avatar
Sylvain Gugger committed
997
998
def ignore_undocumented(name: str) -> bool:
    """Rules to determine if `name` should be undocumented (returns `True` if it should not be documented)."""
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
    # NOT DOCUMENTED ON PURPOSE.
    # Constants uppercase are not documented.
    if name.isupper():
        return True
    # PreTrainedModels / Encoders / Decoders / Layers / Embeddings / Attention are not documented.
    if (
        name.endswith("PreTrainedModel")
        or name.endswith("Decoder")
        or name.endswith("Encoder")
        or name.endswith("Layer")
        or name.endswith("Embeddings")
        or name.endswith("Attention")
    ):
        return True
    # Submodules are not documented.
    if os.path.isdir(os.path.join(PATH_TO_TRANSFORMERS, name)) or os.path.isfile(
        os.path.join(PATH_TO_TRANSFORMERS, f"{name}.py")
    ):
        return True
    # All load functions are not documented.
    if name.startswith("load_tf") or name.startswith("load_pytorch"):
        return True
    # is_xxx_available functions are not documented.
    if name.startswith("is_") and name.endswith("_available"):
        return True
    # Deprecated objects are not documented.
    if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS:
        return True
    # MMBT model does not really work.
    if name.startswith("MMBT"):
        return True
    if name in SHOULD_HAVE_THEIR_OWN_PAGE:
        return True
    return False


def check_all_objects_are_documented():
Patrick von Platen's avatar
Patrick von Platen committed
1036
    """Check all models are properly documented."""
1037
    documented_objs = find_all_documented_objects()
1038
1039
1040
    modules = transformers._modules
    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]
    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]
1041
1042
1043
1044
1045
    if len(undocumented_objs) > 0:
        raise Exception(
            "The following objects are in the public init so should be documented:\n - "
            + "\n - ".join(undocumented_objs)
        )
1046
    check_docstrings_are_in_md()
1047
1048
1049
1050
1051
1052
    check_model_type_doc_match()


def check_model_type_doc_match():
    """Check all doc pages have a corresponding model type."""
    model_doc_folder = Path(PATH_TO_DOC) / "model_doc"
1053
    model_docs = [m.stem for m in model_doc_folder.glob("*.md")]
1054
1055

    model_types = list(transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys())
1056
    model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types]
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074

    errors = []
    for m in model_docs:
        if m not in model_types and m != "auto":
            close_matches = get_close_matches(m, model_types)
            error_message = f"{m} is not a proper model identifier."
            if len(close_matches) > 0:
                close_matches = "/".join(close_matches)
                error_message += f" Did you mean {close_matches}?"
            errors.append(error_message)

    if len(errors) > 0:
        raise ValueError(
            "Some model doc pages do not match any existing model type:\n"
            + "\n".join(errors)
            + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in "
            "models/auto/configuration_auto.py."
        )
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084


# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`.
_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`")
# Re pattern to catch things between double backquotes.
_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")
# Re pattern to catch example introduction.
_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE)


Sylvain Gugger's avatar
Sylvain Gugger committed
1085
def is_rst_docstring(docstring: str) -> True:
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
    """
    Returns `True` if `docstring` is written in rst.
    """
    if _re_rst_special_words.search(docstring) is not None:
        return True
    if _re_double_backquotes.search(docstring) is not None:
        return True
    if _re_rst_example.search(docstring) is not None:
        return True
    return False


def check_docstrings_are_in_md():
Sylvain Gugger's avatar
Sylvain Gugger committed
1099
    """Check all docstrings are written in md and nor rst."""
1100
1101
    files_with_rst = []
    for file in Path(PATH_TO_TRANSFORMERS).glob("**/*.py"):
1102
        with open(file, encoding="utf-8") as f:
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
            code = f.read()
        docstrings = code.split('"""')

        for idx, docstring in enumerate(docstrings):
            if idx % 2 == 0 or not is_rst_docstring(docstring):
                continue
            files_with_rst.append(file)
            break

    if len(files_with_rst) > 0:
        raise ValueError(
            "The following files have docstrings written in rst:\n"
            + "\n".join([f"- {f}" for f in files_with_rst])
Kamal Raj's avatar
Kamal Raj committed
1116
            + "\nTo fix this run `doc-builder convert path_to_py_file` after installing `doc-builder`\n"
1117
1118
            "(`pip install git+https://github.com/huggingface/doc-builder`)"
        )
1119
1120


1121
def check_deprecated_constant_is_up_to_date():
Sylvain Gugger's avatar
Sylvain Gugger committed
1122
1123
1124
    """
    Check if the constant `DEPRECATED_MODELS` in `models/auto/configuration_auto.py` is up to date.
    """
1125
1126
1127
1128
1129
1130
1131
1132
1133
    deprecated_folder = os.path.join(PATH_TO_TRANSFORMERS, "models", "deprecated")
    deprecated_models = [m for m in os.listdir(deprecated_folder) if not m.startswith("_")]

    constant_to_check = transformers.models.auto.configuration_auto.DEPRECATED_MODELS
    message = []
    missing_models = sorted(set(deprecated_models) - set(constant_to_check))
    if len(missing_models) != 0:
        missing_models = ", ".join(missing_models)
        message.append(
1134
            "The following models are in the deprecated folder, make sure to add them to `DEPRECATED_MODELS` in "
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
            f"`models/auto/configuration_auto.py`: {missing_models}."
        )

    extra_models = sorted(set(constant_to_check) - set(deprecated_models))
    if len(extra_models) != 0:
        extra_models = ", ".join(extra_models)
        message.append(
            "The following models are in the `DEPRECATED_MODELS` constant but not in the deprecated folder. Either "
            f"remove them from the constant or move to the deprecated folder: {extra_models}."
        )

    if len(message) > 0:
        raise Exception("\n".join(message))


1150
def check_repo_quality():
Patrick von Platen's avatar
Patrick von Platen committed
1151
    """Check all models are properly tested and documented."""
1152
1153
    print("Checking all models are included.")
    check_model_list()
1154
1155
    print("Checking all models are public.")
    check_models_are_in_init()
1156
    print("Checking all models are properly tested.")
Sylvain Gugger's avatar
Sylvain Gugger committed
1157
    check_all_decorator_order()
1158
    check_all_models_are_tested()
1159
    print("Checking all objects are properly documented.")
1160
    check_all_objects_are_documented()
1161
1162
    print("Checking all models are in at least one auto class.")
    check_all_models_are_auto_configured()
1163
1164
    print("Checking all names in auto name mappings are defined.")
    check_all_auto_object_names_being_defined()
Yih-Dar's avatar
Yih-Dar committed
1165
1166
    print("Checking all keys in auto name mappings are defined in `CONFIG_MAPPING_NAMES`.")
    check_all_auto_mapping_names_in_config_mapping_names()
1167
1168
    print("Checking all auto mappings could be imported.")
    check_all_auto_mappings_importable()
1169
1170
    print("Checking all objects are equally (across frameworks) in the main __init__.")
    check_objects_being_equally_in_main_init()
1171
1172
    print("Checking the DEPRECATED_MODELS constant is up to date.")
    check_deprecated_constant_is_up_to_date()
1173
1174
1175
1176


if __name__ == "__main__":
    check_repo_quality()