check_repo.py 46.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
"""
Utility that performs several consistency checks on the repo. This includes:
- checking all models are properly defined in the __init__ of models/
- checking all models are in the main __init__
- checking all models are properly tested
- checking all object in the main __init__ are documented
- checking all models are in at least one auto class
- checking all the auto mapping are properly defined (no typos, importable)
- checking the list of deprecated models is up to date

Use from the root of the repo with (as used in `make repo-consistency`):

```bash
python utils/check_repo.py
```

It has no auto-fix mode.
"""
33
34
35
import inspect
import os
import re
36
import sys
Sylvain Gugger's avatar
Sylvain Gugger committed
37
import types
38
import warnings
39
from collections import OrderedDict
40
from difflib import get_close_matches
41
from pathlib import Path
Sylvain Gugger's avatar
Sylvain Gugger committed
42
from typing import List, Tuple
43

44
from transformers import is_flax_available, is_tf_available, is_torch_available
45
from transformers.models.auto import get_values
Yih-Dar's avatar
Yih-Dar committed
46
from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
47
48
49
50
from transformers.models.auto.feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING_NAMES
from transformers.models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING_NAMES
from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES
from transformers.models.auto.tokenization_auto import TOKENIZER_MAPPING_NAMES
51
from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import
52

53
54
55
56
57

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_repo.py
PATH_TO_TRANSFORMERS = "src/transformers"
PATH_TO_TESTS = "tests"
58
PATH_TO_DOC = "docs/source/en"
59

60
61
# Update this list with models that are supposed to be private.
PRIVATE_MODELS = [
Jongjyh's avatar
Jongjyh committed
62
    "AltRobertaModel",
63
    "DPRSpanPredictor",
Daniel Stancl's avatar
Daniel Stancl committed
64
    "LongT5Stack",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
65
    "RealmBertModel",
66
    "T5Stack",
67
    "MT5Stack",
68
    "UMT5Stack",
Susnato Dhar's avatar
Susnato Dhar committed
69
    "Pop2PianoStack",
70
    "SwitchTransformersStack",
71
    "TFDPRSpanPredictor",
72
73
    "MaskFormerSwinModel",
    "MaskFormerSwinPreTrainedModel",
74
75
    "BridgeTowerTextModel",
    "BridgeTowerVisionModel",
Yih-Dar's avatar
Yih-Dar committed
76
77
78
    "Kosmos2TextModel",
    "Kosmos2TextForCausalLM",
    "Kosmos2VisionModel",
Yoach Lacombe's avatar
Yoach Lacombe committed
79
80
81
    "SeamlessM4Tv2TextToUnitModel",
    "SeamlessM4Tv2CodeHifiGan",
    "SeamlessM4Tv2TextToUnitForConditionalGeneration",
82
83
]

84
85
# Update this list for models that are not tested with a comment explaining the reason it should not be.
# Being in this list is an exception and should **not** be the rule.
86
IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [
87
    # models to ignore for not tested
Pablo Montalvo's avatar
Pablo Montalvo committed
88
    "FuyuForCausalLM",  # Not tested fort now
NielsRogge's avatar
NielsRogge committed
89
    "InstructBlipQFormerModel",  # Building part of bigger (tested) model.
90
    "UMT5EncoderModel",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
91
    "Blip2QFormerModel",  # Building part of bigger (tested) model.
92
    "ErnieMForInformationExtraction",
93
94
    "FastSpeech2ConformerHifiGan",  # Already tested by SpeechT5HifiGan (# Copied from)
    "FastSpeech2ConformerWithHifiGan",  # Built with two smaller (tested) models.
95
    "GraphormerDecoderHead",  # Building part of bigger (tested) model.
96
97
    "JukeboxVQVAE",  # Building part of bigger (tested) model.
    "JukeboxPrior",  # Building part of bigger (tested) model.
98
    "DecisionTransformerGPT2Model",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
99
    "SegformerDecodeHead",  # Building part of bigger (tested) model.
wangpeng's avatar
wangpeng committed
100
    "MgpstrModel",  # Building part of bigger (tested) model.
101
    "BertLMHeadModel",  # Needs to be setup as decoder.
102
    "MegatronBertLMHeadModel",  # Building part of bigger (tested) model.
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
103
104
105
106
    "RealmBertModel",  # Building part of bigger (tested) model.
    "RealmReader",  # Not regular model.
    "RealmScorer",  # Not regular model.
    "RealmForOpenQA",  # Not regular model.
107
108
109
    "ReformerForMaskedLM",  # Needs to be setup as decoder.
    "TFElectraMainLayer",  # Building part of bigger (tested) model (should it be a TFPreTrainedModel ?)
    "TFRobertaForMultipleChoice",  # TODO: fix
110
    "TFRobertaPreLayerNormForMultipleChoice",  # TODO: fix
abhishek thakur's avatar
abhishek thakur committed
111
    "SeparableConv1D",  # Building part of bigger (tested) model.
112
    "FlaxBartForCausalLM",  # Building part of bigger (tested) model.
113
    "FlaxBertForCausalLM",  # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM.
Younes Belkada's avatar
Younes Belkada committed
114
    "OPTDecoderWrapper",
115
    "TFSegformerDecodeHead",  # Not a regular model.
Jongjyh's avatar
Jongjyh committed
116
    "AltRobertaModel",  # Building part of bigger (tested) model.
Younes Belkada's avatar
Younes Belkada committed
117
    "BlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
Matt's avatar
Matt committed
118
    "TFBlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
119
120
    "BridgeTowerTextModel",  # No need to test it as it is tested by BridgeTowerModel model.
    "BridgeTowerVisionModel",  # No need to test it as it is tested by BridgeTowerModel model.
Yoach Lacombe's avatar
Yoach Lacombe committed
121
    "BarkCausalModel",  # Building part of bigger (tested) model.
jiqing-feng's avatar
jiqing-feng committed
122
    "BarkModel",  # Does not have a forward signature - generation tested with integration tests.
123
124
125
    "SeamlessM4TTextToUnitModel",  # Building part of bigger (tested) model.
    "SeamlessM4TCodeHifiGan",  # Building part of bigger (tested) model.
    "SeamlessM4TTextToUnitForConditionalGeneration",  # Building part of bigger (tested) model.
126
127
128
129
130
]

# Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't
# trigger the common tests.
TEST_FILES_WITH_NO_COMMON_TESTS = [
Yih-Dar's avatar
Yih-Dar committed
131
132
133
134
135
136
137
138
139
140
141
142
143
    "models/decision_transformer/test_modeling_decision_transformer.py",
    "models/camembert/test_modeling_camembert.py",
    "models/mt5/test_modeling_flax_mt5.py",
    "models/mbart/test_modeling_mbart.py",
    "models/mt5/test_modeling_mt5.py",
    "models/pegasus/test_modeling_pegasus.py",
    "models/camembert/test_modeling_tf_camembert.py",
    "models/mt5/test_modeling_tf_mt5.py",
    "models/xlm_roberta/test_modeling_tf_xlm_roberta.py",
    "models/xlm_roberta/test_modeling_flax_xlm_roberta.py",
    "models/xlm_prophetnet/test_modeling_xlm_prophetnet.py",
    "models/xlm_roberta/test_modeling_xlm_roberta.py",
    "models/vision_text_dual_encoder/test_modeling_vision_text_dual_encoder.py",
Matt's avatar
Matt committed
144
    "models/vision_text_dual_encoder/test_modeling_tf_vision_text_dual_encoder.py",
Yih-Dar's avatar
Yih-Dar committed
145
146
    "models/vision_text_dual_encoder/test_modeling_flax_vision_text_dual_encoder.py",
    "models/decision_transformer/test_modeling_decision_transformer.py",
Yoach Lacombe's avatar
Yoach Lacombe committed
147
    "models/bark/test_modeling_bark.py",
148
149
]

150
151
# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and
# should **not** be the rule.
152
IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [
153
    # models to ignore for model xxx mapping
154
155
    "AlignTextModel",
    "AlignVisionModel",
156
157
158
159
    "ClapTextModel",
    "ClapTextModelWithProjection",
    "ClapAudioModel",
    "ClapAudioModelWithProjection",
NielsRogge's avatar
NielsRogge committed
160
161
162
    "Blip2ForConditionalGeneration",
    "Blip2QFormerModel",
    "Blip2VisionModel",
163
    "ErnieMForInformationExtraction",
164
165
    "FastSpeech2ConformerHifiGan",
    "FastSpeech2ConformerWithHifiGan",
166
    "GitVisionModel",
167
168
    "GraphormerModel",
    "GraphormerForGraphClassification",
Younes Belkada's avatar
Younes Belkada committed
169
170
171
172
173
174
    "BlipForConditionalGeneration",
    "BlipForImageTextRetrieval",
    "BlipForQuestionAnswering",
    "BlipVisionModel",
    "BlipTextLMHeadModel",
    "BlipTextModel",
Jinho Park's avatar
Jinho Park committed
175
176
    "BrosSpadeEEForTokenClassification",
    "BrosSpadeELForTokenClassification",
Matt's avatar
Matt committed
177
178
179
180
181
182
    "TFBlipForConditionalGeneration",
    "TFBlipForImageTextRetrieval",
    "TFBlipForQuestionAnswering",
    "TFBlipVisionModel",
    "TFBlipTextLMHeadModel",
    "TFBlipTextModel",
NielsRogge's avatar
NielsRogge committed
183
    "Swin2SRForImageSuperResolution",
184
185
    "BridgeTowerForImageAndTextRetrieval",
    "BridgeTowerForMaskedLM",
186
    "BridgeTowerForContrastiveLearning",
NielsRogge's avatar
NielsRogge committed
187
188
189
    "CLIPSegForImageSegmentation",
    "CLIPSegVisionModel",
    "CLIPSegTextModel",
Matt's avatar
Matt committed
190
    "EsmForProteinFolding",
191
    "GPTSanJapaneseModel",
192
    "TimeSeriesTransformerForPrediction",
193
    "InformerForPrediction",
194
    "AutoformerForPrediction",
195
196
    "PatchTSTForPretraining",
    "PatchTSTForPrediction",
197
198
    "JukeboxVQVAE",
    "JukeboxPrior",
199
    "SamModel",
NielsRogge's avatar
NielsRogge committed
200
    "DPTForDepthEstimation",
201
    "DecisionTransformerGPT2Model",
NielsRogge's avatar
NielsRogge committed
202
    "GLPNForDepthEstimation",
NielsRogge's avatar
NielsRogge committed
203
204
    "ViltForImagesAndTextClassification",
    "ViltForImageAndTextRetrieval",
205
    "ViltForTokenClassification",
NielsRogge's avatar
NielsRogge committed
206
    "ViltForMaskedLM",
NielsRogge's avatar
NielsRogge committed
207
208
    "PerceiverForMultimodalAutoencoding",
    "PerceiverForOpticalFlow",
NielsRogge's avatar
NielsRogge committed
209
    "SegformerDecodeHead",
210
    "TFSegformerDecodeHead",
Kamal Raj's avatar
Kamal Raj committed
211
    "FlaxBeitForMaskedImageModeling",
NielsRogge's avatar
NielsRogge committed
212
    "BeitForMaskedImageModeling",
213
214
    "ChineseCLIPTextModel",
    "ChineseCLIPVisionModel",
Suraj Patil's avatar
Suraj Patil committed
215
    "CLIPTextModel",
216
    "CLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
217
    "CLIPVisionModel",
218
    "CLIPVisionModelWithProjection",
Susnato Dhar's avatar
Susnato Dhar committed
219
220
    "ClvpForCausalLM",
    "ClvpModel",
221
222
    "GroupViTTextModel",
    "GroupViTVisionModel",
Yih-Dar's avatar
Yih-Dar committed
223
224
    "TFCLIPTextModel",
    "TFCLIPVisionModel",
225
226
    "TFGroupViTTextModel",
    "TFGroupViTVisionModel",
Suraj Patil's avatar
Suraj Patil committed
227
    "FlaxCLIPTextModel",
228
    "FlaxCLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
229
    "FlaxCLIPVisionModel",
230
    "FlaxWav2Vec2ForCTC",
NielsRogge's avatar
NielsRogge committed
231
    "DetrForSegmentation",
Younes Belkada's avatar
Younes Belkada committed
232
233
234
    "Pix2StructVisionModel",
    "Pix2StructTextModel",
    "Pix2StructForConditionalGeneration",
235
    "ConditionalDetrForSegmentation",
236
237
    "DPRReader",
    "FlaubertForQuestionAnswering",
238
239
240
241
    "FlavaImageCodebook",
    "FlavaTextModel",
    "FlavaImageModel",
    "FlavaMultimodalModel",
242
    "GPT2DoubleHeadsModel",
243
    "GPTSw3DoubleHeadsModel",
NielsRogge's avatar
NielsRogge committed
244
245
    "InstructBlipVisionModel",
    "InstructBlipQFormerModel",
246
    "LayoutLMForQuestionAnswering",
Ryokan RI's avatar
Ryokan RI committed
247
    "LukeForMaskedLM",
NielsRogge's avatar
NielsRogge committed
248
249
250
    "LukeForEntityClassification",
    "LukeForEntityPairClassification",
    "LukeForEntitySpanClassification",
wangpeng's avatar
wangpeng committed
251
    "MgpstrModel",
252
    "OpenAIGPTDoubleHeadsModel",
253
254
    "OwlViTTextModel",
    "OwlViTVisionModel",
NielsRogge's avatar
NielsRogge committed
255
256
    "Owlv2TextModel",
    "Owlv2VisionModel",
257
    "OwlViTForObjectDetection",
258
259
    "PatchTSMixerForPrediction",
    "PatchTSMixerForPretraining",
260
261
262
    "RagModel",
    "RagSequenceForGeneration",
    "RagTokenForGeneration",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
263
264
265
266
    "RealmEmbedder",
    "RealmForOpenQA",
    "RealmScorer",
    "RealmReader",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
267
    "TFDPRReader",
268
    "TFGPT2DoubleHeadsModel",
269
    "TFLayoutLMForQuestionAnswering",
270
    "TFOpenAIGPTDoubleHeadsModel",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
271
272
273
    "TFRagModel",
    "TFRagSequenceForGeneration",
    "TFRagTokenForGeneration",
274
    "Wav2Vec2ForCTC",
Patrick von Platen's avatar
Patrick von Platen committed
275
    "HubertForCTC",
276
277
    "SEWForCTC",
    "SEWDForCTC",
278
279
    "XLMForQuestionAnswering",
    "XLNetForQuestionAnswering",
abhishek thakur's avatar
abhishek thakur committed
280
    "SeparableConv1D",
Gunjan Chhablani's avatar
Gunjan Chhablani committed
281
282
283
284
    "VisualBertForRegionToPhraseAlignment",
    "VisualBertForVisualReasoning",
    "VisualBertForQuestionAnswering",
    "VisualBertForMultipleChoice",
Will Rice's avatar
Will Rice committed
285
    "TFWav2Vec2ForCTC",
Will Rice's avatar
Will Rice committed
286
    "TFHubertForCTC",
NielsRogge's avatar
NielsRogge committed
287
288
    "XCLIPVisionModel",
    "XCLIPTextModel",
Jongjyh's avatar
Jongjyh committed
289
290
291
    "AltCLIPTextModel",
    "AltCLIPVisionModel",
    "AltRobertaModel",
Zineng Tang's avatar
Zineng Tang committed
292
    "TvltForAudioVisualClassification",
Yoach Lacombe's avatar
Yoach Lacombe committed
293
294
295
296
297
298
    "BarkCausalModel",
    "BarkCoarseModel",
    "BarkFineModel",
    "BarkSemanticModel",
    "MusicgenModel",
    "MusicgenForConditionalGeneration",
299
300
301
    "SpeechT5ForSpeechToSpeech",
    "SpeechT5ForTextToSpeech",
    "SpeechT5HifiGan",
NielsRogge's avatar
NielsRogge committed
302
    "VitMatteForImageMatting",
303
304
305
306
    "SeamlessM4TTextToUnitModel",
    "SeamlessM4TTextToUnitForConditionalGeneration",
    "SeamlessM4TCodeHifiGan",
    "SeamlessM4TForSpeechToSpeech",  # no auto class for speech-to-speech
jiqing-feng's avatar
jiqing-feng committed
307
    "TvpForVideoGrounding",
Yoach Lacombe's avatar
Yoach Lacombe committed
308
309
310
311
    "SeamlessM4Tv2NARTextToUnitModel",
    "SeamlessM4Tv2NARTextToUnitForConditionalGeneration",
    "SeamlessM4Tv2CodeHifiGan",
    "SeamlessM4Tv2ForSpeechToSpeech",  # no auto class for speech-to-speech
312
313
]

314
# DO NOT edit this list!
Sylvain Gugger's avatar
Sylvain Gugger committed
315
# (The corresponding pytorch objects should never have been in the main `__init__`, but it's too late to remove)
316
317
318
319
320
321
322
323
324
325
326
327
328
OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK = [
    "FlaxBertLayer",
    "FlaxBigBirdLayer",
    "FlaxRoFormerLayer",
    "TFBertLayer",
    "TFLxmertEncoder",
    "TFLxmertXLayer",
    "TFMPNetLayer",
    "TFMobileBertLayer",
    "TFSegformerLayer",
    "TFViTMAELayer",
]

Sylvain Gugger's avatar
Sylvain Gugger committed
329
# Update this list for models that have multiple model types for the same model doc.
330
331
332
333
MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(
    [
        ("data2vec-text", "data2vec"),
        ("data2vec-audio", "data2vec"),
334
        ("data2vec-vision", "data2vec"),
NielsRogge's avatar
NielsRogge committed
335
        ("donut-swin", "donut"),
336
337
338
339
    ]
)


340
# This is to make sure the transformers module imported is the one in the repo.
341
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
342
343


344
def check_missing_backends():
Sylvain Gugger's avatar
Sylvain Gugger committed
345
346
347
348
    """
    Checks if all backends are installed (otherwise the check of this script is incomplete). Will error in the CI if
    that's not the case but only throw a warning for users running this.
    """
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
    missing_backends = []
    if not is_torch_available():
        missing_backends.append("PyTorch")
    if not is_tf_available():
        missing_backends.append("TensorFlow")
    if not is_flax_available():
        missing_backends.append("Flax")
    if len(missing_backends) > 0:
        missing = ", ".join(missing_backends)
        if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES:
            raise Exception(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}."
            )
        else:
            warnings.warn(
                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "
                f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you "
                "didn't make any change in one of those backends modeling files, you should probably execute the "
                "command above to be on the safe side."
            )


372
def check_model_list():
Sylvain Gugger's avatar
Sylvain Gugger committed
373
374
375
    """
    Checks the model listed as subfolders of `models` match the models available in `transformers.models`.
    """
376
377
378
379
    # Get the models from the directory structure of `src/transformers/models/`
    models_dir = os.path.join(PATH_TO_TRANSFORMERS, "models")
    _models = []
    for model in os.listdir(models_dir):
Sylvain Gugger's avatar
Sylvain Gugger committed
380
381
        if model == "deprecated":
            continue
382
383
384
385
        model_dir = os.path.join(models_dir, model)
        if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir):
            _models.append(model)

Sylvain Gugger's avatar
Sylvain Gugger committed
386
    # Get the models in the submodule `transformers.models`
387
388
    models = [model for model in dir(transformers.models) if not model.startswith("__")]

389
    missing_models = sorted(set(_models).difference(models))
390
391
392
393
394
395
    if missing_models:
        raise Exception(
            f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}."
        )


396
397
# If some modeling modules should be ignored for all checks, they should be added in the nested list
# _ignore_modules of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
398
399
def get_model_modules() -> List[str]:
    """Get all the model modules inside the transformers library (except deprecated models)."""
400
401
402
403
404
405
406
407
    _ignore_modules = [
        "modeling_auto",
        "modeling_encoder_decoder",
        "modeling_marian",
        "modeling_mmbt",
        "modeling_outputs",
        "modeling_retribert",
        "modeling_utils",
Sylvain Gugger's avatar
Sylvain Gugger committed
408
        "modeling_flax_auto",
409
        "modeling_flax_encoder_decoder",
Stas Bekman's avatar
Stas Bekman committed
410
        "modeling_flax_utils",
411
        "modeling_speech_encoder_decoder",
412
        "modeling_flax_speech_encoder_decoder",
413
        "modeling_flax_vision_encoder_decoder",
amyeroberts's avatar
amyeroberts committed
414
        "modeling_timm_backbone",
415
        "modeling_tf_auto",
416
        "modeling_tf_encoder_decoder",
417
418
419
        "modeling_tf_outputs",
        "modeling_tf_pytorch_utils",
        "modeling_tf_utils",
420
        "modeling_tf_vision_encoder_decoder",
421
        "modeling_vision_encoder_decoder",
422
423
    ]
    modules = []
Sylvain Gugger's avatar
Sylvain Gugger committed
424
425
    for model in dir(transformers.models):
        # There are some magic dunder attributes in the dir, we ignore them
Sylvain Gugger's avatar
Sylvain Gugger committed
426
427
428
429
430
431
432
433
434
        if model == "deprecated" or model.startswith("__"):
            continue

        model_module = getattr(transformers.models, model)
        for submodule in dir(model_module):
            if submodule.startswith("modeling") and submodule not in _ignore_modules:
                modeling_module = getattr(model_module, submodule)
                if inspect.ismodule(modeling_module):
                    modules.append(modeling_module)
435
436
437
    return modules


Sylvain Gugger's avatar
Sylvain Gugger committed
438
439
440
441
442
443
444
445
446
447
448
449
450
def get_models(module: types.ModuleType, include_pretrained: bool = False) -> List[Tuple[str, type]]:
    """
    Get the objects in a module that are models.

    Args:
        module (`types.ModuleType`):
            The module from which we are extracting models.
        include_pretrained (`bool`, *optional*, defaults to `False`):
            Whether or not to include the `PreTrainedModel` subclass (like `BertPreTrainedModel`) or not.

    Returns:
        List[Tuple[str, type]]: List of models as tuples (class name, actual class).
    """
451
    models = []
452
    model_classes = (transformers.PreTrainedModel, transformers.TFPreTrainedModel, transformers.FlaxPreTrainedModel)
453
    for attr_name in dir(module):
454
        if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name):
455
456
457
458
459
460
461
            continue
        attr = getattr(module, attr_name)
        if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__:
            models.append((attr_name, attr))
    return models


Sylvain Gugger's avatar
Sylvain Gugger committed
462
463
464
465
def is_building_block(model: str) -> bool:
    """
    Returns `True` if a model is a building block part of a bigger model.
    """
466
467
468
469
470
471
    if model.endswith("Wrapper"):
        return True
    if model.endswith("Encoder"):
        return True
    if model.endswith("Decoder"):
        return True
472
473
    if model.endswith("Prenet"):
        return True
Sylvain Gugger's avatar
Sylvain Gugger committed
474
475
476
477
478
479
480


def is_a_private_model(model: str) -> bool:
    """Returns `True` if the model should not be in the main init."""
    if model in PRIVATE_MODELS:
        return True
    return is_building_block(model)
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497


def check_models_are_in_init():
    """Checks all models defined in the library are in the main init."""
    models_not_in_init = []
    dir_transformers = dir(transformers)
    for module in get_model_modules():
        models_not_in_init += [
            model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers
        ]

    # Remove private models
    models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)]
    if len(models_not_in_init) > 0:
        raise Exception(f"The following models should be in the main init: {','.join(models_not_in_init)}.")


498
499
# If some test_modeling files should be ignored when checking models are all tested, they should be added in the
# nested list _ignore_files of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
500
501
502
def get_model_test_files() -> List[str]:
    """
    Get the model test files.
Yih-Dar's avatar
Yih-Dar committed
503

Sylvain Gugger's avatar
Sylvain Gugger committed
504
505
506
507
    Returns:
        `List[str]`: The list of test files. The returned files will NOT contain the `tests` (i.e. `PATH_TO_TESTS`
        defined in this script). They will be considered as paths relative to `tests`. A caller has to use
        `os.path.join(PATH_TO_TESTS, ...)` to access the files.
Yih-Dar's avatar
Yih-Dar committed
508
509
    """

510
511
512
    _ignore_files = [
        "test_modeling_common",
        "test_modeling_encoder_decoder",
513
        "test_modeling_flax_encoder_decoder",
514
        "test_modeling_flax_speech_encoder_decoder",
515
516
        "test_modeling_marian",
        "test_modeling_tf_common",
517
        "test_modeling_tf_encoder_decoder",
518
519
    ]
    test_files = []
Yih-Dar's avatar
Yih-Dar committed
520
521
522
523
524
525
526
527
528
529
530
531
    model_test_root = os.path.join(PATH_TO_TESTS, "models")
    model_test_dirs = []
    for x in os.listdir(model_test_root):
        x = os.path.join(model_test_root, x)
        if os.path.isdir(x):
            model_test_dirs.append(x)

    for target_dir in [PATH_TO_TESTS] + model_test_dirs:
        for file_or_dir in os.listdir(target_dir):
            path = os.path.join(target_dir, file_or_dir)
            if os.path.isfile(path):
                filename = os.path.split(path)[-1]
532
                if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files:
Yih-Dar's avatar
Yih-Dar committed
533
534
535
                    file = os.path.join(*path.split(os.sep)[1:])
                    test_files.append(file)

536
537
538
539
540
    return test_files


# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class
# for the all_model_classes variable.
Sylvain Gugger's avatar
Sylvain Gugger committed
541
542
543
544
545
546
547
548
549
550
551
def find_tested_models(test_file: str) -> List[str]:
    """
    Parse the content of test_file to detect what's in `all_model_classes`. This detects the models that inherit from
    the common test class.

    Args:
        test_file (`str`): The path to the test file to check

    Returns:
        `List[str]`: The list of models tested in that file.
    """
552
    with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f:
553
        content = f.read()
Sylvain Gugger's avatar
Sylvain Gugger committed
554
    all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content)
555
556
    # Check with one less parenthesis as well
    all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
557
    if len(all_models) > 0:
558
        model_tested = []
Sylvain Gugger's avatar
Sylvain Gugger committed
559
560
561
562
563
        for entry in all_models:
            for line in entry.split(","):
                name = line.strip()
                if len(name) > 0:
                    model_tested.append(name)
564
565
566
        return model_tested


Sylvain Gugger's avatar
Sylvain Gugger committed
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
def should_be_tested(model_name: str) -> bool:
    """
    Whether or not a model should be tested.
    """
    if model_name in IGNORE_NON_TESTED:
        return False
    return not is_building_block(model_name)


def check_models_are_tested(module: types.ModuleType, test_file: str) -> List[str]:
    """Check models defined in a module are all tested in a given file.

    Args:
        module (`types.ModuleType`): The module in which we get the models.
        test_file (`str`): The path to the file where the module is tested.

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
586
    # XxxPreTrainedModel are not tested
587
588
589
    defined_models = get_models(module)
    tested_models = find_tested_models(test_file)
    if tested_models is None:
590
        if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS:
591
592
593
594
595
596
597
598
            return
        return [
            f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. "
            + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file "
            + "`utils/check_repo.py`."
        ]
    failures = []
    for model_name, _ in defined_models:
Sylvain Gugger's avatar
Sylvain Gugger committed
599
        if model_name not in tested_models and should_be_tested(model_name):
600
601
602
603
604
605
606
607
608
609
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not tested in "
                + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file."
                + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`"
                + "in the file `utils/check_repo.py`."
            )
    return failures


def check_all_models_are_tested():
Patrick von Platen's avatar
Patrick von Platen committed
610
    """Check all models are properly tested."""
611
612
613
614
    modules = get_model_modules()
    test_files = get_model_test_files()
    failures = []
    for module in modules:
Sylvain Gugger's avatar
Sylvain Gugger committed
615
        # Matches a module to its test file.
616
617
        test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file]
        if len(test_file) == 0:
618
            failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")
619
620
621
622
        elif len(test_file) > 1:
            failures.append(f"{module.__name__} has several test files: {test_file}.")
        else:
            test_file = test_file[0]
623
624
625
            new_failures = check_models_are_tested(module, test_file)
            if new_failures is not None:
                failures += new_failures
626
627
628
629
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
630
def get_all_auto_configured_models() -> List[str]:
Patrick von Platen's avatar
Patrick von Platen committed
631
    """Return the list of all models in at least one auto class."""
632
    result = set()  # To avoid duplicates we concatenate all model classes in a set.
633
634
    if is_torch_available():
        for attr_name in dir(transformers.models.auto.modeling_auto):
635
            if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"):
636
637
638
                result = result | set(get_values(getattr(transformers.models.auto.modeling_auto, attr_name)))
    if is_tf_available():
        for attr_name in dir(transformers.models.auto.modeling_tf_auto):
639
            if attr_name.startswith("TF_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
640
641
642
                result = result | set(get_values(getattr(transformers.models.auto.modeling_tf_auto, attr_name)))
    if is_flax_available():
        for attr_name in dir(transformers.models.auto.modeling_flax_auto):
643
            if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
644
                result = result | set(get_values(getattr(transformers.models.auto.modeling_flax_auto, attr_name)))
645
    return list(result)
646
647


Sylvain Gugger's avatar
Sylvain Gugger committed
648
649
def ignore_unautoclassed(model_name: str) -> bool:
    """Rules to determine if a model should be in an auto class."""
650
651
652
653
654
655
656
657
658
    # Special white list
    if model_name in IGNORE_NON_AUTO_CONFIGURED:
        return True
    # Encoder and Decoder should be ignored
    if "Encoder" in model_name or "Decoder" in model_name:
        return True
    return False


Sylvain Gugger's avatar
Sylvain Gugger committed
659
660
661
662
663
664
665
666
667
668
669
670
671
def check_models_are_auto_configured(module: types.ModuleType, all_auto_models: List[str]) -> List[str]:
    """
    Check models defined in module are each in an auto class.

    Args:
        module (`types.ModuleType`):
            The module in which we get the models.
        all_auto_models (`List[str]`):
            The list of all models in an auto class (as obtained with `get_all_auto_configured_models()`).

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
672
673
674
    defined_models = get_models(module)
    failures = []
    for model_name, _ in defined_models:
675
        if model_name not in all_auto_models and not ignore_unautoclassed(model_name):
676
677
678
679
680
681
682
683
684
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "
                "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "
                "`utils/check_repo.py`."
            )
    return failures


def check_all_models_are_auto_configured():
Patrick von Platen's avatar
Patrick von Platen committed
685
    """Check all models are each in an auto class."""
Sylvain Gugger's avatar
Sylvain Gugger committed
686
    # This is where we need to check we have all backends or the check is incomplete.
687
    check_missing_backends()
688
689
690
691
692
693
694
695
696
697
698
    modules = get_model_modules()
    all_auto_models = get_all_auto_configured_models()
    failures = []
    for module in modules:
        new_failures = check_models_are_auto_configured(module, all_auto_models)
        if new_failures is not None:
            failures += new_failures
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


699
700
def check_all_auto_object_names_being_defined():
    """Check all names defined in auto (name) mappings exist in the library."""
Sylvain Gugger's avatar
Sylvain Gugger committed
701
    # This is where we need to check we have all backends or the check is incomplete.
702
    check_missing_backends()
703

704
    failures = []
705
    mappings_to_check = {
706
707
708
709
710
711
        "TOKENIZER_MAPPING_NAMES": TOKENIZER_MAPPING_NAMES,
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

712
713
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
714
715
716
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
717
718
719
720
721
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
722
        for _, class_names in mapping.items():
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
            if not isinstance(class_names, tuple):
                class_names = (class_names,)
                for class_name in class_names:
                    if class_name is None:
                        continue
                    # dummy object is accepted
                    if not hasattr(transformers, class_name):
                        # If the class name is in a model name mapping, let's not check if there is a definition in any modeling
                        # module, if it's a private model defined in this file.
                        if name.endswith("MODEL_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
                        failures.append(
                            f"`{class_name}` appears in the mapping `{name}` but it is not defined in the library."
                        )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Yih-Dar's avatar
Yih-Dar committed
741
742
def check_all_auto_mapping_names_in_config_mapping_names():
    """Check all keys defined in auto mappings (mappings of names) appear in `CONFIG_MAPPING_NAMES`."""
Sylvain Gugger's avatar
Sylvain Gugger committed
743
    # This is where we need to check we have all backends or the check is incomplete.
744
    check_missing_backends()
Yih-Dar's avatar
Yih-Dar committed
745

746
    failures = []
Yih-Dar's avatar
Yih-Dar committed
747
    # `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer` is special, and don't need to follow the rule.
748
    mappings_to_check = {
Yih-Dar's avatar
Yih-Dar committed
749
750
751
752
753
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

754
755
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
756
757
758
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
759
760
761
762
763
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
764
        for model_type in mapping:
Yih-Dar's avatar
Yih-Dar committed
765
766
767
768
769
770
771
772
773
            if model_type not in CONFIG_MAPPING_NAMES:
                failures.append(
                    f"`{model_type}` appears in the mapping `{name}` but it is not defined in the keys of "
                    "`CONFIG_MAPPING_NAMES`."
                )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


774
def check_all_auto_mappings_importable():
Sylvain Gugger's avatar
Sylvain Gugger committed
775
776
    """Check all auto mappings can be imported."""
    # This is where we need to check we have all backends or the check is incomplete.
777
778
779
780
781
782
783
784
785
786
787
788
789
    check_missing_backends()

    failures = []
    mappings_to_check = {}
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

Sylvain Gugger's avatar
Sylvain Gugger committed
790
    for name in mappings_to_check:
791
792
        name = name.replace("_MAPPING_NAMES", "_MAPPING")
        if not hasattr(transformers, name):
793
794
795
796
797
798
            failures.append(f"`{name}`")
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


def check_objects_being_equally_in_main_init():
Sylvain Gugger's avatar
Sylvain Gugger committed
799
800
801
    """
    Check if a (TensorFlow or Flax) object is in the main __init__ iif its counterpart in PyTorch is.
    """
802
803
804
805
806
    attrs = dir(transformers)

    failures = []
    for attr in attrs:
        obj = getattr(transformers, attr)
Sylvain Gugger's avatar
Sylvain Gugger committed
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
        if not hasattr(obj, "__module__") or "models.deprecated" in obj.__module__:
            continue

        module_path = obj.__module__
        module_name = module_path.split(".")[-1]
        module_dir = ".".join(module_path.split(".")[:-1])
        if (
            module_name.startswith("modeling_")
            and not module_name.startswith("modeling_tf_")
            and not module_name.startswith("modeling_flax_")
        ):
            parent_module = sys.modules[module_dir]

            frameworks = []
            if is_tf_available():
                frameworks.append("TF")
            if is_flax_available():
                frameworks.append("Flax")

            for framework in frameworks:
                other_module_path = module_path.replace("modeling_", f"modeling_{framework.lower()}_")
                if os.path.isfile("src/" + other_module_path.replace(".", "/") + ".py"):
                    other_module_name = module_name.replace("modeling_", f"modeling_{framework.lower()}_")
                    other_module = getattr(parent_module, other_module_name)
                    if hasattr(other_module, f"{framework}{attr}"):
                        if not hasattr(transformers, f"{framework}{attr}"):
                            if f"{framework}{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}{attr}")
                    if hasattr(other_module, f"{framework}_{attr}"):
                        if not hasattr(transformers, f"{framework}_{attr}"):
                            if f"{framework}_{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}_{attr}")
839
840
841
842
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
843
844
845
_re_decorator = re.compile(r"^\s*@(\S+)\s+$")


Sylvain Gugger's avatar
Sylvain Gugger committed
846
847
848
849
850
851
852
853
854
855
def check_decorator_order(filename: str) -> List[int]:
    """
    Check that in a given test file, the slow decorator is always last.

    Args:
        filename (`str`): The path to a test file to check.

    Returns:
        `List[int]`: The list of failures as a list of indices where there are problems.
    """
856
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
Sylvain Gugger's avatar
Sylvain Gugger committed
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
        lines = f.readlines()
    decorator_before = None
    errors = []
    for i, line in enumerate(lines):
        search = _re_decorator.search(line)
        if search is not None:
            decorator_name = search.groups()[0]
            if decorator_before is not None and decorator_name.startswith("parameterized"):
                errors.append(i)
            decorator_before = decorator_name
        elif decorator_before is not None:
            decorator_before = None
    return errors


def check_all_decorator_order():
Patrick von Platen's avatar
Patrick von Platen committed
873
    """Check that in all test files, the slow decorator is always last."""
Sylvain Gugger's avatar
Sylvain Gugger committed
874
875
876
877
878
879
880
881
882
    errors = []
    for fname in os.listdir(PATH_TO_TESTS):
        if fname.endswith(".py"):
            filename = os.path.join(PATH_TO_TESTS, fname)
            new_errors = check_decorator_order(filename)
            errors += [f"- {filename}, line {i}" for i in new_errors]
    if len(errors) > 0:
        msg = "\n".join(errors)
        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
883
884
            "The parameterized decorator (and its variants) should always be first, but this is not the case in the"
            f" following files:\n{msg}"
Sylvain Gugger's avatar
Sylvain Gugger committed
885
886
887
        )


Sylvain Gugger's avatar
Sylvain Gugger committed
888
889
890
891
892
893
894
def find_all_documented_objects() -> List[str]:
    """
    Parse the content of all doc files to detect which classes and functions it documents.

    Returns:
        `List[str]`: The list of all object names being documented.
    """
895
896
    documented_obj = []
    for doc_file in Path(PATH_TO_DOC).glob("**/*.rst"):
Julien Plu's avatar
Julien Plu committed
897
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
898
899
900
            content = f.read()
        raw_doc_objs = re.findall(r"(?:autoclass|autofunction):: transformers.(\S+)\s+", content)
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
901
    for doc_file in Path(PATH_TO_DOC).glob("**/*.md"):
Sylvain Gugger's avatar
Sylvain Gugger committed
902
903
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
            content = f.read()
904
        raw_doc_objs = re.findall(r"\[\[autodoc\]\]\s+(\S+)\s+", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
905
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
906
907
908
909
910
911
    return documented_obj


# One good reason for not being documented is to be deprecated. Put in this list deprecated objects.
DEPRECATED_OBJECTS = [
    "AutoModelWithLMHead",
912
    "BartPretrainedModel",
913
914
    "DataCollator",
    "DataCollatorForSOP",
915
916
917
918
919
    "GlueDataset",
    "GlueDataTrainingArguments",
    "LineByLineTextDataset",
    "LineByLineWithRefDataset",
    "LineByLineWithSOPTextDataset",
Steven Liu's avatar
Steven Liu committed
920
    "NerPipeline",
921
922
923
924
925
926
927
928
929
930
    "PretrainedBartModel",
    "PretrainedFSMTModel",
    "SingleSentenceClassificationProcessor",
    "SquadDataTrainingArguments",
    "SquadDataset",
    "SquadExample",
    "SquadFeatures",
    "SquadV1Processor",
    "SquadV2Processor",
    "TFAutoModelWithLMHead",
931
    "TFBartPretrainedModel",
932
933
    "TextDataset",
    "TextDatasetForNextSentencePrediction",
934
    "Wav2Vec2ForMaskedLM",
935
    "Wav2Vec2Tokenizer",
936
937
938
939
940
941
942
943
944
945
    "glue_compute_metrics",
    "glue_convert_examples_to_features",
    "glue_output_modes",
    "glue_processors",
    "glue_tasks_num_labels",
    "squad_convert_examples_to_features",
    "xnli_compute_metrics",
    "xnli_output_modes",
    "xnli_processors",
    "xnli_tasks_num_labels",
946
947
    "TFTrainer",
    "TFTrainingArguments",
948
949
950
951
952
953
954
]

# Exceptionally, some objects should not be documented after all rules passed.
# ONLY PUT SOMETHING IN THIS LIST AS A LAST RESORT!
UNDOCUMENTED_OBJECTS = [
    "AddedToken",  # This is a tokenizers class.
    "BasicTokenizer",  # Internal, should never have been in the main init.
955
    "CharacterTokenizer",  # Internal, should never have been in the main init.
956
    "DPRPretrainedReader",  # Like an Encoder.
Sylvain Gugger's avatar
Sylvain Gugger committed
957
    "DummyObject",  # Just picked by mistake sometimes.
958
    "MecabTokenizer",  # Internal, should never have been in the main init.
959
960
961
962
963
964
965
966
967
968
969
    "ModelCard",  # Internal type.
    "SqueezeBertModule",  # Internal building block (should have been called SqueezeBertLayer)
    "TFDPRPretrainedReader",  # Like an Encoder.
    "TransfoXLCorpus",  # Internal type.
    "WordpieceTokenizer",  # Internal, should never have been in the main init.
    "absl",  # External module
    "add_end_docstrings",  # Internal, should never have been in the main init.
    "add_start_docstrings",  # Internal, should never have been in the main init.
    "convert_tf_weight_name_to_pt_weight_name",  # Internal used to convert model weights
    "logger",  # Internal logger
    "logging",  # External module
970
    "requires_backends",  # Internal function
Jongjyh's avatar
Jongjyh committed
971
    "AltRobertaModel",  # Internal module
972
973
974
975
976
977
978
979
980
]

# This list should be empty. Objects in it should get their own doc page.
SHOULD_HAVE_THEIR_OWN_PAGE = [
    # Benchmarks
    "PyTorchBenchmark",
    "PyTorchBenchmarkArguments",
    "TensorFlowBenchmark",
    "TensorFlowBenchmarkArguments",
981
    "AutoBackbone",
NielsRogge's avatar
NielsRogge committed
982
    "BeitBackbone",
NielsRogge's avatar
NielsRogge committed
983
984
    "BitBackbone",
    "ConvNextBackbone",
Alara Dirik's avatar
Alara Dirik committed
985
    "ConvNextV2Backbone",
986
    "DinatBackbone",
987
    "Dinov2Backbone",
Alara Dirik's avatar
Alara Dirik committed
988
    "FocalNetBackbone",
NielsRogge's avatar
NielsRogge committed
989
    "MaskFormerSwinBackbone",
990
991
    "MaskFormerSwinConfig",
    "MaskFormerSwinModel",
NielsRogge's avatar
NielsRogge committed
992
993
    "NatBackbone",
    "ResNetBackbone",
NielsRogge's avatar
NielsRogge committed
994
    "SwinBackbone",
NielsRogge's avatar
NielsRogge committed
995
    "Swinv2Backbone",
amyeroberts's avatar
amyeroberts committed
996
997
    "TimmBackbone",
    "TimmBackboneConfig",
NielsRogge's avatar
NielsRogge committed
998
    "VitDetBackbone",
999
1000
1001
]


Sylvain Gugger's avatar
Sylvain Gugger committed
1002
1003
def ignore_undocumented(name: str) -> bool:
    """Rules to determine if `name` should be undocumented (returns `True` if it should not be documented)."""
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
    # NOT DOCUMENTED ON PURPOSE.
    # Constants uppercase are not documented.
    if name.isupper():
        return True
    # PreTrainedModels / Encoders / Decoders / Layers / Embeddings / Attention are not documented.
    if (
        name.endswith("PreTrainedModel")
        or name.endswith("Decoder")
        or name.endswith("Encoder")
        or name.endswith("Layer")
        or name.endswith("Embeddings")
        or name.endswith("Attention")
    ):
        return True
    # Submodules are not documented.
    if os.path.isdir(os.path.join(PATH_TO_TRANSFORMERS, name)) or os.path.isfile(
        os.path.join(PATH_TO_TRANSFORMERS, f"{name}.py")
    ):
        return True
    # All load functions are not documented.
    if name.startswith("load_tf") or name.startswith("load_pytorch"):
        return True
    # is_xxx_available functions are not documented.
    if name.startswith("is_") and name.endswith("_available"):
        return True
    # Deprecated objects are not documented.
    if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS:
        return True
    # MMBT model does not really work.
    if name.startswith("MMBT"):
        return True
    if name in SHOULD_HAVE_THEIR_OWN_PAGE:
        return True
    return False


def check_all_objects_are_documented():
Patrick von Platen's avatar
Patrick von Platen committed
1041
    """Check all models are properly documented."""
1042
    documented_objs = find_all_documented_objects()
1043
1044
1045
    modules = transformers._modules
    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]
    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]
1046
1047
1048
1049
1050
    if len(undocumented_objs) > 0:
        raise Exception(
            "The following objects are in the public init so should be documented:\n - "
            + "\n - ".join(undocumented_objs)
        )
1051
    check_docstrings_are_in_md()
1052
1053
1054
1055
1056
1057
    check_model_type_doc_match()


def check_model_type_doc_match():
    """Check all doc pages have a corresponding model type."""
    model_doc_folder = Path(PATH_TO_DOC) / "model_doc"
1058
    model_docs = [m.stem for m in model_doc_folder.glob("*.md")]
1059
1060

    model_types = list(transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys())
1061
    model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types]
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079

    errors = []
    for m in model_docs:
        if m not in model_types and m != "auto":
            close_matches = get_close_matches(m, model_types)
            error_message = f"{m} is not a proper model identifier."
            if len(close_matches) > 0:
                close_matches = "/".join(close_matches)
                error_message += f" Did you mean {close_matches}?"
            errors.append(error_message)

    if len(errors) > 0:
        raise ValueError(
            "Some model doc pages do not match any existing model type:\n"
            + "\n".join(errors)
            + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in "
            "models/auto/configuration_auto.py."
        )
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089


# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`.
_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`")
# Re pattern to catch things between double backquotes.
_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")
# Re pattern to catch example introduction.
_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE)


Sylvain Gugger's avatar
Sylvain Gugger committed
1090
def is_rst_docstring(docstring: str) -> True:
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
    """
    Returns `True` if `docstring` is written in rst.
    """
    if _re_rst_special_words.search(docstring) is not None:
        return True
    if _re_double_backquotes.search(docstring) is not None:
        return True
    if _re_rst_example.search(docstring) is not None:
        return True
    return False


def check_docstrings_are_in_md():
Sylvain Gugger's avatar
Sylvain Gugger committed
1104
    """Check all docstrings are written in md and nor rst."""
1105
1106
    files_with_rst = []
    for file in Path(PATH_TO_TRANSFORMERS).glob("**/*.py"):
1107
        with open(file, encoding="utf-8") as f:
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
            code = f.read()
        docstrings = code.split('"""')

        for idx, docstring in enumerate(docstrings):
            if idx % 2 == 0 or not is_rst_docstring(docstring):
                continue
            files_with_rst.append(file)
            break

    if len(files_with_rst) > 0:
        raise ValueError(
            "The following files have docstrings written in rst:\n"
            + "\n".join([f"- {f}" for f in files_with_rst])
Kamal Raj's avatar
Kamal Raj committed
1121
            + "\nTo fix this run `doc-builder convert path_to_py_file` after installing `doc-builder`\n"
1122
1123
            "(`pip install git+https://github.com/huggingface/doc-builder`)"
        )
1124
1125


1126
def check_deprecated_constant_is_up_to_date():
Sylvain Gugger's avatar
Sylvain Gugger committed
1127
1128
1129
    """
    Check if the constant `DEPRECATED_MODELS` in `models/auto/configuration_auto.py` is up to date.
    """
1130
1131
1132
1133
1134
1135
1136
1137
1138
    deprecated_folder = os.path.join(PATH_TO_TRANSFORMERS, "models", "deprecated")
    deprecated_models = [m for m in os.listdir(deprecated_folder) if not m.startswith("_")]

    constant_to_check = transformers.models.auto.configuration_auto.DEPRECATED_MODELS
    message = []
    missing_models = sorted(set(deprecated_models) - set(constant_to_check))
    if len(missing_models) != 0:
        missing_models = ", ".join(missing_models)
        message.append(
1139
            "The following models are in the deprecated folder, make sure to add them to `DEPRECATED_MODELS` in "
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
            f"`models/auto/configuration_auto.py`: {missing_models}."
        )

    extra_models = sorted(set(constant_to_check) - set(deprecated_models))
    if len(extra_models) != 0:
        extra_models = ", ".join(extra_models)
        message.append(
            "The following models are in the `DEPRECATED_MODELS` constant but not in the deprecated folder. Either "
            f"remove them from the constant or move to the deprecated folder: {extra_models}."
        )

    if len(message) > 0:
        raise Exception("\n".join(message))


1155
def check_repo_quality():
Patrick von Platen's avatar
Patrick von Platen committed
1156
    """Check all models are properly tested and documented."""
1157
1158
    print("Checking all models are included.")
    check_model_list()
1159
1160
    print("Checking all models are public.")
    check_models_are_in_init()
1161
    print("Checking all models are properly tested.")
Sylvain Gugger's avatar
Sylvain Gugger committed
1162
    check_all_decorator_order()
1163
    check_all_models_are_tested()
1164
    print("Checking all objects are properly documented.")
1165
    check_all_objects_are_documented()
1166
1167
    print("Checking all models are in at least one auto class.")
    check_all_models_are_auto_configured()
1168
1169
    print("Checking all names in auto name mappings are defined.")
    check_all_auto_object_names_being_defined()
Yih-Dar's avatar
Yih-Dar committed
1170
1171
    print("Checking all keys in auto name mappings are defined in `CONFIG_MAPPING_NAMES`.")
    check_all_auto_mapping_names_in_config_mapping_names()
1172
1173
    print("Checking all auto mappings could be imported.")
    check_all_auto_mappings_importable()
1174
1175
    print("Checking all objects are equally (across frameworks) in the main __init__.")
    check_objects_being_equally_in_main_init()
1176
1177
    print("Checking the DEPRECATED_MODELS constant is up to date.")
    check_deprecated_constant_is_up_to_date()
1178
1179
1180
1181


if __name__ == "__main__":
    check_repo_quality()