check_repo.py 46.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
"""
Utility that performs several consistency checks on the repo. This includes:
- checking all models are properly defined in the __init__ of models/
- checking all models are in the main __init__
- checking all models are properly tested
- checking all object in the main __init__ are documented
- checking all models are in at least one auto class
- checking all the auto mapping are properly defined (no typos, importable)
- checking the list of deprecated models is up to date

Use from the root of the repo with (as used in `make repo-consistency`):

```bash
python utils/check_repo.py
```

It has no auto-fix mode.
"""
33
34
35
import inspect
import os
import re
36
import sys
Sylvain Gugger's avatar
Sylvain Gugger committed
37
import types
38
import warnings
39
from collections import OrderedDict
40
from difflib import get_close_matches
41
from pathlib import Path
Sylvain Gugger's avatar
Sylvain Gugger committed
42
from typing import List, Tuple
43

44
from transformers import is_flax_available, is_tf_available, is_torch_available
45
from transformers.models.auto import get_values
Yih-Dar's avatar
Yih-Dar committed
46
from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
47
48
49
50
from transformers.models.auto.feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING_NAMES
from transformers.models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING_NAMES
from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES
from transformers.models.auto.tokenization_auto import TOKENIZER_MAPPING_NAMES
51
from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import
52

53
54
55
56
57

# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_repo.py
PATH_TO_TRANSFORMERS = "src/transformers"
PATH_TO_TESTS = "tests"
58
PATH_TO_DOC = "docs/source/en"
59

60
61
# Update this list with models that are supposed to be private.
PRIVATE_MODELS = [
Jongjyh's avatar
Jongjyh committed
62
    "AltRobertaModel",
63
    "DPRSpanPredictor",
NielsRogge's avatar
NielsRogge committed
64
    "UdopStack",
Daniel Stancl's avatar
Daniel Stancl committed
65
    "LongT5Stack",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
66
    "RealmBertModel",
67
    "T5Stack",
68
    "MT5Stack",
69
    "UMT5Stack",
Susnato Dhar's avatar
Susnato Dhar committed
70
    "Pop2PianoStack",
71
    "SwitchTransformersStack",
72
    "TFDPRSpanPredictor",
73
74
    "MaskFormerSwinModel",
    "MaskFormerSwinPreTrainedModel",
75
76
    "BridgeTowerTextModel",
    "BridgeTowerVisionModel",
Yih-Dar's avatar
Yih-Dar committed
77
78
79
    "Kosmos2TextModel",
    "Kosmos2TextForCausalLM",
    "Kosmos2VisionModel",
Yoach Lacombe's avatar
Yoach Lacombe committed
80
81
82
    "SeamlessM4Tv2TextToUnitModel",
    "SeamlessM4Tv2CodeHifiGan",
    "SeamlessM4Tv2TextToUnitForConditionalGeneration",
83
84
]

85
86
# Update this list for models that are not tested with a comment explaining the reason it should not be.
# Being in this list is an exception and should **not** be the rule.
87
IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [
88
    # models to ignore for not tested
Pablo Montalvo's avatar
Pablo Montalvo committed
89
    "FuyuForCausalLM",  # Not tested fort now
NielsRogge's avatar
NielsRogge committed
90
    "InstructBlipQFormerModel",  # Building part of bigger (tested) model.
91
    "UMT5EncoderModel",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
92
    "Blip2QFormerModel",  # Building part of bigger (tested) model.
93
    "ErnieMForInformationExtraction",
94
95
    "FastSpeech2ConformerHifiGan",  # Already tested by SpeechT5HifiGan (# Copied from)
    "FastSpeech2ConformerWithHifiGan",  # Built with two smaller (tested) models.
96
    "GraphormerDecoderHead",  # Building part of bigger (tested) model.
97
98
    "JukeboxVQVAE",  # Building part of bigger (tested) model.
    "JukeboxPrior",  # Building part of bigger (tested) model.
99
    "DecisionTransformerGPT2Model",  # Building part of bigger (tested) model.
NielsRogge's avatar
NielsRogge committed
100
    "SegformerDecodeHead",  # Building part of bigger (tested) model.
wangpeng's avatar
wangpeng committed
101
    "MgpstrModel",  # Building part of bigger (tested) model.
102
    "BertLMHeadModel",  # Needs to be setup as decoder.
103
    "MegatronBertLMHeadModel",  # Building part of bigger (tested) model.
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
104
105
106
107
    "RealmBertModel",  # Building part of bigger (tested) model.
    "RealmReader",  # Not regular model.
    "RealmScorer",  # Not regular model.
    "RealmForOpenQA",  # Not regular model.
108
109
110
    "ReformerForMaskedLM",  # Needs to be setup as decoder.
    "TFElectraMainLayer",  # Building part of bigger (tested) model (should it be a TFPreTrainedModel ?)
    "TFRobertaForMultipleChoice",  # TODO: fix
111
    "TFRobertaPreLayerNormForMultipleChoice",  # TODO: fix
abhishek thakur's avatar
abhishek thakur committed
112
    "SeparableConv1D",  # Building part of bigger (tested) model.
113
    "FlaxBartForCausalLM",  # Building part of bigger (tested) model.
114
    "FlaxBertForCausalLM",  # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM.
Younes Belkada's avatar
Younes Belkada committed
115
    "OPTDecoderWrapper",
116
    "TFSegformerDecodeHead",  # Not a regular model.
Jongjyh's avatar
Jongjyh committed
117
    "AltRobertaModel",  # Building part of bigger (tested) model.
Younes Belkada's avatar
Younes Belkada committed
118
    "BlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
Matt's avatar
Matt committed
119
    "TFBlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models
120
121
    "BridgeTowerTextModel",  # No need to test it as it is tested by BridgeTowerModel model.
    "BridgeTowerVisionModel",  # No need to test it as it is tested by BridgeTowerModel model.
Yoach Lacombe's avatar
Yoach Lacombe committed
122
    "BarkCausalModel",  # Building part of bigger (tested) model.
jiqing-feng's avatar
jiqing-feng committed
123
    "BarkModel",  # Does not have a forward signature - generation tested with integration tests.
124
125
126
    "SeamlessM4TTextToUnitModel",  # Building part of bigger (tested) model.
    "SeamlessM4TCodeHifiGan",  # Building part of bigger (tested) model.
    "SeamlessM4TTextToUnitForConditionalGeneration",  # Building part of bigger (tested) model.
127
128
129
130
131
]

# Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't
# trigger the common tests.
TEST_FILES_WITH_NO_COMMON_TESTS = [
Yih-Dar's avatar
Yih-Dar committed
132
133
134
135
136
137
138
139
140
141
142
143
144
    "models/decision_transformer/test_modeling_decision_transformer.py",
    "models/camembert/test_modeling_camembert.py",
    "models/mt5/test_modeling_flax_mt5.py",
    "models/mbart/test_modeling_mbart.py",
    "models/mt5/test_modeling_mt5.py",
    "models/pegasus/test_modeling_pegasus.py",
    "models/camembert/test_modeling_tf_camembert.py",
    "models/mt5/test_modeling_tf_mt5.py",
    "models/xlm_roberta/test_modeling_tf_xlm_roberta.py",
    "models/xlm_roberta/test_modeling_flax_xlm_roberta.py",
    "models/xlm_prophetnet/test_modeling_xlm_prophetnet.py",
    "models/xlm_roberta/test_modeling_xlm_roberta.py",
    "models/vision_text_dual_encoder/test_modeling_vision_text_dual_encoder.py",
Matt's avatar
Matt committed
145
    "models/vision_text_dual_encoder/test_modeling_tf_vision_text_dual_encoder.py",
Yih-Dar's avatar
Yih-Dar committed
146
147
    "models/vision_text_dual_encoder/test_modeling_flax_vision_text_dual_encoder.py",
    "models/decision_transformer/test_modeling_decision_transformer.py",
Yoach Lacombe's avatar
Yoach Lacombe committed
148
    "models/bark/test_modeling_bark.py",
149
150
]

151
152
# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and
# should **not** be the rule.
153
IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [
154
    # models to ignore for model xxx mapping
155
156
    "AlignTextModel",
    "AlignVisionModel",
157
158
159
160
    "ClapTextModel",
    "ClapTextModelWithProjection",
    "ClapAudioModel",
    "ClapAudioModelWithProjection",
NielsRogge's avatar
NielsRogge committed
161
162
163
    "Blip2ForConditionalGeneration",
    "Blip2QFormerModel",
    "Blip2VisionModel",
164
    "ErnieMForInformationExtraction",
165
166
    "FastSpeech2ConformerHifiGan",
    "FastSpeech2ConformerWithHifiGan",
167
    "GitVisionModel",
168
169
    "GraphormerModel",
    "GraphormerForGraphClassification",
Younes Belkada's avatar
Younes Belkada committed
170
171
172
173
174
175
    "BlipForConditionalGeneration",
    "BlipForImageTextRetrieval",
    "BlipForQuestionAnswering",
    "BlipVisionModel",
    "BlipTextLMHeadModel",
    "BlipTextModel",
Jinho Park's avatar
Jinho Park committed
176
177
    "BrosSpadeEEForTokenClassification",
    "BrosSpadeELForTokenClassification",
Matt's avatar
Matt committed
178
179
180
181
182
183
    "TFBlipForConditionalGeneration",
    "TFBlipForImageTextRetrieval",
    "TFBlipForQuestionAnswering",
    "TFBlipVisionModel",
    "TFBlipTextLMHeadModel",
    "TFBlipTextModel",
NielsRogge's avatar
NielsRogge committed
184
    "Swin2SRForImageSuperResolution",
185
186
    "BridgeTowerForImageAndTextRetrieval",
    "BridgeTowerForMaskedLM",
187
    "BridgeTowerForContrastiveLearning",
NielsRogge's avatar
NielsRogge committed
188
189
190
    "CLIPSegForImageSegmentation",
    "CLIPSegVisionModel",
    "CLIPSegTextModel",
Matt's avatar
Matt committed
191
    "EsmForProteinFolding",
192
    "GPTSanJapaneseModel",
193
    "TimeSeriesTransformerForPrediction",
194
    "InformerForPrediction",
195
    "AutoformerForPrediction",
196
197
    "PatchTSTForPretraining",
    "PatchTSTForPrediction",
198
199
    "JukeboxVQVAE",
    "JukeboxPrior",
200
    "SamModel",
NielsRogge's avatar
NielsRogge committed
201
    "DPTForDepthEstimation",
202
    "DecisionTransformerGPT2Model",
NielsRogge's avatar
NielsRogge committed
203
    "GLPNForDepthEstimation",
NielsRogge's avatar
NielsRogge committed
204
205
    "ViltForImagesAndTextClassification",
    "ViltForImageAndTextRetrieval",
206
    "ViltForTokenClassification",
NielsRogge's avatar
NielsRogge committed
207
    "ViltForMaskedLM",
NielsRogge's avatar
NielsRogge committed
208
209
    "PerceiverForMultimodalAutoencoding",
    "PerceiverForOpticalFlow",
NielsRogge's avatar
NielsRogge committed
210
    "SegformerDecodeHead",
211
    "TFSegformerDecodeHead",
Kamal Raj's avatar
Kamal Raj committed
212
    "FlaxBeitForMaskedImageModeling",
NielsRogge's avatar
NielsRogge committed
213
    "BeitForMaskedImageModeling",
214
215
    "ChineseCLIPTextModel",
    "ChineseCLIPVisionModel",
Suraj Patil's avatar
Suraj Patil committed
216
    "CLIPTextModel",
217
218
    "CLIPTextModelWithProjection",
    "CLIPVisionModelWithProjection",
Susnato Dhar's avatar
Susnato Dhar committed
219
220
    "ClvpForCausalLM",
    "ClvpModel",
221
222
    "GroupViTTextModel",
    "GroupViTVisionModel",
Yih-Dar's avatar
Yih-Dar committed
223
224
    "TFCLIPTextModel",
    "TFCLIPVisionModel",
225
226
    "TFGroupViTTextModel",
    "TFGroupViTVisionModel",
Suraj Patil's avatar
Suraj Patil committed
227
    "FlaxCLIPTextModel",
228
    "FlaxCLIPTextModelWithProjection",
Suraj Patil's avatar
Suraj Patil committed
229
    "FlaxCLIPVisionModel",
230
    "FlaxWav2Vec2ForCTC",
NielsRogge's avatar
NielsRogge committed
231
    "DetrForSegmentation",
Younes Belkada's avatar
Younes Belkada committed
232
233
234
    "Pix2StructVisionModel",
    "Pix2StructTextModel",
    "Pix2StructForConditionalGeneration",
235
    "ConditionalDetrForSegmentation",
236
237
    "DPRReader",
    "FlaubertForQuestionAnswering",
238
239
240
241
    "FlavaImageCodebook",
    "FlavaTextModel",
    "FlavaImageModel",
    "FlavaMultimodalModel",
242
    "GPT2DoubleHeadsModel",
243
    "GPTSw3DoubleHeadsModel",
NielsRogge's avatar
NielsRogge committed
244
245
    "InstructBlipVisionModel",
    "InstructBlipQFormerModel",
246
    "LayoutLMForQuestionAnswering",
Ryokan RI's avatar
Ryokan RI committed
247
    "LukeForMaskedLM",
NielsRogge's avatar
NielsRogge committed
248
249
250
    "LukeForEntityClassification",
    "LukeForEntityPairClassification",
    "LukeForEntitySpanClassification",
wangpeng's avatar
wangpeng committed
251
    "MgpstrModel",
252
    "OpenAIGPTDoubleHeadsModel",
253
254
    "OwlViTTextModel",
    "OwlViTVisionModel",
NielsRogge's avatar
NielsRogge committed
255
256
    "Owlv2TextModel",
    "Owlv2VisionModel",
257
    "OwlViTForObjectDetection",
258
259
    "PatchTSMixerForPrediction",
    "PatchTSMixerForPretraining",
260
261
262
    "RagModel",
    "RagSequenceForGeneration",
    "RagTokenForGeneration",
Li-Huai (Allan) Lin's avatar
Li-Huai (Allan) Lin committed
263
264
265
266
    "RealmEmbedder",
    "RealmForOpenQA",
    "RealmScorer",
    "RealmReader",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
267
    "TFDPRReader",
268
    "TFGPT2DoubleHeadsModel",
269
    "TFLayoutLMForQuestionAnswering",
270
    "TFOpenAIGPTDoubleHeadsModel",
Ratthachat (Jung)'s avatar
Ratthachat (Jung) committed
271
272
273
    "TFRagModel",
    "TFRagSequenceForGeneration",
    "TFRagTokenForGeneration",
274
    "Wav2Vec2ForCTC",
Patrick von Platen's avatar
Patrick von Platen committed
275
    "HubertForCTC",
276
277
    "SEWForCTC",
    "SEWDForCTC",
278
279
    "XLMForQuestionAnswering",
    "XLNetForQuestionAnswering",
abhishek thakur's avatar
abhishek thakur committed
280
    "SeparableConv1D",
Gunjan Chhablani's avatar
Gunjan Chhablani committed
281
282
283
284
    "VisualBertForRegionToPhraseAlignment",
    "VisualBertForVisualReasoning",
    "VisualBertForQuestionAnswering",
    "VisualBertForMultipleChoice",
Will Rice's avatar
Will Rice committed
285
    "TFWav2Vec2ForCTC",
Will Rice's avatar
Will Rice committed
286
    "TFHubertForCTC",
NielsRogge's avatar
NielsRogge committed
287
288
    "XCLIPVisionModel",
    "XCLIPTextModel",
Jongjyh's avatar
Jongjyh committed
289
290
291
    "AltCLIPTextModel",
    "AltCLIPVisionModel",
    "AltRobertaModel",
Zineng Tang's avatar
Zineng Tang committed
292
    "TvltForAudioVisualClassification",
Yoach Lacombe's avatar
Yoach Lacombe committed
293
294
295
296
    "BarkCausalModel",
    "BarkCoarseModel",
    "BarkFineModel",
    "BarkSemanticModel",
Yoach Lacombe's avatar
Yoach Lacombe committed
297
    "MusicgenMelodyModel",
Yoach Lacombe's avatar
Yoach Lacombe committed
298
299
    "MusicgenModel",
    "MusicgenForConditionalGeneration",
300
301
302
    "SpeechT5ForSpeechToSpeech",
    "SpeechT5ForTextToSpeech",
    "SpeechT5HifiGan",
NielsRogge's avatar
NielsRogge committed
303
    "VitMatteForImageMatting",
304
305
306
307
    "SeamlessM4TTextToUnitModel",
    "SeamlessM4TTextToUnitForConditionalGeneration",
    "SeamlessM4TCodeHifiGan",
    "SeamlessM4TForSpeechToSpeech",  # no auto class for speech-to-speech
jiqing-feng's avatar
jiqing-feng committed
308
    "TvpForVideoGrounding",
NielsRogge's avatar
NielsRogge committed
309
    "UdopForConditionalGeneration",
Yoach Lacombe's avatar
Yoach Lacombe committed
310
311
312
313
    "SeamlessM4Tv2NARTextToUnitModel",
    "SeamlessM4Tv2NARTextToUnitForConditionalGeneration",
    "SeamlessM4Tv2CodeHifiGan",
    "SeamlessM4Tv2ForSpeechToSpeech",  # no auto class for speech-to-speech
Eduardo Pacheco's avatar
Eduardo Pacheco committed
314
    "SegGptForImageSegmentation",
NielsRogge's avatar
NielsRogge committed
315
316
    "SiglipVisionModel",
    "SiglipTextModel",
317
318
]

319
# DO NOT edit this list!
Sylvain Gugger's avatar
Sylvain Gugger committed
320
# (The corresponding pytorch objects should never have been in the main `__init__`, but it's too late to remove)
321
322
323
324
325
326
327
328
329
330
331
332
333
OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK = [
    "FlaxBertLayer",
    "FlaxBigBirdLayer",
    "FlaxRoFormerLayer",
    "TFBertLayer",
    "TFLxmertEncoder",
    "TFLxmertXLayer",
    "TFMPNetLayer",
    "TFMobileBertLayer",
    "TFSegformerLayer",
    "TFViTMAELayer",
]

Sylvain Gugger's avatar
Sylvain Gugger committed
334
# Update this list for models that have multiple model types for the same model doc.
335
336
337
338
MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(
    [
        ("data2vec-text", "data2vec"),
        ("data2vec-audio", "data2vec"),
339
        ("data2vec-vision", "data2vec"),
NielsRogge's avatar
NielsRogge committed
340
        ("donut-swin", "donut"),
341
342
343
344
    ]
)


345
# This is to make sure the transformers module imported is the one in the repo.
346
transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)
347
348


349
def check_missing_backends():
Sylvain Gugger's avatar
Sylvain Gugger committed
350
351
352
353
    """
    Checks if all backends are installed (otherwise the check of this script is incomplete). Will error in the CI if
    that's not the case but only throw a warning for users running this.
    """
354
355
356
357
358
359
360
361
362
363
364
    missing_backends = []
    if not is_torch_available():
        missing_backends.append("PyTorch")
    if not is_tf_available():
        missing_backends.append("TensorFlow")
    if not is_flax_available():
        missing_backends.append("Flax")
    if len(missing_backends) > 0:
        missing = ", ".join(missing_backends)
        if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES:
            raise Exception(
365
                "Full repo consistency checks require all backends to be installed (with `pip install -e '.[dev]'` in the "
366
367
368
369
                f"Transformers repo, the following are missing: {missing}."
            )
        else:
            warnings.warn(
370
                "Full repo consistency checks require all backends to be installed (with `pip install -e '.[dev]'` in the "
371
372
373
374
375
376
                f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you "
                "didn't make any change in one of those backends modeling files, you should probably execute the "
                "command above to be on the safe side."
            )


377
def check_model_list():
Sylvain Gugger's avatar
Sylvain Gugger committed
378
379
380
    """
    Checks the model listed as subfolders of `models` match the models available in `transformers.models`.
    """
381
382
383
384
    # Get the models from the directory structure of `src/transformers/models/`
    models_dir = os.path.join(PATH_TO_TRANSFORMERS, "models")
    _models = []
    for model in os.listdir(models_dir):
Sylvain Gugger's avatar
Sylvain Gugger committed
385
386
        if model == "deprecated":
            continue
387
388
389
390
        model_dir = os.path.join(models_dir, model)
        if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir):
            _models.append(model)

Sylvain Gugger's avatar
Sylvain Gugger committed
391
    # Get the models in the submodule `transformers.models`
392
393
    models = [model for model in dir(transformers.models) if not model.startswith("__")]

394
    missing_models = sorted(set(_models).difference(models))
395
396
397
398
399
400
    if missing_models:
        raise Exception(
            f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}."
        )


401
402
# If some modeling modules should be ignored for all checks, they should be added in the nested list
# _ignore_modules of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
403
404
def get_model_modules() -> List[str]:
    """Get all the model modules inside the transformers library (except deprecated models)."""
405
406
407
408
409
410
411
412
    _ignore_modules = [
        "modeling_auto",
        "modeling_encoder_decoder",
        "modeling_marian",
        "modeling_mmbt",
        "modeling_outputs",
        "modeling_retribert",
        "modeling_utils",
Sylvain Gugger's avatar
Sylvain Gugger committed
413
        "modeling_flax_auto",
414
        "modeling_flax_encoder_decoder",
Stas Bekman's avatar
Stas Bekman committed
415
        "modeling_flax_utils",
416
        "modeling_speech_encoder_decoder",
417
        "modeling_flax_speech_encoder_decoder",
418
        "modeling_flax_vision_encoder_decoder",
amyeroberts's avatar
amyeroberts committed
419
        "modeling_timm_backbone",
420
        "modeling_tf_auto",
421
        "modeling_tf_encoder_decoder",
422
423
424
        "modeling_tf_outputs",
        "modeling_tf_pytorch_utils",
        "modeling_tf_utils",
425
        "modeling_tf_vision_encoder_decoder",
426
        "modeling_vision_encoder_decoder",
427
428
    ]
    modules = []
Sylvain Gugger's avatar
Sylvain Gugger committed
429
430
    for model in dir(transformers.models):
        # There are some magic dunder attributes in the dir, we ignore them
Sylvain Gugger's avatar
Sylvain Gugger committed
431
432
433
434
435
436
437
438
439
        if model == "deprecated" or model.startswith("__"):
            continue

        model_module = getattr(transformers.models, model)
        for submodule in dir(model_module):
            if submodule.startswith("modeling") and submodule not in _ignore_modules:
                modeling_module = getattr(model_module, submodule)
                if inspect.ismodule(modeling_module):
                    modules.append(modeling_module)
440
441
442
    return modules


Sylvain Gugger's avatar
Sylvain Gugger committed
443
444
445
446
447
448
449
450
451
452
453
454
455
def get_models(module: types.ModuleType, include_pretrained: bool = False) -> List[Tuple[str, type]]:
    """
    Get the objects in a module that are models.

    Args:
        module (`types.ModuleType`):
            The module from which we are extracting models.
        include_pretrained (`bool`, *optional*, defaults to `False`):
            Whether or not to include the `PreTrainedModel` subclass (like `BertPreTrainedModel`) or not.

    Returns:
        List[Tuple[str, type]]: List of models as tuples (class name, actual class).
    """
456
    models = []
457
    model_classes = (transformers.PreTrainedModel, transformers.TFPreTrainedModel, transformers.FlaxPreTrainedModel)
458
    for attr_name in dir(module):
459
        if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name):
460
461
462
463
464
465
466
            continue
        attr = getattr(module, attr_name)
        if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__:
            models.append((attr_name, attr))
    return models


Sylvain Gugger's avatar
Sylvain Gugger committed
467
468
469
470
def is_building_block(model: str) -> bool:
    """
    Returns `True` if a model is a building block part of a bigger model.
    """
471
472
473
474
475
476
    if model.endswith("Wrapper"):
        return True
    if model.endswith("Encoder"):
        return True
    if model.endswith("Decoder"):
        return True
477
478
    if model.endswith("Prenet"):
        return True
Sylvain Gugger's avatar
Sylvain Gugger committed
479
480
481
482
483
484
485


def is_a_private_model(model: str) -> bool:
    """Returns `True` if the model should not be in the main init."""
    if model in PRIVATE_MODELS:
        return True
    return is_building_block(model)
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502


def check_models_are_in_init():
    """Checks all models defined in the library are in the main init."""
    models_not_in_init = []
    dir_transformers = dir(transformers)
    for module in get_model_modules():
        models_not_in_init += [
            model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers
        ]

    # Remove private models
    models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)]
    if len(models_not_in_init) > 0:
        raise Exception(f"The following models should be in the main init: {','.join(models_not_in_init)}.")


503
504
# If some test_modeling files should be ignored when checking models are all tested, they should be added in the
# nested list _ignore_files of this function.
Sylvain Gugger's avatar
Sylvain Gugger committed
505
506
507
def get_model_test_files() -> List[str]:
    """
    Get the model test files.
Yih-Dar's avatar
Yih-Dar committed
508

Sylvain Gugger's avatar
Sylvain Gugger committed
509
510
511
512
    Returns:
        `List[str]`: The list of test files. The returned files will NOT contain the `tests` (i.e. `PATH_TO_TESTS`
        defined in this script). They will be considered as paths relative to `tests`. A caller has to use
        `os.path.join(PATH_TO_TESTS, ...)` to access the files.
Yih-Dar's avatar
Yih-Dar committed
513
514
    """

515
516
517
    _ignore_files = [
        "test_modeling_common",
        "test_modeling_encoder_decoder",
518
        "test_modeling_flax_encoder_decoder",
519
        "test_modeling_flax_speech_encoder_decoder",
520
521
        "test_modeling_marian",
        "test_modeling_tf_common",
522
        "test_modeling_tf_encoder_decoder",
523
524
    ]
    test_files = []
Yih-Dar's avatar
Yih-Dar committed
525
526
527
528
529
530
531
532
533
534
535
536
    model_test_root = os.path.join(PATH_TO_TESTS, "models")
    model_test_dirs = []
    for x in os.listdir(model_test_root):
        x = os.path.join(model_test_root, x)
        if os.path.isdir(x):
            model_test_dirs.append(x)

    for target_dir in [PATH_TO_TESTS] + model_test_dirs:
        for file_or_dir in os.listdir(target_dir):
            path = os.path.join(target_dir, file_or_dir)
            if os.path.isfile(path):
                filename = os.path.split(path)[-1]
537
                if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files:
Yih-Dar's avatar
Yih-Dar committed
538
539
540
                    file = os.path.join(*path.split(os.sep)[1:])
                    test_files.append(file)

541
542
543
544
545
    return test_files


# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class
# for the all_model_classes variable.
Sylvain Gugger's avatar
Sylvain Gugger committed
546
547
548
549
550
551
552
553
554
555
556
def find_tested_models(test_file: str) -> List[str]:
    """
    Parse the content of test_file to detect what's in `all_model_classes`. This detects the models that inherit from
    the common test class.

    Args:
        test_file (`str`): The path to the test file to check

    Returns:
        `List[str]`: The list of models tested in that file.
    """
557
    with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f:
558
        content = f.read()
Sylvain Gugger's avatar
Sylvain Gugger committed
559
    all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content)
560
561
    # Check with one less parenthesis as well
    all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
562
    if len(all_models) > 0:
563
        model_tested = []
Sylvain Gugger's avatar
Sylvain Gugger committed
564
565
566
567
568
        for entry in all_models:
            for line in entry.split(","):
                name = line.strip()
                if len(name) > 0:
                    model_tested.append(name)
569
570
571
        return model_tested


Sylvain Gugger's avatar
Sylvain Gugger committed
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
def should_be_tested(model_name: str) -> bool:
    """
    Whether or not a model should be tested.
    """
    if model_name in IGNORE_NON_TESTED:
        return False
    return not is_building_block(model_name)


def check_models_are_tested(module: types.ModuleType, test_file: str) -> List[str]:
    """Check models defined in a module are all tested in a given file.

    Args:
        module (`types.ModuleType`): The module in which we get the models.
        test_file (`str`): The path to the file where the module is tested.

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
591
    # XxxPreTrainedModel are not tested
592
593
594
    defined_models = get_models(module)
    tested_models = find_tested_models(test_file)
    if tested_models is None:
595
        if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS:
596
597
598
599
600
601
602
603
            return
        return [
            f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. "
            + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file "
            + "`utils/check_repo.py`."
        ]
    failures = []
    for model_name, _ in defined_models:
Sylvain Gugger's avatar
Sylvain Gugger committed
604
        if model_name not in tested_models and should_be_tested(model_name):
605
606
607
608
609
610
611
612
613
614
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not tested in "
                + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file."
                + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`"
                + "in the file `utils/check_repo.py`."
            )
    return failures


def check_all_models_are_tested():
Patrick von Platen's avatar
Patrick von Platen committed
615
    """Check all models are properly tested."""
616
617
618
619
    modules = get_model_modules()
    test_files = get_model_test_files()
    failures = []
    for module in modules:
Sylvain Gugger's avatar
Sylvain Gugger committed
620
        # Matches a module to its test file.
621
622
        test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file]
        if len(test_file) == 0:
623
            failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")
624
625
626
627
        elif len(test_file) > 1:
            failures.append(f"{module.__name__} has several test files: {test_file}.")
        else:
            test_file = test_file[0]
628
629
630
            new_failures = check_models_are_tested(module, test_file)
            if new_failures is not None:
                failures += new_failures
631
632
633
634
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
635
def get_all_auto_configured_models() -> List[str]:
Patrick von Platen's avatar
Patrick von Platen committed
636
    """Return the list of all models in at least one auto class."""
637
    result = set()  # To avoid duplicates we concatenate all model classes in a set.
638
639
    if is_torch_available():
        for attr_name in dir(transformers.models.auto.modeling_auto):
640
            if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"):
641
642
643
                result = result | set(get_values(getattr(transformers.models.auto.modeling_auto, attr_name)))
    if is_tf_available():
        for attr_name in dir(transformers.models.auto.modeling_tf_auto):
644
            if attr_name.startswith("TF_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
645
646
647
                result = result | set(get_values(getattr(transformers.models.auto.modeling_tf_auto, attr_name)))
    if is_flax_available():
        for attr_name in dir(transformers.models.auto.modeling_flax_auto):
648
            if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
649
                result = result | set(get_values(getattr(transformers.models.auto.modeling_flax_auto, attr_name)))
650
    return list(result)
651
652


Sylvain Gugger's avatar
Sylvain Gugger committed
653
654
def ignore_unautoclassed(model_name: str) -> bool:
    """Rules to determine if a model should be in an auto class."""
655
656
657
658
659
660
661
662
663
    # Special white list
    if model_name in IGNORE_NON_AUTO_CONFIGURED:
        return True
    # Encoder and Decoder should be ignored
    if "Encoder" in model_name or "Decoder" in model_name:
        return True
    return False


Sylvain Gugger's avatar
Sylvain Gugger committed
664
665
666
667
668
669
670
671
672
673
674
675
676
def check_models_are_auto_configured(module: types.ModuleType, all_auto_models: List[str]) -> List[str]:
    """
    Check models defined in module are each in an auto class.

    Args:
        module (`types.ModuleType`):
            The module in which we get the models.
        all_auto_models (`List[str]`):
            The list of all models in an auto class (as obtained with `get_all_auto_configured_models()`).

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
677
678
679
    defined_models = get_models(module)
    failures = []
    for model_name, _ in defined_models:
680
        if model_name not in all_auto_models and not ignore_unautoclassed(model_name):
681
682
683
684
685
686
687
688
689
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "
                "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "
                "`utils/check_repo.py`."
            )
    return failures


def check_all_models_are_auto_configured():
Patrick von Platen's avatar
Patrick von Platen committed
690
    """Check all models are each in an auto class."""
Sylvain Gugger's avatar
Sylvain Gugger committed
691
    # This is where we need to check we have all backends or the check is incomplete.
692
    check_missing_backends()
693
694
695
696
697
698
699
700
701
702
703
    modules = get_model_modules()
    all_auto_models = get_all_auto_configured_models()
    failures = []
    for module in modules:
        new_failures = check_models_are_auto_configured(module, all_auto_models)
        if new_failures is not None:
            failures += new_failures
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


704
705
def check_all_auto_object_names_being_defined():
    """Check all names defined in auto (name) mappings exist in the library."""
Sylvain Gugger's avatar
Sylvain Gugger committed
706
    # This is where we need to check we have all backends or the check is incomplete.
707
    check_missing_backends()
708

709
    failures = []
710
    mappings_to_check = {
711
712
713
714
715
716
        "TOKENIZER_MAPPING_NAMES": TOKENIZER_MAPPING_NAMES,
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

717
718
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
719
720
721
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
722
723
724
725
726
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
727
        for _, class_names in mapping.items():
728
729
730
731
732
733
734
735
736
737
738
            if not isinstance(class_names, tuple):
                class_names = (class_names,)
                for class_name in class_names:
                    if class_name is None:
                        continue
                    # dummy object is accepted
                    if not hasattr(transformers, class_name):
                        # If the class name is in a model name mapping, let's not check if there is a definition in any modeling
                        # module, if it's a private model defined in this file.
                        if name.endswith("MODEL_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
739
740
                        if name.endswith("MODEL_FOR_IMAGE_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
741
742
743
744
745
746
747
                        failures.append(
                            f"`{class_name}` appears in the mapping `{name}` but it is not defined in the library."
                        )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Yih-Dar's avatar
Yih-Dar committed
748
749
def check_all_auto_mapping_names_in_config_mapping_names():
    """Check all keys defined in auto mappings (mappings of names) appear in `CONFIG_MAPPING_NAMES`."""
Sylvain Gugger's avatar
Sylvain Gugger committed
750
    # This is where we need to check we have all backends or the check is incomplete.
751
    check_missing_backends()
Yih-Dar's avatar
Yih-Dar committed
752

753
    failures = []
Yih-Dar's avatar
Yih-Dar committed
754
    # `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer` is special, and don't need to follow the rule.
755
    mappings_to_check = {
Yih-Dar's avatar
Yih-Dar committed
756
757
758
759
760
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

761
762
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
763
764
765
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
766
767
768
769
770
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
Sylvain Gugger's avatar
Sylvain Gugger committed
771
        for model_type in mapping:
Yih-Dar's avatar
Yih-Dar committed
772
773
774
775
776
777
778
779
780
            if model_type not in CONFIG_MAPPING_NAMES:
                failures.append(
                    f"`{model_type}` appears in the mapping `{name}` but it is not defined in the keys of "
                    "`CONFIG_MAPPING_NAMES`."
                )
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


781
def check_all_auto_mappings_importable():
Sylvain Gugger's avatar
Sylvain Gugger committed
782
783
    """Check all auto mappings can be imported."""
    # This is where we need to check we have all backends or the check is incomplete.
784
785
786
787
788
789
790
791
792
793
794
795
796
    check_missing_backends()

    failures = []
    mappings_to_check = {}
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

Sylvain Gugger's avatar
Sylvain Gugger committed
797
    for name in mappings_to_check:
798
799
        name = name.replace("_MAPPING_NAMES", "_MAPPING")
        if not hasattr(transformers, name):
800
801
802
803
804
805
            failures.append(f"`{name}`")
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


def check_objects_being_equally_in_main_init():
Sylvain Gugger's avatar
Sylvain Gugger committed
806
807
808
    """
    Check if a (TensorFlow or Flax) object is in the main __init__ iif its counterpart in PyTorch is.
    """
809
810
811
812
813
    attrs = dir(transformers)

    failures = []
    for attr in attrs:
        obj = getattr(transformers, attr)
Sylvain Gugger's avatar
Sylvain Gugger committed
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
        if not hasattr(obj, "__module__") or "models.deprecated" in obj.__module__:
            continue

        module_path = obj.__module__
        module_name = module_path.split(".")[-1]
        module_dir = ".".join(module_path.split(".")[:-1])
        if (
            module_name.startswith("modeling_")
            and not module_name.startswith("modeling_tf_")
            and not module_name.startswith("modeling_flax_")
        ):
            parent_module = sys.modules[module_dir]

            frameworks = []
            if is_tf_available():
                frameworks.append("TF")
            if is_flax_available():
                frameworks.append("Flax")

            for framework in frameworks:
                other_module_path = module_path.replace("modeling_", f"modeling_{framework.lower()}_")
                if os.path.isfile("src/" + other_module_path.replace(".", "/") + ".py"):
                    other_module_name = module_name.replace("modeling_", f"modeling_{framework.lower()}_")
                    other_module = getattr(parent_module, other_module_name)
                    if hasattr(other_module, f"{framework}{attr}"):
                        if not hasattr(transformers, f"{framework}{attr}"):
                            if f"{framework}{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}{attr}")
                    if hasattr(other_module, f"{framework}_{attr}"):
                        if not hasattr(transformers, f"{framework}_{attr}"):
                            if f"{framework}_{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}_{attr}")
846
847
848
849
    if len(failures) > 0:
        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))


Sylvain Gugger's avatar
Sylvain Gugger committed
850
851
852
_re_decorator = re.compile(r"^\s*@(\S+)\s+$")


Sylvain Gugger's avatar
Sylvain Gugger committed
853
854
855
856
857
858
859
860
861
862
def check_decorator_order(filename: str) -> List[int]:
    """
    Check that in a given test file, the slow decorator is always last.

    Args:
        filename (`str`): The path to a test file to check.

    Returns:
        `List[int]`: The list of failures as a list of indices where there are problems.
    """
863
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
Sylvain Gugger's avatar
Sylvain Gugger committed
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
        lines = f.readlines()
    decorator_before = None
    errors = []
    for i, line in enumerate(lines):
        search = _re_decorator.search(line)
        if search is not None:
            decorator_name = search.groups()[0]
            if decorator_before is not None and decorator_name.startswith("parameterized"):
                errors.append(i)
            decorator_before = decorator_name
        elif decorator_before is not None:
            decorator_before = None
    return errors


def check_all_decorator_order():
Patrick von Platen's avatar
Patrick von Platen committed
880
    """Check that in all test files, the slow decorator is always last."""
Sylvain Gugger's avatar
Sylvain Gugger committed
881
882
883
884
885
886
887
888
889
    errors = []
    for fname in os.listdir(PATH_TO_TESTS):
        if fname.endswith(".py"):
            filename = os.path.join(PATH_TO_TESTS, fname)
            new_errors = check_decorator_order(filename)
            errors += [f"- {filename}, line {i}" for i in new_errors]
    if len(errors) > 0:
        msg = "\n".join(errors)
        raise ValueError(
Sylvain Gugger's avatar
Sylvain Gugger committed
890
891
            "The parameterized decorator (and its variants) should always be first, but this is not the case in the"
            f" following files:\n{msg}"
Sylvain Gugger's avatar
Sylvain Gugger committed
892
893
894
        )


Sylvain Gugger's avatar
Sylvain Gugger committed
895
896
897
898
899
900
901
def find_all_documented_objects() -> List[str]:
    """
    Parse the content of all doc files to detect which classes and functions it documents.

    Returns:
        `List[str]`: The list of all object names being documented.
    """
902
903
    documented_obj = []
    for doc_file in Path(PATH_TO_DOC).glob("**/*.rst"):
Julien Plu's avatar
Julien Plu committed
904
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
905
906
907
            content = f.read()
        raw_doc_objs = re.findall(r"(?:autoclass|autofunction):: transformers.(\S+)\s+", content)
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
908
    for doc_file in Path(PATH_TO_DOC).glob("**/*.md"):
Sylvain Gugger's avatar
Sylvain Gugger committed
909
910
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
            content = f.read()
911
        raw_doc_objs = re.findall(r"\[\[autodoc\]\]\s+(\S+)\s+", content)
Sylvain Gugger's avatar
Sylvain Gugger committed
912
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]
913
914
915
916
917
918
    return documented_obj


# One good reason for not being documented is to be deprecated. Put in this list deprecated objects.
DEPRECATED_OBJECTS = [
    "AutoModelWithLMHead",
919
    "BartPretrainedModel",
920
921
    "DataCollator",
    "DataCollatorForSOP",
922
923
924
925
926
    "GlueDataset",
    "GlueDataTrainingArguments",
    "LineByLineTextDataset",
    "LineByLineWithRefDataset",
    "LineByLineWithSOPTextDataset",
Steven Liu's avatar
Steven Liu committed
927
    "NerPipeline",
928
929
930
931
932
933
934
935
936
937
    "PretrainedBartModel",
    "PretrainedFSMTModel",
    "SingleSentenceClassificationProcessor",
    "SquadDataTrainingArguments",
    "SquadDataset",
    "SquadExample",
    "SquadFeatures",
    "SquadV1Processor",
    "SquadV2Processor",
    "TFAutoModelWithLMHead",
938
    "TFBartPretrainedModel",
939
940
    "TextDataset",
    "TextDatasetForNextSentencePrediction",
941
    "Wav2Vec2ForMaskedLM",
942
    "Wav2Vec2Tokenizer",
943
944
945
946
947
948
949
950
951
952
    "glue_compute_metrics",
    "glue_convert_examples_to_features",
    "glue_output_modes",
    "glue_processors",
    "glue_tasks_num_labels",
    "squad_convert_examples_to_features",
    "xnli_compute_metrics",
    "xnli_output_modes",
    "xnli_processors",
    "xnli_tasks_num_labels",
953
    "TFTrainingArguments",
954
955
956
957
958
959
960
]

# Exceptionally, some objects should not be documented after all rules passed.
# ONLY PUT SOMETHING IN THIS LIST AS A LAST RESORT!
UNDOCUMENTED_OBJECTS = [
    "AddedToken",  # This is a tokenizers class.
    "BasicTokenizer",  # Internal, should never have been in the main init.
961
    "CharacterTokenizer",  # Internal, should never have been in the main init.
962
    "DPRPretrainedReader",  # Like an Encoder.
Sylvain Gugger's avatar
Sylvain Gugger committed
963
    "DummyObject",  # Just picked by mistake sometimes.
964
    "MecabTokenizer",  # Internal, should never have been in the main init.
965
966
967
968
969
970
971
972
973
974
975
    "ModelCard",  # Internal type.
    "SqueezeBertModule",  # Internal building block (should have been called SqueezeBertLayer)
    "TFDPRPretrainedReader",  # Like an Encoder.
    "TransfoXLCorpus",  # Internal type.
    "WordpieceTokenizer",  # Internal, should never have been in the main init.
    "absl",  # External module
    "add_end_docstrings",  # Internal, should never have been in the main init.
    "add_start_docstrings",  # Internal, should never have been in the main init.
    "convert_tf_weight_name_to_pt_weight_name",  # Internal used to convert model weights
    "logger",  # Internal logger
    "logging",  # External module
976
    "requires_backends",  # Internal function
Jongjyh's avatar
Jongjyh committed
977
    "AltRobertaModel",  # Internal module
978
979
980
981
982
983
984
985
986
]

# This list should be empty. Objects in it should get their own doc page.
SHOULD_HAVE_THEIR_OWN_PAGE = [
    # Benchmarks
    "PyTorchBenchmark",
    "PyTorchBenchmarkArguments",
    "TensorFlowBenchmark",
    "TensorFlowBenchmarkArguments",
987
    "AutoBackbone",
NielsRogge's avatar
NielsRogge committed
988
    "BeitBackbone",
NielsRogge's avatar
NielsRogge committed
989
990
    "BitBackbone",
    "ConvNextBackbone",
Alara Dirik's avatar
Alara Dirik committed
991
    "ConvNextV2Backbone",
992
    "DinatBackbone",
993
    "Dinov2Backbone",
Alara Dirik's avatar
Alara Dirik committed
994
    "FocalNetBackbone",
NielsRogge's avatar
NielsRogge committed
995
    "MaskFormerSwinBackbone",
996
997
    "MaskFormerSwinConfig",
    "MaskFormerSwinModel",
NielsRogge's avatar
NielsRogge committed
998
    "NatBackbone",
Nate Cibik's avatar
Nate Cibik committed
999
    "PvtV2Backbone",
NielsRogge's avatar
NielsRogge committed
1000
    "ResNetBackbone",
NielsRogge's avatar
NielsRogge committed
1001
    "SwinBackbone",
NielsRogge's avatar
NielsRogge committed
1002
    "Swinv2Backbone",
amyeroberts's avatar
amyeroberts committed
1003
1004
    "TimmBackbone",
    "TimmBackboneConfig",
NielsRogge's avatar
NielsRogge committed
1005
    "VitDetBackbone",
1006
1007
1008
]


Sylvain Gugger's avatar
Sylvain Gugger committed
1009
1010
def ignore_undocumented(name: str) -> bool:
    """Rules to determine if `name` should be undocumented (returns `True` if it should not be documented)."""
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
    # NOT DOCUMENTED ON PURPOSE.
    # Constants uppercase are not documented.
    if name.isupper():
        return True
    # PreTrainedModels / Encoders / Decoders / Layers / Embeddings / Attention are not documented.
    if (
        name.endswith("PreTrainedModel")
        or name.endswith("Decoder")
        or name.endswith("Encoder")
        or name.endswith("Layer")
        or name.endswith("Embeddings")
        or name.endswith("Attention")
    ):
        return True
    # Submodules are not documented.
    if os.path.isdir(os.path.join(PATH_TO_TRANSFORMERS, name)) or os.path.isfile(
        os.path.join(PATH_TO_TRANSFORMERS, f"{name}.py")
    ):
        return True
    # All load functions are not documented.
    if name.startswith("load_tf") or name.startswith("load_pytorch"):
        return True
    # is_xxx_available functions are not documented.
    if name.startswith("is_") and name.endswith("_available"):
        return True
    # Deprecated objects are not documented.
    if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS:
        return True
    # MMBT model does not really work.
    if name.startswith("MMBT"):
        return True
    if name in SHOULD_HAVE_THEIR_OWN_PAGE:
        return True
    return False


def check_all_objects_are_documented():
Patrick von Platen's avatar
Patrick von Platen committed
1048
    """Check all models are properly documented."""
1049
    documented_objs = find_all_documented_objects()
1050
1051
1052
    modules = transformers._modules
    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]
    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]
1053
1054
1055
1056
1057
    if len(undocumented_objs) > 0:
        raise Exception(
            "The following objects are in the public init so should be documented:\n - "
            + "\n - ".join(undocumented_objs)
        )
1058
    check_docstrings_are_in_md()
1059
1060
1061
1062
1063
1064
    check_model_type_doc_match()


def check_model_type_doc_match():
    """Check all doc pages have a corresponding model type."""
    model_doc_folder = Path(PATH_TO_DOC) / "model_doc"
1065
    model_docs = [m.stem for m in model_doc_folder.glob("*.md")]
1066
1067

    model_types = list(transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys())
1068
    model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types]
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086

    errors = []
    for m in model_docs:
        if m not in model_types and m != "auto":
            close_matches = get_close_matches(m, model_types)
            error_message = f"{m} is not a proper model identifier."
            if len(close_matches) > 0:
                close_matches = "/".join(close_matches)
                error_message += f" Did you mean {close_matches}?"
            errors.append(error_message)

    if len(errors) > 0:
        raise ValueError(
            "Some model doc pages do not match any existing model type:\n"
            + "\n".join(errors)
            + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in "
            "models/auto/configuration_auto.py."
        )
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096


# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`.
_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`")
# Re pattern to catch things between double backquotes.
_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")
# Re pattern to catch example introduction.
_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE)


Sylvain Gugger's avatar
Sylvain Gugger committed
1097
def is_rst_docstring(docstring: str) -> True:
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
    """
    Returns `True` if `docstring` is written in rst.
    """
    if _re_rst_special_words.search(docstring) is not None:
        return True
    if _re_double_backquotes.search(docstring) is not None:
        return True
    if _re_rst_example.search(docstring) is not None:
        return True
    return False


def check_docstrings_are_in_md():
Sylvain Gugger's avatar
Sylvain Gugger committed
1111
    """Check all docstrings are written in md and nor rst."""
1112
1113
    files_with_rst = []
    for file in Path(PATH_TO_TRANSFORMERS).glob("**/*.py"):
1114
        with open(file, encoding="utf-8") as f:
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
            code = f.read()
        docstrings = code.split('"""')

        for idx, docstring in enumerate(docstrings):
            if idx % 2 == 0 or not is_rst_docstring(docstring):
                continue
            files_with_rst.append(file)
            break

    if len(files_with_rst) > 0:
        raise ValueError(
            "The following files have docstrings written in rst:\n"
            + "\n".join([f"- {f}" for f in files_with_rst])
Kamal Raj's avatar
Kamal Raj committed
1128
            + "\nTo fix this run `doc-builder convert path_to_py_file` after installing `doc-builder`\n"
1129
1130
            "(`pip install git+https://github.com/huggingface/doc-builder`)"
        )
1131
1132


1133
def check_deprecated_constant_is_up_to_date():
Sylvain Gugger's avatar
Sylvain Gugger committed
1134
1135
1136
    """
    Check if the constant `DEPRECATED_MODELS` in `models/auto/configuration_auto.py` is up to date.
    """
1137
1138
1139
1140
1141
1142
1143
1144
1145
    deprecated_folder = os.path.join(PATH_TO_TRANSFORMERS, "models", "deprecated")
    deprecated_models = [m for m in os.listdir(deprecated_folder) if not m.startswith("_")]

    constant_to_check = transformers.models.auto.configuration_auto.DEPRECATED_MODELS
    message = []
    missing_models = sorted(set(deprecated_models) - set(constant_to_check))
    if len(missing_models) != 0:
        missing_models = ", ".join(missing_models)
        message.append(
1146
            "The following models are in the deprecated folder, make sure to add them to `DEPRECATED_MODELS` in "
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
            f"`models/auto/configuration_auto.py`: {missing_models}."
        )

    extra_models = sorted(set(constant_to_check) - set(deprecated_models))
    if len(extra_models) != 0:
        extra_models = ", ".join(extra_models)
        message.append(
            "The following models are in the `DEPRECATED_MODELS` constant but not in the deprecated folder. Either "
            f"remove them from the constant or move to the deprecated folder: {extra_models}."
        )

    if len(message) > 0:
        raise Exception("\n".join(message))


1162
def check_repo_quality():
Patrick von Platen's avatar
Patrick von Platen committed
1163
    """Check all models are properly tested and documented."""
1164
1165
    print("Checking all models are included.")
    check_model_list()
1166
1167
    print("Checking all models are public.")
    check_models_are_in_init()
1168
    print("Checking all models are properly tested.")
Sylvain Gugger's avatar
Sylvain Gugger committed
1169
    check_all_decorator_order()
1170
    check_all_models_are_tested()
1171
    print("Checking all objects are properly documented.")
1172
    check_all_objects_are_documented()
1173
1174
    print("Checking all models are in at least one auto class.")
    check_all_models_are_auto_configured()
1175
1176
    print("Checking all names in auto name mappings are defined.")
    check_all_auto_object_names_being_defined()
Yih-Dar's avatar
Yih-Dar committed
1177
1178
    print("Checking all keys in auto name mappings are defined in `CONFIG_MAPPING_NAMES`.")
    check_all_auto_mapping_names_in_config_mapping_names()
1179
1180
    print("Checking all auto mappings could be imported.")
    check_all_auto_mappings_importable()
1181
1182
    print("Checking all objects are equally (across frameworks) in the main __init__.")
    check_objects_being_equally_in_main_init()
1183
1184
    print("Checking the DEPRECATED_MODELS constant is up to date.")
    check_deprecated_constant_is_up_to_date()
1185
1186
1187
1188


if __name__ == "__main__":
    check_repo_quality()