More doctests (#22268)

* all doctests

* Skip failed tests

---------
Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>

utils/documentation_tests.txt

@@ -162,7 +162,6 @@ src/transformers/models/roberta_prelayernorm/configuration_roberta_prelayernorm.
 src/transformers/models/roberta_prelayernorm/modeling_roberta_prelayernorm.py
 src/transformers/models/roberta_prelayernorm/modeling_tf_roberta_prelayernorm.py
 src/transformers/models/roc_bert/modeling_roc_bert.py
-src/transformers/models/roc_bert/tokenization_roc_bert.py
 src/transformers/models/segformer/modeling_segformer.py
 src/transformers/models/sew/configuration_sew.py
 src/transformers/models/sew/modeling_sew.py
@@ -174,7 +173,6 @@ src/transformers/models/speech_to_text/modeling_speech_to_text.py
 src/transformers/models/speech_to_text_2/configuration_speech_to_text_2.py
 src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py
 src/transformers/models/speecht5/modeling_speecht5.py
-src/transformers/models/speecht5/tokenization_speecht5.py
 src/transformers/models/segformer/modeling_tf_segformer.py
 src/transformers/models/squeezebert/configuration_squeezebert.py
 src/transformers/models/swin/configuration_swin.py
@@ -209,10 +207,8 @@ src/transformers/models/vit_msn/modeling_vit_msn.py
 src/transformers/models/visual_bert/configuration_visual_bert.py
 src/transformers/models/wav2vec2/configuration_wav2vec2.py
 src/transformers/models/wav2vec2/modeling_wav2vec2.py
-src/transformers/models/wav2vec2/tokenization_wav2vec2.py
 src/transformers/models/wav2vec2_conformer/configuration_wav2vec2_conformer.py
 src/transformers/models/wav2vec2_conformer/modeling_wav2vec2_conformer.py
-src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py
 src/transformers/models/wavlm/configuration_wavlm.py
 src/transformers/models/wavlm/modeling_wavlm.py
 src/transformers/models/whisper/configuration_whisper.py
@@ -230,3 +226,221 @@ src/transformers/models/x_clip/modeling_x_clip.py
 src/transformers/models/yoso/configuration_yoso.py
 src/transformers/models/timesformer/configuration_timesformer.py
 src/transformers/pipelines/
+src/transformers/models/albert/tokenization_albert.py
+src/transformers/models/albert/tokenization_albert_fast.py
+src/transformers/models/barthez/tokenization_barthez.py
+src/transformers/models/barthez/tokenization_barthez_fast.py
+src/transformers/models/bartpho/tokenization_bartpho.py
+src/transformers/models/bert/tokenization_bert.py
+src/transformers/models/bert/tokenization_bert_fast.py
+src/transformers/models/bert/tokenization_bert_tf.py
+src/transformers/models/bert_generation/tokenization_bert_generation.py
+src/transformers/models/bert_japanese/tokenization_bert_japanese.py
+src/transformers/models/big_bird/tokenization_big_bird.py
+src/transformers/models/big_bird/tokenization_big_bird_fast.py
+src/transformers/models/biogpt/tokenization_biogpt.py
+src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
+src/transformers/models/blenderbot_small/tokenization_blenderbot_small_fast.py
+src/transformers/models/byt5/tokenization_byt5.py
+src/transformers/models/camembert/tokenization_camembert.py
+src/transformers/models/camembert/tokenization_camembert_fast.py
+src/transformers/models/canine/tokenization_canine.py
+src/transformers/models/clip/tokenization_clip.py
+src/transformers/models/clip/tokenization_clip_fast.py
+src/transformers/models/convbert/tokenization_convbert.py
+src/transformers/models/convbert/tokenization_convbert_fast.py
+src/transformers/models/cpm/tokenization_cpm.py
+src/transformers/models/cpm/tokenization_cpm_fast.py
+src/transformers/models/ctrl/tokenization_ctrl.py
+src/transformers/models/deberta_v2/tokenization_deberta_v2.py
+src/transformers/models/deberta_v2/tokenization_deberta_v2_fast.py
+src/transformers/models/distilbert/tokenization_distilbert.py
+src/transformers/models/distilbert/tokenization_distilbert_fast.py
+src/transformers/models/electra/tokenization_electra.py
+src/transformers/models/electra/tokenization_electra_fast.py
+src/transformers/models/ernie_m/tokenization_ernie_m.py
+src/transformers/models/esm/tokenization_esm.py
+src/transformers/models/flaubert/tokenization_flaubert.py
+src/transformers/models/fnet/tokenization_fnet.py
+src/transformers/models/fnet/tokenization_fnet_fast.py
+src/transformers/models/fsmt/tokenization_fsmt.py
+src/transformers/models/funnel/tokenization_funnel.py
+src/transformers/models/funnel/tokenization_funnel_fast.py
+src/transformers/models/gpt2/tokenization_gpt2_tf.py
+src/transformers/models/gpt_neox_japanese/tokenization_gpt_neox_japanese.py
+src/transformers/models/herbert/tokenization_herbert.py
+src/transformers/models/herbert/tokenization_herbert_fast.py
+src/transformers/models/jukebox/tokenization_jukebox.py
+src/transformers/models/layoutlm/tokenization_layoutlm.py
+src/transformers/models/layoutlm/tokenization_layoutlm_fast.py
+src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py
+src/transformers/models/layoutlmv2/tokenization_layoutlmv2_fast.py
+src/transformers/models/layoutlmv3/tokenization_layoutlmv3.py
+src/transformers/models/layoutlmv3/tokenization_layoutlmv3_fast.py
+src/transformers/models/layoutxlm/tokenization_layoutxlm.py
+src/transformers/models/layoutxlm/tokenization_layoutxlm_fast.py
+src/transformers/models/llama/tokenization_llama.py
+src/transformers/models/lxmert/tokenization_lxmert.py
+src/transformers/models/lxmert/tokenization_lxmert_fast.py
+src/transformers/models/markuplm/tokenization_markuplm.py
+src/transformers/models/markuplm/tokenization_markuplm_fast.py
+src/transformers/models/mbart/tokenization_mbart.py
+src/transformers/models/mbart/tokenization_mbart_fast.py
+src/transformers/models/mbart50/tokenization_mbart50.py
+src/transformers/models/mbart50/tokenization_mbart50_fast.py
+src/transformers/models/mgp_str/tokenization_mgp_str.py
+src/transformers/models/mluke/tokenization_mluke.py
+src/transformers/models/mobilebert/tokenization_mobilebert.py
+src/transformers/models/mobilebert/tokenization_mobilebert_fast.py
+src/transformers/models/mpnet/tokenization_mpnet.py
+src/transformers/models/mpnet/tokenization_mpnet_fast.py
+src/transformers/models/nllb/tokenization_nllb.py
+src/transformers/models/nllb/tokenization_nllb_fast.py
+src/transformers/models/openai/tokenization_openai.py
+src/transformers/models/openai/tokenization_openai_fast.py
+src/transformers/models/pegasus/tokenization_pegasus.py
+src/transformers/models/pegasus/tokenization_pegasus_fast.py
+src/transformers/models/perceiver/tokenization_perceiver.py
+src/transformers/models/phobert/tokenization_phobert.py
+src/transformers/models/plbart/tokenization_plbart.py
+src/transformers/models/prophetnet/tokenization_prophetnet.py
+src/transformers/models/rag/tokenization_rag.py
+src/transformers/models/realm/tokenization_realm.py
+src/transformers/models/realm/tokenization_realm_fast.py
+src/transformers/models/reformer/tokenization_reformer.py
+src/transformers/models/reformer/tokenization_reformer_fast.py
+src/transformers/models/rembert/tokenization_rembert.py
+src/transformers/models/rembert/tokenization_rembert_fast.py
+src/transformers/models/retribert/tokenization_retribert.py
+src/transformers/models/retribert/tokenization_retribert_fast.py
+src/transformers/models/roc_bert/tokenization_roc_bert.py
+src/transformers/models/roformer/tokenization_utils.py
+src/transformers/models/speecht5/tokenization_speecht5.py
+src/transformers/models/speech_to_text/tokenization_speech_to_text.py
+src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py
+src/transformers/models/splinter/tokenization_splinter.py
+src/transformers/models/splinter/tokenization_splinter_fast.py
+src/transformers/models/squeezebert/tokenization_squeezebert.py
+src/transformers/models/squeezebert/tokenization_squeezebert_fast.py
+src/transformers/models/t5/tokenization_t5.py
+src/transformers/models/t5/tokenization_t5_fast.py
+src/transformers/models/tapas/tokenization_tapas.py
+src/transformers/models/tapex/tokenization_tapex.py
+src/transformers/models/wav2vec2/tokenization_wav2vec2.py
+src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py
+src/transformers/models/whisper/tokenization_whisper.py
+src/transformers/models/whisper/tokenization_whisper_fast.py
+src/transformers/models/xglm/tokenization_xglm.py
+src/transformers/models/xglm/tokenization_xglm_fast.py
+src/transformers/models/xlm/tokenization_xlm.py
+src/transformers/models/xlm_prophetnet/tokenization_xlm_prophetnet.py
+src/transformers/models/xlm_roberta/tokenization_xlm_roberta.py
+src/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.py
+src/transformers/models/xlnet/tokenization_xlnet.py
+src/transformers/models/xlnet/tokenization_xlnet_fast.py
+src/transformers/models/beit/image_processing_beit.py
+src/transformers/models/bit/image_processing_bit.py
+src/transformers/models/blip/image_processing_blip.py
+src/transformers/models/bridgetower/image_processing_bridgetower.py
+src/transformers/models/chinese_clip/image_processing_chinese_clip.py
+src/transformers/models/clip/image_processing_clip.py
+src/transformers/models/conditional_detr/image_processing_conditional_detr.py
+src/transformers/models/convnext/image_processing_convnext.py
+src/transformers/models/deformable_detr/image_processing_deformable_detr.py
+src/transformers/models/deit/image_processing_deit.py
+src/transformers/models/deta/image_processing_deta.py
+src/transformers/models/detr/image_processing_detr.py
+src/transformers/models/donut/image_processing_donut.py
+src/transformers/models/dpt/image_processing_dpt.py
+src/transformers/models/efficientformer/image_processing_efficientformer.py
+src/transformers/models/efficientnet/image_processing_efficientnet.py
+src/transformers/models/flava/image_processing_flava.py
+src/transformers/models/glpn/image_processing_glpn.py
+src/transformers/models/imagegpt/image_processing_imagegpt.py
+src/transformers/models/layoutlmv2/image_processing_layoutlmv2.py
+src/transformers/models/layoutlmv3/image_processing_layoutlmv3.py
+src/transformers/models/levit/image_processing_levit.py
+src/transformers/models/mask2former/image_processing_mask2former.py
+src/transformers/models/maskformer/image_processing_maskformer.py
+src/transformers/models/mobilenet_v1/image_processing_mobilenet_v1.py
+src/transformers/models/mobilenet_v2/image_processing_mobilenet_v2.py
+src/transformers/models/mobilevit/image_processing_mobilevit.py
+src/transformers/models/oneformer/image_processing_oneformer.py
+src/transformers/models/owlvit/image_processing_owlvit.py
+src/transformers/models/perceiver/image_processing_perceiver.py
+src/transformers/models/poolformer/image_processing_poolformer.py
+src/transformers/models/segformer/image_processing_segformer.py
+src/transformers/models/swin2sr/image_processing_swin2sr.py
+src/transformers/models/tvlt/image_processing_tvlt.py
+src/transformers/models/videomae/image_processing_videomae.py
+src/transformers/models/vilt/image_processing_vilt.py
+src/transformers/models/vit/image_processing_vit.py
+src/transformers/models/vit_hybrid/image_processing_vit_hybrid.py
+src/transformers/models/yolos/image_processing_yolos.py
+src/transformers/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.py
+src/transformers/models/beit/feature_extraction_beit.py
+src/transformers/models/chinese_clip/feature_extraction_chinese_clip.py
+src/transformers/models/clap/feature_extraction_clap.py
+src/transformers/models/clip/feature_extraction_clip.py
+src/transformers/models/conditional_detr/feature_extraction_conditional_detr.py
+src/transformers/models/convnext/feature_extraction_convnext.py
+src/transformers/models/deformable_detr/feature_extraction_deformable_detr.py
+src/transformers/models/deit/feature_extraction_deit.py
+src/transformers/models/detr/feature_extraction_detr.py
+src/transformers/models/donut/feature_extraction_donut.py
+src/transformers/models/dpt/feature_extraction_dpt.py
+src/transformers/models/flava/feature_extraction_flava.py
+src/transformers/models/glpn/feature_extraction_glpn.py
+src/transformers/models/imagegpt/feature_extraction_imagegpt.py
+src/transformers/models/layoutlmv2/feature_extraction_layoutlmv2.py
+src/transformers/models/layoutlmv3/feature_extraction_layoutlmv3.py
+src/transformers/models/levit/feature_extraction_levit.py
+src/transformers/models/maskformer/feature_extraction_maskformer.py
+src/transformers/models/mctct/feature_extraction_mctct.py
+src/transformers/models/mobilenet_v1/feature_extraction_mobilenet_v1.py
+src/transformers/models/mobilenet_v2/feature_extraction_mobilenet_v2.py
+src/transformers/models/mobilevit/feature_extraction_mobilevit.py
+src/transformers/models/owlvit/feature_extraction_owlvit.py
+src/transformers/models/perceiver/feature_extraction_perceiver.py
+src/transformers/models/poolformer/feature_extraction_poolformer.py
+src/transformers/models/segformer/feature_extraction_segformer.py
+src/transformers/models/speecht5/feature_extraction_speecht5.py
+src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py
+src/transformers/models/tvlt/feature_extraction_tvlt.py
+src/transformers/models/videomae/feature_extraction_videomae.py
+src/transformers/models/vilt/feature_extraction_vilt.py
+src/transformers/models/vit/feature_extraction_vit.py
+src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py
+src/transformers/models/whisper/feature_extraction_whisper.py
+src/transformers/models/yolos/feature_extraction_yolos.py
+src/transformers/models/align/processing_align.py
+src/transformers/models/altclip/processing_altclip.py
+src/transformers/models/blip/processing_blip.py
+src/transformers/models/blip_2/processing_blip_2.py
+src/transformers/models/bridgetower/processing_bridgetower.py
+src/transformers/models/chinese_clip/processing_chinese_clip.py
+src/transformers/models/clap/processing_clap.py
+src/transformers/models/clip/processing_clip.py
+src/transformers/models/clipseg/processing_clipseg.py
+src/transformers/models/donut/processing_donut.py
+src/transformers/models/flava/processing_flava.py
+src/transformers/models/git/processing_git.py
+src/transformers/models/layoutlmv2/processing_layoutlmv2.py
+src/transformers/models/layoutlmv3/processing_layoutlmv3.py
+src/transformers/models/layoutxlm/processing_layoutxlm.py
+src/transformers/models/markuplm/processing_markuplm.py
+src/transformers/models/mctct/processing_mctct.py
+src/transformers/models/mgp_str/processing_mgp_str.py
+src/transformers/models/oneformer/processing_oneformer.py
+src/transformers/models/owlvit/processing_owlvit.py
+src/transformers/models/speecht5/processing_speecht5.py
+src/transformers/models/speech_to_text/processing_speech_to_text.py
+src/transformers/models/speech_to_text_2/processing_speech_to_text_2.py
+src/transformers/models/trocr/processing_trocr.py
+src/transformers/models/tvlt/processing_tvlt.py
+src/transformers/models/vilt/processing_vilt.py
+src/transformers/models/vision_text_dual_encoder/processing_vision_text_dual_encoder.py
+src/transformers/models/wav2vec2/processing_wav2vec2.py
+src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py
+src/transformers/models/whisper/processing_whisper.py
+src/transformers/models/x_clip/processing_x_clip.py