Generate: move generation_*.py src files into generation/*.py (#20096)

* move generation_*.py src files into generation/*.py

* populate generation.__init__ with lazy loading

* move imports and references from generation.xxx.object to generation.object

tests/models/speech_to_text_2/test_modeling_speech_to_text_2.py

@@ -19,7 +19,7 @@ import unittest
 from transformers import Speech2Text2Config
 from transformers.testing_utils import is_torch_available, require_torch, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 

tests/models/t5/test_modeling_flax_t5.py

@@ -27,7 +27,7 @@ from transformers.testing_utils import (
     slow,
 )
 
-from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin
+from ...generation.test_flax_utils import FlaxGenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
 

tests/models/t5/test_modeling_t5.py

@@ -22,7 +22,7 @@ from transformers import T5Config, is_torch_available
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 from transformers.utils import cached_property
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 

tests/models/table_transformer/test_modeling_table_transformer.py

@@ -23,7 +23,7 @@ from huggingface_hub import hf_hub_download
 from transformers import TableTransformerConfig, is_timm_available, is_vision_available
 from transformers.testing_utils import require_timm, require_vision, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
 

tests/models/trajectory_transformer/test_modeling_trajectory_transformer.py

@@ -23,7 +23,7 @@ import numpy as np
 from transformers import TrajectoryTransformerConfig, is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, _config_zero_init, random_attention_mask
 

tests/models/transfo_xl/test_modeling_transfo_xl.py

@@ -20,7 +20,7 @@ import unittest
 from transformers import TransfoXLConfig, is_torch_available
 from transformers.testing_utils import require_torch, require_torch_multi_gpu, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 

tests/models/trocr/test_modeling_trocr.py

@@ -19,7 +19,7 @@ import unittest
 from transformers import TrOCRConfig
 from transformers.testing_utils import is_torch_available, require_torch, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 

tests/models/vision_encoder_decoder/test_modeling_flax_vision_encoder_decoder.py

@@ -215,7 +215,7 @@ class FlaxEncoderDecoderMixin:
         eos_token_id = enc_dec_model.config.decoder.eos_token_id
         decoder_start_token_id = enc_dec_model.config.decoder.decoder_start_token_id
 
-        # Copied from generation_utils (GPT2 doesn't have `pad_token_id`)
+        # Copied from generation.utils (GPT2 doesn't have `pad_token_id`)
         if pad_token_id is None and eos_token_id is not None:
             pad_token_id = eos_token_id
         if decoder_start_token_id is None:

tests/models/whisper/test_modeling_whisper.py

@@ -25,7 +25,7 @@ from transformers.testing_utils import is_torch_available, require_torch, requir
 from transformers.utils import cached_property
 from transformers.utils.import_utils import is_datasets_available
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
 

tests/models/xglm/test_modeling_flax_xglm.py

@@ -21,7 +21,7 @@ import transformers
 from transformers import XGLMConfig, XGLMTokenizer, is_flax_available, is_torch_available
 from transformers.testing_utils import is_pt_flax_cross_test, require_flax, require_sentencepiece, slow
 
-from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin
+from ...generation.test_flax_utils import FlaxGenerationTesterMixin
 from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
 
 

tests/models/xglm/test_modeling_xglm.py

@@ -20,7 +20,7 @@ import unittest
 from transformers import XGLMConfig, is_torch_available
 from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
 

tests/models/xlm/test_modeling_xlm.py

@@ -18,7 +18,7 @@ import unittest
 from transformers import XLMConfig, is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
 

tests/models/xlm_roberta_xl/test_modeling_xlm_roberta_xl.py

@@ -19,7 +19,7 @@ import unittest
 from transformers import XLMRobertaXLConfig, is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
 

tests/models/xlnet/test_modeling_xlnet.py

@@ -19,7 +19,7 @@ import unittest
 from transformers import XLNetConfig, is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
 

tests/test_modeling_tf_common.py

@@ -85,7 +85,7 @@ if is_tf_available():
         TFRagModel,
         TFSharedEmbeddings,
     )
-    from transformers.generation_tf_utils import (
+    from transformers.generation import (
         TFBeamSampleDecoderOnlyOutput,
         TFBeamSampleEncoderDecoderOutput,
         TFBeamSearchDecoderOnlyOutput,

utils/documentation_tests.txt

@@ -11,7 +11,8 @@ docs/source/en/model_doc/byt5.mdx
 docs/source/en/model_doc/tapex.mdx
 docs/source/en/model_doc/donut.mdx
 docs/source/en/model_doc/encoder-decoder.mdx
-src/transformers/generation_utils.py
+src/transformers/generation/utils.py
+src/transformers/generation/tf_utils.py
 src/transformers/models/albert/configuration_albert.py
 src/transformers/models/albert/modeling_albert.py
 src/transformers/models/albert/modeling_tf_albert.py
@@ -108,7 +109,7 @@ src/transformers/models/mobilebert/modeling_tf_mobilebert.py
 src/transformers/models/mobilevit/modeling_mobilevit.py
 src/transformers/models/mobilevit/modeling_tf_mobilevit.py
 src/transformers/models/nezha/configuration_nezha.py
-src/transformers/models/openai/configuration_openai.py 
+src/transformers/models/openai/configuration_openai.py
 src/transformers/models/opt/configuration_opt.py
 src/transformers/models/opt/modeling_opt.py
 src/transformers/models/opt/modeling_tf_opt.py