Generate: move generation_*.py src files into generation/*.py (#20096)

* move generation_*.py src files into generation/*.py

* populate generation.__init__ with lazy loading

* move imports and references from generation.xxx.object to generation.object

src/transformers/utils/dummy_tf_objects.py

@@ -31,6 +31,13 @@ class TFForcedEOSTokenLogitsProcessor(metaclass=DummyObject):
         requires_backends(self, ["tf"])
 
 
+class TFGenerationMixin(metaclass=DummyObject):
+    _backends = ["tf"]
+
+    def __init__(self, *args, **kwargs):
+        requires_backends(self, ["tf"])
+
+
 class TFLogitsProcessor(metaclass=DummyObject):
     _backends = ["tf"]
 

templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/test_modeling_{{cookiecutter.lowercase_modelname}}.py

@@ -490,7 +490,7 @@ from transformers.utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
 from ...test_configuration_common import ConfigTester
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 
 

tests/generation/test_generation_beam_constraints.py → tests/generation/test_beam_constraints.py

@@ -23,7 +23,7 @@ from transformers.testing_utils import require_torch
 if is_torch_available():
     import torch
 
-    from transformers.generation_beam_constraints import DisjunctiveConstraint
+    from transformers.generation import DisjunctiveConstraint
 
 
 @require_torch

tests/generation/test_generation_beam_search.py → tests/generation/test_beam_search.py

@@ -25,8 +25,13 @@ from ..test_modeling_common import floats_tensor, ids_tensor
 if is_torch_available():
     import torch
 
-    from transformers.generation_beam_constraints import DisjunctiveConstraint, PhrasalConstraint
-    from transformers.generation_beam_search import BeamHypotheses, BeamSearchScorer, ConstrainedBeamSearchScorer
+    from transformers.generation import (
+        BeamHypotheses,
+        BeamSearchScorer,
+        ConstrainedBeamSearchScorer,
+        DisjunctiveConstraint,
+        PhrasalConstraint,
+    )
 
 
 class BeamSearchTester:

tests/generation/test_generation_flax_logits_process.py → tests/generation/test_flax_logits_process.py

@@ -27,7 +27,7 @@ from ..test_modeling_flax_common import ids_tensor
 if is_flax_available():
     import jax
     import jax.numpy as jnp
-    from transformers.generation_flax_logits_process import (
+    from transformers.generation import (
         FlaxForcedBOSTokenLogitsProcessor,
         FlaxForcedEOSTokenLogitsProcessor,
         FlaxLogitsProcessorList,

tests/generation/test_generation_logits_process.py → tests/generation/test_logits_process.py

@@ -26,7 +26,7 @@ if is_torch_available():
     import torch
     from torch import nn
 
-    from transformers.generation_logits_process import (
+    from transformers.generation import (
         EncoderNoRepeatNGramLogitsProcessor,
         ExponentialDecayLengthPenalty,
         ForcedBOSTokenLogitsProcessor,

tests/generation/test_generation_stopping_criteria.py → tests/generation/test_stopping_criteria.py

@@ -25,7 +25,7 @@ from ..test_modeling_common import ids_tensor
 if is_torch_available():
     import torch
 
-    from transformers.generation_stopping_criteria import (
+    from transformers.generation import (
         MaxLengthCriteria,
         MaxNewTokensCriteria,
         MaxTimeCriteria,

tests/generation/test_generation_tf_logits_process.py → tests/generation/test_tf_logits_process.py

@@ -26,7 +26,7 @@ from transformers.testing_utils import require_tf
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.generation_tf_logits_process import (
+    from transformers.generation import (
         TFForcedBOSTokenLogitsProcessor,
         TFForcedEOSTokenLogitsProcessor,
         TFForceTokensLogitsProcessor,

tests/generation/test_generation_utils.py → tests/generation/test_utils.py

@@ -42,33 +42,35 @@ if is_torch_available():
         pipeline,
         top_k_top_p_filtering,
     )
-    from transformers.generation_beam_constraints import DisjunctiveConstraint, PhrasalConstraint
-    from transformers.generation_beam_search import BeamSearchScorer, ConstrainedBeamSearchScorer
-    from transformers.generation_logits_process import (
+    from transformers.generation import (
+        BeamSampleDecoderOnlyOutput,
+        BeamSampleEncoderDecoderOutput,
+        BeamSearchDecoderOnlyOutput,
+        BeamSearchEncoderDecoderOutput,
+        BeamSearchScorer,
+        ConstrainedBeamSearchScorer,
+        DisjunctiveConstraint,
         ForcedBOSTokenLogitsProcessor,
         ForcedEOSTokenLogitsProcessor,
+        GreedySearchDecoderOnlyOutput,
+        GreedySearchEncoderDecoderOutput,
         HammingDiversityLogitsProcessor,
         InfNanRemoveLogitsProcessor,
         LogitsProcessorList,
+        MaxLengthCriteria,
         MinLengthLogitsProcessor,
         NoBadWordsLogitsProcessor,
         NoRepeatNGramLogitsProcessor,
+        PhrasalConstraint,
         RepetitionPenaltyLogitsProcessor,
+        SampleDecoderOnlyOutput,
+        SampleEncoderDecoderOutput,
+        StoppingCriteria,
+        StoppingCriteriaList,
         TemperatureLogitsWarper,
         TopKLogitsWarper,
         TopPLogitsWarper,
     )
-    from transformers.generation_stopping_criteria import MaxLengthCriteria, StoppingCriteria, StoppingCriteriaList
-    from transformers.generation_utils import (
-        BeamSampleDecoderOnlyOutput,
-        BeamSampleEncoderDecoderOutput,
-        BeamSearchDecoderOnlyOutput,
-        BeamSearchEncoderDecoderOutput,
-        GreedySearchDecoderOnlyOutput,
-        GreedySearchEncoderDecoderOutput,
-        SampleDecoderOnlyOutput,
-        SampleEncoderDecoderOutput,
-    )
 
 
 class GenerationTesterMixin:

tests/models/bart/test_modeling_bart.py

@@ -25,7 +25,7 @@ from transformers import BartConfig, is_torch_available
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 from transformers.utils import cached_property
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
 

tests/models/bart/test_modeling_flax_bart.py

@@ -19,7 +19,7 @@ import timeout_decorator  # noqa
 from transformers import BartConfig, BartTokenizer, is_flax_available
 from transformers.testing_utils import require_flax, slow
 
-from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin
+from ...generation.test_flax_utils import FlaxGenerationTesterMixin
 from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
 
 

tests/models/bert/test_modeling_bert.py

@@ -20,7 +20,7 @@ from transformers import BertConfig, is_torch_available
 from transformers.models.auto import get_values
 from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
 

tests/models/bert_generation/test_modeling_bert_generation.py

@@ -19,7 +19,7 @@ import unittest
 from transformers import BertGenerationConfig, is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
 

tests/models/bigbird_pegasus/test_modeling_bigbird_pegasus.py

@@ -22,7 +22,7 @@ import unittest
 from transformers import BigBirdPegasusConfig, is_torch_available
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 

tests/models/blenderbot/test_modeling_blenderbot.py

@@ -21,7 +21,7 @@ from transformers import BlenderbotConfig, is_torch_available
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 from transformers.utils import cached_property
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 

tests/models/blenderbot/test_modeling_flax_blenderbot.py

@@ -20,7 +20,7 @@ import timeout_decorator  # noqa
 from transformers import BlenderbotConfig, is_flax_available
 from transformers.testing_utils import jax_device, require_flax, slow
 
-from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin
+from ...generation.test_flax_utils import FlaxGenerationTesterMixin
 from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
 
 

tests/models/blenderbot_small/test_modeling_blenderbot_small.py

@@ -21,7 +21,7 @@ from transformers import BlenderbotSmallConfig, is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 from transformers.utils import cached_property
 
-from ...generation.test_generation_utils import GenerationTesterMixin
+from ...generation.test_utils import GenerationTesterMixin
 from ...test_configuration_common import ConfigTester
 from ...test_modeling_common import ModelTesterMixin, ids_tensor
 

tests/models/blenderbot_small/test_modeling_flax_blenderbot_small.py

@@ -20,7 +20,7 @@ import timeout_decorator  # noqa
 from transformers import BlenderbotSmallConfig, is_flax_available
 from transformers.testing_utils import require_flax, slow
 
-from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin
+from ...generation.test_flax_utils import FlaxGenerationTesterMixin
 from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor