[Test refactor 1/5] Per-folder tests reorganization (#15725)

* Per-folder tests reorganization
Co-authored-by: sgugger <sylvain.gugger@gmail.com>
Co-authored-by: Stas Bekman <stas@stason.org>

tests/test_modeling_m2m_100.py → tests/m2m_100/test_modeling_m2m_100.py

@@ -23,9 +23,9 @@ from transformers import M2M100Config, is_torch_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
-from .test_modeling_common import ModelTesterMixin, ids_tensor
+from ..generation.test_generation_utils import GenerationTesterMixin
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, ids_tensor
 
 
 if is_torch_available():

tests/test_tokenization_m2m_100.py → tests/m2m_100/test_tokenization_m2m_100.py

@@ -15,6 +15,7 @@
 import os
 import tempfile
 import unittest
+from os.path import dirname
 from pathlib import Path
 from shutil import copyfile
 
@@ -26,11 +27,11 @@ from transformers.testing_utils import nested_simplify, require_sentencepiece, r
 if is_sentencepiece_available():
     from transformers.models.m2m_100.tokenization_m2m_100 import save_json, VOCAB_FILES_NAMES
 
-from .test_tokenization_common import TokenizerTesterMixin
+from ..test_tokenization_common import TokenizerTesterMixin
 
 
 if is_sentencepiece_available():
-    SAMPLE_SP = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
+    SAMPLE_SP = os.path.join(dirname(dirname(os.path.abspath(__file__))), "fixtures/test_sentencepiece.model")
 
 
 if is_torch_available():

tests/test_modeling_flax_marian.py → tests/marian/test_modeling_flax_marian.py

@@ -21,8 +21,8 @@ from transformers import MarianConfig, is_flax_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow
 
-from .test_generation_flax_utils import FlaxGenerationTesterMixin
-from .test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
+from ..generation.test_generation_flax_utils import FlaxGenerationTesterMixin
+from ..test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
 
 
 if is_flax_available():

tests/test_modeling_marian.py → tests/marian/test_modeling_marian.py

@@ -22,9 +22,9 @@ from transformers import MarianConfig, is_torch_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
-from .test_modeling_common import ModelTesterMixin, ids_tensor
+from ..generation.test_generation_utils import GenerationTesterMixin
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, ids_tensor
 
 
 if is_torch_available():

tests/test_modeling_tf_marian.py → tests/marian/test_modeling_tf_marian.py

@@ -22,8 +22,8 @@ from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, Translati
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 
 
 if is_tf_available():

tests/test_tokenization_marian.py → tests/marian/test_tokenization_marian.py

@@ -16,6 +16,7 @@
 import os
 import tempfile
 import unittest
+from os.path import dirname
 from pathlib import Path
 from shutil import copyfile
 
@@ -27,10 +28,10 @@ from transformers.testing_utils import require_sentencepiece, slow
 if is_sentencepiece_available():
     from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json
 
-from .test_tokenization_common import TokenizerTesterMixin
+from ..test_tokenization_common import TokenizerTesterMixin
 
 
-SAMPLE_SP = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
+SAMPLE_SP = os.path.join(dirname(dirname(os.path.abspath(__file__))), "fixtures/test_sentencepiece.model")
 
 mock_tokenizer_config = {"target_lang": "fi", "source_lang": "en"}
 zh_code = ">>zh<<"

tests/test_modeling_flax_mbart.py → tests/mbart/test_modeling_flax_mbart.py

@@ -21,8 +21,8 @@ from transformers import MBartConfig, is_flax_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, slow
 
-from .test_generation_flax_utils import FlaxGenerationTesterMixin
-from .test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
+from ..generation.test_generation_flax_utils import FlaxGenerationTesterMixin
+from ..test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor
 
 
 if is_flax_available():

tests/test_modeling_mbart.py → tests/mbart/test_modeling_mbart.py

@@ -23,9 +23,9 @@ from transformers import MBartConfig, is_torch_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_generation_utils import GenerationTesterMixin
-from .test_modeling_common import ModelTesterMixin, ids_tensor
+from ..generation.test_generation_utils import GenerationTesterMixin
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, ids_tensor
 
 
 if is_torch_available():

tests/test_modeling_tf_mbart.py → tests/mbart/test_modeling_tf_mbart.py

@@ -20,8 +20,8 @@ from transformers import AutoTokenizer, MBartConfig, is_tf_available
 from transformers.file_utils import cached_property
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 
 
 if is_tf_available():

tests/test_tokenization_mbart.py → tests/mbart/test_tokenization_mbart.py

@@ -20,10 +20,10 @@ import unittest
 from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
 from transformers.testing_utils import nested_simplify, require_sentencepiece, require_tokenizers, require_torch
 
-from .test_tokenization_common import TokenizerTesterMixin
+from ..test_tokenization_common import TokenizerTesterMixin
 
 
-SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
+SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../fixtures/test_sentencepiece.model")
 
 
 if is_torch_available():

tests/test_tokenization_mbart50.py → tests/mbart50/test_tokenization_mbart50.py

@@ -16,15 +16,15 @@ import os
 import shutil
 import tempfile
 import unittest
+from os.path import dirname
 
 from transformers import SPIECE_UNDERLINE, BatchEncoding, MBart50Tokenizer, MBart50TokenizerFast, is_torch_available
 from transformers.testing_utils import nested_simplify, require_sentencepiece, require_tokenizers, require_torch, slow
 
-from .test_tokenization_common import TokenizerTesterMixin
+from ..test_tokenization_common import TokenizerTesterMixin
 
 
-SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/test_sentencepiece.model")
-
+SAMPLE_VOCAB = os.path.join(dirname(dirname(os.path.abspath(__file__))), "fixtures/test_sentencepiece.model")
 
 if is_torch_available():
     from transformers.models.mbart.modeling_mbart import shift_tokens_right

tests/test_modeling_megatron_bert.py → tests/megatron_bert/test_modeling_megatron_bert.py

@@ -23,8 +23,8 @@ from transformers import MegatronBertConfig, is_torch_available
 from transformers.models.auto import get_values
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
 
 
 if is_torch_available():