[Test refactor 1/5] Per-folder tests reorganization (#15725)

* Per-folder tests reorganization
Co-authored-by: sgugger <sylvain.gugger@gmail.com>
Co-authored-by: Stas Bekman <stas@stason.org>

tests/test_tokenization_distilbert.py → tests/distilbert/test_tokenization_distilbert.py

@@ -17,7 +17,7 @@
 from transformers import DistilBertTokenizer, DistilBertTokenizerFast
 from transformers.testing_utils import require_tokenizers, slow
 
-from .test_tokenization_bert import BertTokenizationTest
+from ..bert.test_tokenization_bert import BertTokenizationTest
 
 
 @require_tokenizers

tests/test_modeling_dpr.py → tests/dpr/test_modeling_dpr.py

@@ -20,8 +20,8 @@ import unittest
 from transformers import DPRConfig, is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
 
 
 if is_torch_available():

tests/test_modeling_tf_dpr.py → tests/dpr/test_modeling_tf_dpr.py

@@ -18,8 +18,8 @@ import unittest
 from transformers import is_tf_available
 from transformers.testing_utils import require_tf, slow
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 
 
 if is_tf_available():

tests/test_tokenization_dpr.py → tests/dpr/test_tokenization_dpr.py

@@ -26,7 +26,7 @@ from transformers import (
 from transformers.testing_utils import require_tokenizers, slow
 from transformers.tokenization_utils_base import BatchEncoding
 
-from .test_tokenization_bert import BertTokenizationTest
+from ..bert.test_tokenization_bert import BertTokenizationTest
 
 
 @require_tokenizers

tests/test_modeling_electra.py → tests/electra/test_modeling_electra.py

@@ -20,8 +20,8 @@ from transformers import ElectraConfig, is_torch_available
 from transformers.models.auto import get_values
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
 
 
 if is_torch_available():

tests/test_modeling_flax_electra.py → tests/electra/test_modeling_flax_electra.py

@@ -5,7 +5,7 @@ import numpy as np
 from transformers import ElectraConfig, is_flax_available
 from transformers.testing_utils import require_flax, slow
 
-from .test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask
+from ..test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask
 
 
 if is_flax_available():

tests/test_modeling_tf_electra.py → tests/electra/test_modeling_tf_electra.py

@@ -19,8 +19,8 @@ import unittest
 from transformers import ElectraConfig, is_tf_available
 from transformers.testing_utils import require_tf, slow
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 
 
 if is_tf_available():

tests/test_modeling_encoder_decoder.py → tests/encoder_decoder/test_modeling_encoder_decoder.py

@@ -20,13 +20,13 @@ import unittest
 from transformers import is_torch_available
 from transformers.testing_utils import require_torch, slow, torch_device
 
-from .test_modeling_bart import BartStandaloneDecoderModelTester
-from .test_modeling_bert import BertModelTester
-from .test_modeling_bert_generation import BertGenerationEncoderTester
-from .test_modeling_common import ids_tensor
-from .test_modeling_gpt2 import GPT2ModelTester
-from .test_modeling_prophetnet import ProphetNetStandaloneDecoderModelTester
-from .test_modeling_roberta import RobertaModelTester
+from ..bart.test_modeling_bart import BartStandaloneDecoderModelTester
+from ..bert.test_modeling_bert import BertModelTester
+from ..bert_generation.test_modeling_bert_generation import BertGenerationEncoderTester
+from ..gpt2.test_modeling_gpt2 import GPT2ModelTester
+from ..prophetnet.test_modeling_prophetnet import ProphetNetStandaloneDecoderModelTester
+from ..roberta.test_modeling_roberta import RobertaModelTester
+from ..test_modeling_common import ids_tensor
 
 
 if is_torch_available():
@@ -840,7 +840,7 @@ class GPT2EncoderDecoderModelTest(EncoderDecoderMixin, unittest.TestCase):
         }
 
     def get_pretrained_model(self):
-        return EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-cased", "gpt2")
+        return EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-cased", "../gpt2")
 
     def test_encoder_decoder_model_shared_weights(self):
         pass
@@ -851,7 +851,7 @@ class GPT2EncoderDecoderModelTest(EncoderDecoderMixin, unittest.TestCase):
 
         model.to(torch_device)
         tokenizer_in = AutoTokenizer.from_pretrained("bert-base-cased")
-        tokenizer_out = AutoTokenizer.from_pretrained("gpt2")
+        tokenizer_out = AutoTokenizer.from_pretrained("../gpt2")
 
         ARTICLE_STUDENTS = """(CNN)Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing a racist chant. SAE's national chapter suspended the students, but University of Oklahoma President David Boren took it a step further, saying the university's affiliation with the fraternity is permanently done. The news is shocking, but it's not the first time SAE has faced controversy. SAE was founded March 9, 1856, at the University of Alabama, five years before the American Civil War, according to the fraternity website. When the war began, the group had fewer than 400 members, of which "369 went to war for the Confederate States and seven for the Union Army," the website says. The fraternity now boasts more than 200,000 living alumni, along with about 15,000 undergraduates populating 219 chapters and 20 "colonies" seeking full membership at universities. SAE has had to work hard to change recently after a string of member deaths, many blamed on the hazing of new recruits, SAE national President Bradley Cohen wrote in a message on the fraternity's website. The fraternity's website lists more than 130 chapters cited or suspended for "health and safety incidents" since 2010. At least 30 of the incidents involved hazing, and dozens more involved alcohol. However, the list is missing numerous incidents from recent months. Among them, according to various media outlets: Yale University banned the SAEs from campus activities last month after members allegedly tried to interfere with a sexual misconduct investigation connected to an initiation rite. Stanford University in December suspended SAE housing privileges after finding sorority members attending a fraternity function were subjected to graphic sexual content. And Johns Hopkins University in November suspended the fraternity for underage drinking. "The media has labeled us as the 'nation's deadliest fraternity,' " Cohen said. In 2011, for example, a student died while being coerced into excessive alcohol consumption, according to a lawsuit. SAE's previous insurer dumped the fraternity. "As a result, we are paying Lloyd's of London the highest insurance rates in the Greek-letter world," Cohen said. Universities have turned down SAE's attempts to open new chapters, and the fraternity had to close 12 in 18 months over hazing incidents."""
 

tests/test_modeling_flax_encoder_decoder.py → tests/encoder_decoder/test_modeling_flax_encoder_decoder.py

@@ -22,9 +22,9 @@ import numpy as np
 from transformers import is_flax_available, is_torch_available
 from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow, torch_device
 
-from .test_modeling_flax_bert import FlaxBertModelTester
-from .test_modeling_flax_common import ids_tensor
-from .test_modeling_flax_gpt2 import FlaxGPT2ModelTester
+from ..bert.test_modeling_flax_bert import FlaxBertModelTester
+from ..gpt2.test_modeling_flax_gpt2 import FlaxGPT2ModelTester
+from ..test_modeling_flax_common import ids_tensor
 
 
 if is_flax_available():

tests/test_modeling_tf_encoder_decoder.py → tests/encoder_decoder/test_modeling_tf_encoder_decoder.py

@@ -24,11 +24,11 @@ import numpy as np
 from transformers import is_tf_available, is_torch_available
 from transformers.testing_utils import is_pt_tf_cross_test, require_tf, require_torch, slow, torch_device
 
-from .test_modeling_tf_bert import TFBertModelTester
-from .test_modeling_tf_common import ids_tensor
-from .test_modeling_tf_gpt2 import TFGPT2ModelTester
-from .test_modeling_tf_rembert import TFRemBertModelTester
-from .test_modeling_tf_roberta import TFRobertaModelTester
+from ..bert.test_modeling_tf_bert import TFBertModelTester
+from ..gpt2.test_modeling_tf_gpt2 import TFGPT2ModelTester
+from ..rembert.test_modeling_tf_rembert import TFRemBertModelTester
+from ..roberta.test_modeling_tf_roberta import TFRobertaModelTester
+from ..test_modeling_tf_common import ids_tensor
 
 
 if is_tf_available():
@@ -634,7 +634,7 @@ class TFBertEncoderDecoderModelTest(TFEncoderDecoderMixin, unittest.TestCase):
 @require_tf
 class TFGPT2EncoderDecoderModelTest(TFEncoderDecoderMixin, unittest.TestCase):
     def get_pretrained_model(self):
-        return TFEncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-cased", "gpt2")
+        return TFEncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-cased", "../gpt2")
 
     def get_encoder_decoder_model(self, config, decoder_config):
         encoder_model = TFBertModel(config, name="encoder")
@@ -694,7 +694,7 @@ class TFGPT2EncoderDecoderModelTest(TFEncoderDecoderMixin, unittest.TestCase):
         from transformers import EncoderDecoderModel
 
         tokenizer_in = AutoTokenizer.from_pretrained("bert-base-cased")
-        tokenizer_out = AutoTokenizer.from_pretrained("gpt2")
+        tokenizer_out = AutoTokenizer.from_pretrained("../gpt2")
 
         """Not working, because pt checkpoint has `encoder.encoder.layer...` while tf model has `encoder.bert.encoder.layer...`.
         (For GPT2 decoder, there is no issue)

tests/test_modeling_flaubert.py → tests/flaubert/test_modeling_flaubert.py

@@ -19,8 +19,8 @@ import unittest
 from transformers import FlaubertConfig, is_torch_available
 from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
 
 
 if is_torch_available():

tests/test_modeling_tf_flaubert.py → tests/flaubert/test_modeling_tf_flaubert.py

@@ -18,8 +18,8 @@ import unittest
 from transformers import is_tf_available
 from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_tf_common import TFModelTesterMixin, ids_tensor
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_tf_common import TFModelTesterMixin, ids_tensor
 
 
 if is_tf_available():

tests/test_modeling_fnet.py → tests/fnet/test_modeling_fnet.py

@@ -22,8 +22,8 @@ from transformers import FNetConfig, is_torch_available
 from transformers.models.auto import get_values
 from transformers.testing_utils import require_tokenizers, require_torch, slow, torch_device
 
-from .test_configuration_common import ConfigTester
-from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
+from ..test_configuration_common import ConfigTester
+from ..test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
 
 
 if is_torch_available():

tests/test_tokenization_fnet.py → tests/fnet/test_tokenization_fnet.py

@@ -20,10 +20,10 @@ from transformers import FNetTokenizer, FNetTokenizerFast
 from transformers.testing_utils import require_sentencepiece, require_tokenizers, slow, tooslow
 from transformers.tokenization_utils import AddedToken
 
-from .test_tokenization_common import TokenizerTesterMixin
+from ..test_tokenization_common import TokenizerTesterMixin
 
 
-SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/spiece.model")
+SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../fixtures/spiece.model")
 
 
 @require_sentencepiece