Merge pull request #2268 from aaugustin/improve-repository-structure

Improve repository structure

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov pytest-xdist
             - run: sudo pip install tensorboardX scikit-learn
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov
+            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_py3_torch:
         working_directory: ~/transformers
@@ -31,7 +31,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov pytest-xdist
             - run: sudo pip install tensorboardX scikit-learn
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov
+            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_py3_tf:
         working_directory: ~/transformers
@@ -47,7 +47,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov pytest-xdist
             - run: sudo pip install tensorboardX scikit-learn
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov
+            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_py3_custom_tokenizers:
         working_directory: ~/transformers
@@ -58,7 +58,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest pytest-xdist
             - run: sudo pip install mecab-python3
-            - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py
+            - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./tests/test_tokenization_bert_japanese.py
     run_examples_py3_torch:
         working_directory: ~/transformers
         docker:
@@ -97,9 +97,9 @@ jobs:
             - run: sudo pip install --editable .
             - run: sudo pip install torch tensorflow
             - run: sudo pip install black git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort flake8
-            - run: black --check --line-length 119 examples templates transformers utils
-            - run: isort --check-only --recursive examples templates transformers utils
-            - run: flake8 examples templates transformers utils
+            - run: black --check --line-length 119 examples templates tests src utils
+            - run: isort --check-only --recursive examples templates tests src utils
+            - run: flake8 examples templates tests src utils
     check_repository_consistency:
         working_directory: ~/transformers
         docker:

Makefile

 .PHONY: style
 
 style:
-	black --line-length 119 examples templates transformers utils
-	isort --recursive examples templates transformers utils
+	black --line-length 119 examples templates tests src utils
+	isort --recursive examples templates tests src utils

README.md

@@ -101,7 +101,7 @@ pip install [--editable] .
 
 ### Tests
 
-A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
+A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
 
 These tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`).
 
@@ -110,14 +110,14 @@ Depending on which framework is installed (TensorFlow 2.0 and/or PyTorch), the i
 You can run the tests from the root of the cloned repository with the commands:
 
 ```bash
-python -m unittest discover -s transformers/tests -p "*test.py" -t .
-python -m unittest discover -s examples -p "*test.py" -t examples
+python -m unittest -v discover -s tests -t . -v
+python -m unittest -v discover -s examples -t examples -v
 ```
 
 or
 
 ```bash
-python -m pytest -sv ./transformers/tests/
+python -m pytest -sv ./tests/
 python -m pytest -sv ./examples/
 ```
 
@@ -251,7 +251,7 @@ valid_dataset = glue_convert_examples_to_features(data['validation'], tokenizer,
 train_dataset = train_dataset.shuffle(100).batch(32).repeat(2)
 valid_dataset = valid_dataset.batch(64)
 
-# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule 
+# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
 loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')
@@ -281,7 +281,7 @@ print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sen
 
 ## Quick tour of the fine-tuning/usage scripts
 
-**Important**  
+**Important**
 Before running the fine-tuning scripts, please read the
 [instructions](#run-the-examples) on how to
 setup your environment to run the examples.
@@ -442,7 +442,7 @@ python ./examples/run_generation.py \
     --model_name_or_path=gpt2 \
 ```
 
-and from the Salesforce CTRL model: 
+and from the Salesforce CTRL model:
 ```shell
 python ./examples/run_generation.py \
     --model_type=ctrl \
@@ -495,13 +495,13 @@ transformers-cli ls
 ## Quick tour of pipelines
 
 New in version `v2.3`: `Pipeline` are high-level objects which automatically handle tokenization, running your data through a transformers model
-and outputting the result in a structured object. 
+and outputting the result in a structured object.
 
 You can create `Pipeline` objects for the following down-stream tasks:
 
  - `feature-extraction`: Generates a tensor representation for the input sequence
  - `ner`: Generates named entity mapping for each word in the input sequence.
- - `sentiment-analysis`: Gives the polarity (positive / negative) of the whole input sequence. 
+ - `sentiment-analysis`: Gives the polarity (positive / negative) of the whole input sequence.
  - `question-answering`: Provided some context and a question refering to the context, it will extract the answer to the question
  in the context.
 
@@ -516,7 +516,7 @@ nlp('We are very happy to include pipeline into the transformers repository.')
 # Allocate a pipeline for question-answering
 nlp = pipeline('question-answering')
 nlp({
-    'question': 'What is the name of the repository ?', 
+    'question': 'What is the name of the repository ?',
     'context': 'Pipeline have been included in the huggingface/transformers repository'
 })
 >>> {'score': 0.28756016668193496, 'start': 35, 'end': 59, 'answer': 'huggingface/transformers'}

docs/source/installation.md

@@ -22,21 +22,21 @@ pip install [--editable] .
 
 ## Tests
 
-An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
+An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
 
 Tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`).
 
 Run all the tests from the root of the cloned repository with the commands:
 
 ```bash
-python -m unittest discover -s transformers/tests -p "*test.py" -t .
-python -m unittest discover -s examples -p "*test.py" -t examples
+python -m unittest discover -s tests -t . -v
+python -m unittest discover -s examples -t examples -v
 ```
 
 or
 
 ``` bash
-python -m pytest -sv ./transformers/tests/
+python -m pytest -sv ./tests/
 python -m pytest -sv ./examples/
 ```
 

examples/summarization/utils_summarization_test.py → examples/summarization/test_utils_summarization.py

@@ -98,7 +98,3 @@ class SummarizationDataProcessingTest(unittest.TestCase):
 
         result = compute_token_type_ids(batch, separator)
         np.testing.assert_array_equal(result, expected)
-
-
-if __name__ == "__main__":
-    unittest.main()

examples/test_examples.py

@@ -104,7 +104,3 @@ class ExamplesTests(unittest.TestCase):
         with patch.object(sys, "argv", testargs + [model_type, model_name]):
             result = run_generation.main()
             self.assertGreaterEqual(len(result), 10)
-
-
-if __name__ == "__main__":
-    unittest.main()

setup.py

@@ -56,7 +56,8 @@ setup(
     keywords="NLP deep learning transformer pytorch tensorflow BERT GPT GPT-2 google openai CMU",
     license="Apache",
     url="https://github.com/huggingface/transformers",
-    packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]),
+    package_dir = {'': 'src'},
+    packages=find_packages("src"),
     install_requires=[
         "numpy",
         "boto3",