Merge pull request #2268 from aaugustin/improve-repository-structure

Improve repository structure

.circleci/config.yml

@@ -15,7 +15,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov pytest-xdist
             - run: sudo pip install tensorboardX scikit-learn
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov
+            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_py3_torch:
         working_directory: ~/transformers
@@ -31,7 +31,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov pytest-xdist
             - run: sudo pip install tensorboardX scikit-learn
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov
+            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_py3_tf:
         working_directory: ~/transformers
@@ -47,7 +47,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest codecov pytest-cov pytest-xdist
             - run: sudo pip install tensorboardX scikit-learn
-            - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov
+            - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_py3_custom_tokenizers:
         working_directory: ~/transformers
@@ -58,7 +58,7 @@ jobs:
             - run: sudo pip install --progress-bar off .
             - run: sudo pip install pytest pytest-xdist
             - run: sudo pip install mecab-python3
-            - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py
+            - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./tests/test_tokenization_bert_japanese.py
     run_examples_py3_torch:
         working_directory: ~/transformers
         docker:
@@ -97,9 +97,9 @@ jobs:
             - run: sudo pip install --editable .
             - run: sudo pip install torch tensorflow
             - run: sudo pip install black git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort flake8
-            - run: black --check --line-length 119 examples templates transformers utils
-            - run: isort --check-only --recursive examples templates transformers utils
-            - run: flake8 examples templates transformers utils
+            - run: black --check --line-length 119 examples templates tests src utils
+            - run: isort --check-only --recursive examples templates tests src utils
+            - run: flake8 examples templates tests src utils
     check_repository_consistency:
         working_directory: ~/transformers
         docker:

Makefile

 .PHONY: style
 
 style:
-	black --line-length 119 examples templates transformers utils
-	isort --recursive examples templates transformers utils
+	black --line-length 119 examples templates tests src utils
+	isort --recursive examples templates tests src utils

README.md

@@ -101,7 +101,7 @@ pip install [--editable] .
 
 ### Tests
 
-A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
+A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
 
 These tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`).
 
@@ -110,14 +110,14 @@ Depending on which framework is installed (TensorFlow 2.0 and/or PyTorch), the i
 You can run the tests from the root of the cloned repository with the commands:
 
 ```bash
-python -m unittest discover -s transformers/tests -p "*test.py" -t .
-python -m unittest discover -s examples -p "*test.py" -t examples
+python -m unittest -v discover -s tests -t . -v
+python -m unittest -v discover -s examples -t examples -v
 ```
 
 or
 
 ```bash
-python -m pytest -sv ./transformers/tests/
+python -m pytest -sv ./tests/
 python -m pytest -sv ./examples/
 ```
 

docs/source/installation.md

@@ -22,21 +22,21 @@ pip install [--editable] .
 
 ## Tests
 
-An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
+An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples).
 
 Tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`).
 
 Run all the tests from the root of the cloned repository with the commands:
 
 ```bash
-python -m unittest discover -s transformers/tests -p "*test.py" -t .
-python -m unittest discover -s examples -p "*test.py" -t examples
+python -m unittest discover -s tests -t . -v
+python -m unittest discover -s examples -t examples -v
 ```
 
 or
 
 ``` bash
-python -m pytest -sv ./transformers/tests/
+python -m pytest -sv ./tests/
 python -m pytest -sv ./examples/
 ```
 

examples/summarization/utils_summarization_test.py → examples/summarization/test_utils_summarization.py

@@ -98,7 +98,3 @@ class SummarizationDataProcessingTest(unittest.TestCase):
 
         result = compute_token_type_ids(batch, separator)
         np.testing.assert_array_equal(result, expected)
-
-
-if __name__ == "__main__":
-    unittest.main()

examples/test_examples.py

@@ -104,7 +104,3 @@ class ExamplesTests(unittest.TestCase):
         with patch.object(sys, "argv", testargs + [model_type, model_name]):
             result = run_generation.main()
             self.assertGreaterEqual(len(result), 10)
-
-
-if __name__ == "__main__":
-    unittest.main()

setup.py

@@ -56,7 +56,8 @@ setup(
     keywords="NLP deep learning transformer pytorch tensorflow BERT GPT GPT-2 google openai CMU",
     license="Apache",
     url="https://github.com/huggingface/transformers",
-    packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]),
+    package_dir = {'': 'src'},
+    packages=find_packages("src"),
     install_requires=[
         "numpy",
         "boto3",