Include all optional dependencies in extras.

Take advantage of this to simplify the Circle CI configuration.

Don't bother with tensorboardX: it's a fallback for PyTorch < 1.1.0.

.circleci/config.yml

@@ -10,11 +10,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install torch
-            - run: sudo pip install tensorflow
-            - run: sudo pip install --progress-bar off .
-            - run: sudo pip install pytest codecov pytest-cov pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
+            - run: sudo pip install .[sklearn,tf,torch,testing]
+            - run: sudo pip install codecov pytest-cov
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_torch:
@@ -27,10 +24,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install torch
-            - run: sudo pip install --progress-bar off .
-            - run: sudo pip install pytest codecov pytest-cov pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
+            - run: sudo pip install .[sklearn,torch,testing]
+            - run: sudo pip install codecov pytest-cov
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_tf:
@@ -43,10 +38,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install tensorflow
-            - run: sudo pip install --progress-bar off .
-            - run: sudo pip install pytest codecov pytest-cov pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
+            - run: sudo pip install .[sklearn,tf,testing]
+            - run: sudo pip install codecov pytest-cov
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./tests/ --cov
             - run: codecov
     run_tests_custom_tokenizers:
@@ -55,9 +48,7 @@ jobs:
             - image: circleci/python:3.5
         steps:
             - checkout
-            - run: sudo pip install --progress-bar off .
-            - run: sudo pip install pytest pytest-xdist
-            - run: sudo pip install mecab-python3
+            - run: sudo pip install .[mecab,testing]
             - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./tests/test_tokenization_bert_japanese.py
     run_examples_torch:
         working_directory: ~/transformers
@@ -69,10 +60,8 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install torch
-            - run: sudo pip install --progress-bar off .
-            - run: sudo pip install pytest pytest-xdist
-            - run: sudo pip install tensorboardX scikit-learn
+            - run: sudo pip install .[sklearn,torch,testing]
+            - run: sudo pip install -r examples/requirements.txt
             - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/
     deploy_doc:
         working_directory: ~/transformers
@@ -83,8 +72,7 @@ jobs:
                   fingerprints:
                       - "5b:7a:95:18:07:8c:aa:76:4c:60:35:88:ad:60:56:71"
             - checkout
-            - run: sudo pip install --progress-bar off -r docs/requirements.txt
-            - run: sudo pip install --progress-bar off .
+            - run: sudo pip install .[tf,torch,docs]
             - run: ./.circleci/deploy.sh
     check_code_quality:
         working_directory: ~/transformers
@@ -94,9 +82,9 @@ jobs:
         parallelism: 1
         steps:
             - checkout
-            - run: sudo pip install --editable .
-            - run: sudo pip install torch tensorflow
-            - run: sudo pip install black git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort flake8
+            # we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
+            - run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
+            - run: sudo pip install .[tf,torch,quality]
             - run: black --check --line-length 119 examples templates tests src utils
             - run: isort --check-only --recursive examples templates tests src utils
             - run: flake8 examples templates tests src utils

CONTRIBUTING.md

@@ -102,7 +102,7 @@ Follow these steps to start contributing:
    clicking on the 'Fork' button on the repository's page. This creates a copy of the code
    under your github user account.
 2. Clone your fork to your local disk, and add the base repository as a remote:
-   
+
    ```bash
    $ git clone git@github.com:<your Github handle>/transformers.git
    $ cd transformers
@@ -114,43 +114,43 @@ Follow these steps to start contributing:
    ```bash
    $ git checkout -b a-descriptive-name-for-my-changes
    ```
-   
+
    **do not** work on the `master` branch.
-   
+
 4. Set up a development environment by running the following command in a virtual environment:
 
    ```bash
-   $ pip install -r requirements-dev.txt
+   $ pip install -e .[dev]
    ```
 
 5. Develop the features on your branch. Add changed files using `git add` and
    then `git commit` to record your changes locally:
-   
+
    ```bash
    $ git add modified_file.py
    $ git commit
    ```
-   
+
    Please write [good commit
    messages](https://chris.beams.io/posts/git-commit/). It
    is a good idea to sync your copy of the code with the original repository
    regularly. This way you can quickly account for changes:
-   
+
    ```bash
    $ git fetch upstream
    $ git rebase upstream/master
    ```
-   
+
    Push the changes to your account using:
-   
+
    ```bash
    $ git push -u origin a-descriptive-name-for-my-changes
    ```
-   
+
 6. Once you are satisfied (**and the checklist below is happy too**), go to the
    webpage of your fork on Github. Click on 'Pull request' to send your changes
    to the project maintainers for review.
-   
+
 7. It's ok if maintainers ask you for changes. It happens to core contributors
    too! So everyone can see the changes in the Pull request, work in your local
    branch and push the changes to your fork. They will automatically appear in

setup.cfg

@@ -7,6 +7,7 @@ known_third_party =
     fairseq
     fastprogress
     git
+    MeCab
     nltk
     packaging
     PIL

setup.py

@@ -37,12 +37,20 @@ To create the package for pypi.
 from setuptools import find_packages, setup
 
 
-extras = {
-    "serving": ["pydantic", "uvicorn", "fastapi"],
-    "serving-tf": ["pydantic", "uvicorn", "fastapi", "tensorflow"],
-    "serving-torch": ["pydantic", "uvicorn", "fastapi", "torch"],
-}
-extras["all"] = [package for package in extras.values()]
+extras = {}
+
+extras["mecab"] = ["mecab-python3"]
+extras["sklearn"] = ["scikit-learn"]
+extras["tf"] = ["tensorflow"]
+extras["torch"] = ["torch"]
+
+extras["serving"] = ["pydantic", "uvicorn", "fastapi"]
+extras["all"] = extras["serving"] + ["tensorflow", "torch"]
+
+extras["testing"] = ["pytest", "pytest-xdist"]
+extras["quality"] = ["black", "isort", "flake8"]
+extras["docs"] = ["recommonmark", "sphinx", "sphinx-markdown-tables", "sphinx-rtd-theme"]
+extras["dev"] = extras["testing"] + extras["quality"] + ["mecab-python3", "scikit-learn", "tensorflow", "torch"]
 
 setup(
     name="transformers",