setup.py 12.2 KB
Newer Older
Sylvain Gugger's avatar
Sylvain Gugger committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/master/setup.py

To create the package for pypi.

Sylvain Gugger's avatar
Sylvain Gugger committed
20
21
1. Run `make pre-release` (or `make pre-patch` for a patch release) then run `make fix-copies` to fix the index of the
   documentation.
22

Philipp Schmid's avatar
Philipp Schmid committed
23
2. Run Tests for Amazon Sagemaker. The documentation is located in `./tests/sagemaker/README.md`, otherwise @philschmid.
24

Philipp Schmid's avatar
Philipp Schmid committed
25
3. Unpin specific versions from setup.py that use a git install.
Lysandre's avatar
Lysandre committed
26

Philipp Schmid's avatar
Philipp Schmid committed
27
4. Commit these changes with the message: "Release: VERSION"
28

Philipp Schmid's avatar
Philipp Schmid committed
29
5. Add a tag in git to mark the release: "git tag VERSION -m 'Adds tag VERSION for pypi' "
30
31
   Push the tag to git: git push --tags origin master

Philipp Schmid's avatar
Philipp Schmid committed
32
6. Build both the sources and the wheel. Do not change anything in setup.py between
33
34
   creating the wheel and the source distribution (obviously).

thomwolf's avatar
thomwolf committed
35
   For the wheel, run: "python setup.py bdist_wheel" in the top level directory.
36
   (this will build a wheel for the python version you use to build it).
37
38

   For the sources, run: "python setup.py sdist"
thomwolf's avatar
thomwolf committed
39
   You should now have a /dist directory with both .whl and .tar.gz source versions.
40

Philipp Schmid's avatar
Philipp Schmid committed
41
7. Check that everything looks correct by uploading the package to the pypi test server:
42
43
44

   twine upload dist/* -r pypitest
   (pypi suggest using twine as other methods upload files via plaintext.)
Lysandre's avatar
Lysandre committed
45
46
   You may have to specify the repository url, use the following command then:
   twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/
47
48

   Check that you can install it in a virtualenv by running:
49
   pip install -i https://testpypi.python.org/pypi transformers
50

Philipp Schmid's avatar
Philipp Schmid committed
51
8. Upload the final version to actual pypi:
52
53
   twine upload dist/* -r pypi

Philipp Schmid's avatar
Philipp Schmid committed
54
9. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
55

Philipp Schmid's avatar
Philipp Schmid committed
56
10. Run `make post-release` (or `make post-patch` for a patch release).
57
"""
Aymeric Augustin's avatar
Aymeric Augustin committed
58

Stas Bekman's avatar
Stas Bekman committed
59
import os
60
import re
61
import shutil
62
from distutils.core import Command
63
64
from pathlib import Path

thomwolf's avatar
thomwolf committed
65
66
from setuptools import find_packages, setup

67

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
stale_egg_info = Path(__file__).parent / "transformers.egg-info"
if stale_egg_info.exists():
    print(
        (
            "Warning: {} exists.\n\n"
            "If you recently updated transformers to 3.0 or later, this is expected,\n"
            "but it may prevent transformers from installing in editable mode.\n\n"
            "This directory is automatically generated by Python's packaging tools.\n"
            "I will remove it now.\n\n"
            "See https://github.com/pypa/pip/issues/5466 for details.\n"
        ).format(stale_egg_info)
    )
    shutil.rmtree(stale_egg_info)


84
85
86
87
# IMPORTANT:
# 1. all dependencies should be listed here with their version requirements if any
# 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
_deps = [
88
    "Pillow",
Sylvain Gugger's avatar
Sylvain Gugger committed
89
    "black==21.4b0",
90
    "codecarbon==1.2.0",
91
92
93
    "cookiecutter==1.7.2",
    "dataclasses",
    "datasets",
94
    "deepspeed>=0.4.0",
Lysandre Debut's avatar
Lysandre Debut committed
95
    "docutils==0.16.0",
96
    "fairscale>0.3",
97
98
99
100
    "faiss-cpu",
    "fastapi",
    "filelock",
    "flake8>=3.8.3",
101
    "flax>=0.3.4",
102
    "fugashi>=1.0",
103
    "huggingface-hub==0.0.8",
104
    "importlib_metadata",
105
106
    "ipadic>=1.0.0,<2.0",
    "isort>=5.5.4",
107
    "jax>=0.2.8",
108
    "jaxlib>=0.1.65",
109
    "jieba",
110
    "keras2onnx",
111
    "nltk",
Sylvain Gugger's avatar
Sylvain Gugger committed
112
    "numpy>=1.17",
Lysandre's avatar
Lysandre committed
113
    "onnxconverter-common",
114
115
    "onnxruntime-tools>=1.4.2",
    "onnxruntime>=1.4.0",
Lysandre Debut's avatar
Lysandre Debut committed
116
    "optuna",
117
118
119
120
    "packaging",
    "parameterized",
    "protobuf",
    "psutil",
121
    "pyyaml",
122
123
    "pydantic",
    "pytest",
Lysandre Debut's avatar
Lysandre Debut committed
124
    "pytest-sugar",
125
126
    "pytest-xdist",
    "python>=3.6.0",
Lysandre Debut's avatar
Lysandre Debut committed
127
    "ray",
128
129
130
    "recommonmark",
    "regex!=2019.12.17",
    "requests",
131
132
    "rouge-score",
    "sacrebleu>=1.4.12",
133
    "sacremoses",
134
    "sagemaker>=2.31.0",
135
136
    "scikit-learn",
    "sentencepiece==0.1.91",
Patrick von Platen's avatar
Patrick von Platen committed
137
    "soundfile",
138
139
140
141
    "sphinx-copybutton",
    "sphinx-markdown-tables",
    "sphinx-rtd-theme==0.4.3",  # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
    "sphinx==3.2.1",
142
    "sphinxext-opengraph==0.4.1",
143
    "starlette",
144
145
    "tensorflow-cpu>=2.3",
    "tensorflow>=2.3",
146
    "timeout-decorator",
NielsRogge's avatar
NielsRogge committed
147
    "timm",
148
    "tokenizers>=0.10.1,<0.11",
149
    "torch>=1.0",
Suraj Patil's avatar
Suraj Patil committed
150
    "torchaudio",
151
152
153
154
    "tqdm>=4.27",
    "unidic>=1.0.2",
    "unidic_lite>=1.0.7",
    "uvicorn",
155
]
156
157


158
159
160
161
162
163
# this is a lookup table with items like:
#
# tokenizers: "tokenizers==0.9.4"
# packaging: "packaging"
#
# some of the values are versioned whereas others aren't.
164
165
deps = {b: a for a, b in (re.findall(r"^(([^!=<>]+)(?:[!=<>].*)?$)", x)[0] for x in _deps)}

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
# since we save this data in src/transformers/dependency_versions_table.py it can be easily accessed from
# anywhere. If you need to quickly access the data from this table in a shell, you can do so easily with:
#
# python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets
#
# Just pass the desired package names to that script as it's shown with 2 packages above.
#
# If transformers is not yet installed and the work is done from the cloned repo remember to add `PYTHONPATH=src` to the script above
#
# You can then feed this for example to `pip`:
#
# pip install -U $(python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets)
#

182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213

def deps_list(*pkgs):
    return [deps[pkg] for pkg in pkgs]


class DepsTableUpdateCommand(Command):
    """
    A custom distutils command that updates the dependency table.
    usage: python setup.py deps_table_update
    """

    description = "build runtime dependency table"
    user_options = [
        # format: (long option, short option, description).
        ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
    ]

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
        content = [
            "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
            "# 1. modify the `_deps` dict in setup.py",
            "# 2. run `make deps_table_update``",
            "deps = {",
            entries,
            "}",
214
            "",
215
216
217
        ]
        target = "src/transformers/dependency_versions_table.py"
        print(f"updating {target}")
Julien Plu's avatar
Julien Plu committed
218
        with open(target, "w", encoding="utf-8", newline="\n") as f:
219
220
221
222
223
224
225
226
227
228
229
230
            f.write("\n".join(content))


extras = {}

extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic")
extras["sklearn"] = deps_list("scikit-learn")

extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "keras2onnx")
extras["tf-cpu"] = deps_list("tensorflow-cpu", "onnxconverter-common", "keras2onnx")

extras["torch"] = deps_list("torch")
Stas Bekman's avatar
Stas Bekman committed
231
232

if os.name == "nt":  # windows
233
234
    extras["retrieval"] = deps_list("datasets")  # faiss is not supported on windows
    extras["flax"] = []  # jax is not supported on windows
235
else:
236
237
    extras["retrieval"] = deps_list("faiss-cpu", "datasets")
    extras["flax"] = deps_list("jax", "jaxlib", "flax")
238

239
240
extras["tokenizers"] = deps_list("tokenizers")
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
Lysandre Debut's avatar
Lysandre Debut committed
241
extras["onnx"] = deps_list("onnxconverter-common", "keras2onnx") + extras["onnxruntime"]
242
extras["modelcreation"] = deps_list("cookiecutter")
243

Philipp Schmid's avatar
Philipp Schmid committed
244
extras["sagemaker"] = deps_list("sagemaker")
245
246
extras["deepspeed"] = deps_list("deepspeed")
extras["fairscale"] = deps_list("fairscale")
Lysandre Debut's avatar
Lysandre Debut committed
247
248
249
250
extras["optuna"] = deps_list("optuna")
extras["ray"] = deps_list("ray")

extras["integrations"] = extras["optuna"] + extras["ray"]
Philipp Schmid's avatar
Philipp Schmid committed
251

252
extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
Suraj Patil's avatar
Suraj Patil committed
253
extras["speech"] = deps_list("soundfile", "torchaudio")
254
extras["vision"] = deps_list("Pillow")
NielsRogge's avatar
NielsRogge committed
255
extras["timm"] = deps_list("timm")
256
extras["codecarbon"] = deps_list("codecarbon")
Stas Bekman's avatar
Stas Bekman committed
257

258
259
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
extras["testing"] = (
Philipp Schmid's avatar
Philipp Schmid committed
260
    deps_list(
261
        "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-sugar", "black", "sacrebleu", "rouge-score", "nltk"
Philipp Schmid's avatar
Philipp Schmid committed
262
    )
263
264
265
    + extras["retrieval"]
    + extras["modelcreation"]
)
266

267
extras["quality"] = deps_list("black", "isort", "flake8")
Stas Bekman's avatar
Stas Bekman committed
268

269
270
271
272
273
274
275
276
extras["all"] = (
    extras["tf"]
    + extras["torch"]
    + extras["flax"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["speech"]
    + extras["vision"]
Lysandre Debut's avatar
Lysandre Debut committed
277
    + extras["integrations"]
NielsRogge's avatar
NielsRogge committed
278
    + extras["timm"]
279
    + extras["codecarbon"]
280
)
Stas Bekman's avatar
Stas Bekman committed
281

282
283
284
285
286
287
288
289
290
291
292
293
extras["docs_specific"] = deps_list(
    "docutils",
    "recommonmark",
    "sphinx",
    "sphinx-markdown-tables",
    "sphinx-rtd-theme",
    "sphinx-copybutton",
    "sphinxext-opengraph",
)
# "docs" needs "all" to resolve all the references
extras["docs"] = extras["all"] + extras["docs_specific"]

294
295
296
297
298
extras["dev"] = (
    extras["all"]
    + extras["testing"]
    + extras["quality"]
    + extras["ja"]
299
    + extras["docs_specific"]
300
301
302
    + extras["sklearn"]
    + extras["modelcreation"]
)
Stas Bekman's avatar
Stas Bekman committed
303

304
305
extras["torchhub"] = deps_list(
    "filelock",
Sylvain Gugger's avatar
Sylvain Gugger committed
306
    "huggingface-hub",
307
308
309
310
311
312
313
314
315
316
317
318
    "importlib_metadata",
    "numpy",
    "packaging",
    "protobuf",
    "regex",
    "requests",
    "sacremoses",
    "sentencepiece",
    "torch",
    "tokenizers",
    "tqdm",
)
319

320
321
322
# when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
install_requires = [
    deps["dataclasses"] + ";python_version<'3.7'",  # dataclasses for Python versions that don't have it
323
    deps["importlib_metadata"] + ";python_version<'3.8'",  # importlib_metadata for Python versions that don't have it
324
    deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
325
    deps["huggingface-hub"],
326
327
    deps["numpy"],
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
328
    deps["pyyaml"], # used for the model cards metadata
329
330
331
    deps["regex"],  # for OpenAI GPT
    deps["requests"],  # for downloading models over HTTPS
    deps["sacremoses"],  # for XLM
Sylvain Gugger's avatar
Sylvain Gugger committed
332
    deps["tokenizers"],
333
    deps["tqdm"],  # progress bars in model download and training scripts
334
335
]

thomwolf's avatar
thomwolf committed
336
setup(
337
    name="transformers",
Lysandre's avatar
Lysandre committed
338
    version="4.8.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
339
    author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Suraj Patil, Stas Bekman, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
thomwolf's avatar
thomwolf committed
340
    author_email="thomas@huggingface.co",
thomwolf's avatar
thomwolf committed
341
    description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
342
    long_description=open("README.md", "r", encoding="utf-8").read(),
thomwolf's avatar
thomwolf committed
343
    long_description_content_type="text/markdown",
344
345
    keywords="NLP deep learning transformer pytorch tensorflow BERT GPT GPT-2 google openai CMU",
    license="Apache",
346
    url="https://github.com/huggingface/transformers",
Aymeric Augustin's avatar
Aymeric Augustin committed
347
    package_dir={"": "src"},
348
    packages=find_packages("src"),
349
    extras_require=extras,
350
    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
351
    python_requires=">=3.6.0",
352
    install_requires=install_requires,
thomwolf's avatar
thomwolf committed
353
    classifiers=[
Aymeric Augustin's avatar
Aymeric Augustin committed
354
355
356
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
357
358
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
Aymeric Augustin's avatar
Aymeric Augustin committed
359
        "Operating System :: OS Independent",
360
        "Programming Language :: Python :: 3",
Aymeric Augustin's avatar
Aymeric Augustin committed
361
362
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
363
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
thomwolf's avatar
thomwolf committed
364
    ],
365
    cmdclass={"deps_table_update": DepsTableUpdateCommand},
thomwolf's avatar
thomwolf committed
366
)