setup.py 13.4 KB
Newer Older
1
# Copyright 2021 The HuggingFace Team. All rights reserved.
Sylvain Gugger's avatar
Sylvain Gugger committed
2
3
4
5
6
7
8
9
10
11
12
13
14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/master/setup.py

To create the package for pypi.

Sylvain Gugger's avatar
Sylvain Gugger committed
20
21
1. Run `make pre-release` (or `make pre-patch` for a patch release) then run `make fix-copies` to fix the index of the
   documentation.
22

Philipp Schmid's avatar
Philipp Schmid committed
23
2. Run Tests for Amazon Sagemaker. The documentation is located in `./tests/sagemaker/README.md`, otherwise @philschmid.
24

Philipp Schmid's avatar
Philipp Schmid committed
25
3. Unpin specific versions from setup.py that use a git install.
Lysandre's avatar
Lysandre committed
26

27
4. Commit these changes with the message: "Release: VERSION" and push.
28

29
30
31
5. Wait for the tests on master to be completed and be green (otherwise revert and fix bugs)

6. Add a tag in git to mark the release: "git tag <VERSION> -m 'Adds tag <VERSION> for pypi' "
32
33
   Push the tag to git: git push --tags origin master

34
7. Build both the sources and the wheel. Do not change anything in setup.py between
35
36
   creating the wheel and the source distribution (obviously).

thomwolf's avatar
thomwolf committed
37
   For the wheel, run: "python setup.py bdist_wheel" in the top level directory.
38
   (this will build a wheel for the python version you use to build it).
39
40

   For the sources, run: "python setup.py sdist"
thomwolf's avatar
thomwolf committed
41
   You should now have a /dist directory with both .whl and .tar.gz source versions.
42

43
8. Check that everything looks correct by uploading the package to the pypi test server:
44
45
46

   twine upload dist/* -r pypitest
   (pypi suggest using twine as other methods upload files via plaintext.)
Lysandre's avatar
Lysandre committed
47
48
   You may have to specify the repository url, use the following command then:
   twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/
49
50

   Check that you can install it in a virtualenv by running:
51
   pip install -i https://testpypi.python.org/pypi transformers
52

53
54
55
56
57
   Check you can run the following commands:
   python -c "from transformers import pipeline; classifier = pipeline('text-classification'); print(classifier('What a nice release'))" 
   python -c "from transformers import *"

9. Upload the final version to actual pypi:
58
59
   twine upload dist/* -r pypi

60
10. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
61

62
63
11. Run `make post-release` (or, for a patch release, `make post-patch`). If you were on a branch for the release, 
    you need to go back to master before executing this.
64
"""
Aymeric Augustin's avatar
Aymeric Augustin committed
65

Stas Bekman's avatar
Stas Bekman committed
66
import os
67
import re
68
import shutil
69
from distutils.core import Command
70
71
from pathlib import Path

thomwolf's avatar
thomwolf committed
72
73
from setuptools import find_packages, setup

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
stale_egg_info = Path(__file__).parent / "transformers.egg-info"
if stale_egg_info.exists():
    print(
        (
            "Warning: {} exists.\n\n"
            "If you recently updated transformers to 3.0 or later, this is expected,\n"
            "but it may prevent transformers from installing in editable mode.\n\n"
            "This directory is automatically generated by Python's packaging tools.\n"
            "I will remove it now.\n\n"
            "See https://github.com/pypa/pip/issues/5466 for details.\n"
        ).format(stale_egg_info)
    )
    shutil.rmtree(stale_egg_info)


90
91
92
93
# IMPORTANT:
# 1. all dependencies should be listed here with their version requirements if any
# 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
_deps = [
94
    "Pillow",
Sylvain Gugger's avatar
Sylvain Gugger committed
95
    "black==21.4b0",
96
    "codecarbon==1.2.0",
97
98
99
    "cookiecutter==1.7.2",
    "dataclasses",
    "datasets",
100
    "deepspeed>=0.5.3",
Lysandre Debut's avatar
Lysandre Debut committed
101
    "docutils==0.16.0",
102
    "fairscale>0.3",
103
104
105
106
    "faiss-cpu",
    "fastapi",
    "filelock",
    "flake8>=3.8.3",
107
    "flax>=0.3.4",
108
    "fugashi>=1.0",
109
    "GitPython<3.1.19",
110
    "huggingface-hub>=0.1.0,<1.0",
111
    "importlib_metadata",
112
113
    "ipadic>=1.0.0,<2.0",
    "isort>=5.5.4",
114
    "jax>=0.2.8",
115
    "jaxlib>=0.1.65",
116
    "jieba",
117
    "keras!=2.7.0", # Remove when they fix their release
118
    "keras2onnx",
119
    "nltk",
Sylvain Gugger's avatar
Sylvain Gugger committed
120
    "numpy>=1.17",
Lysandre's avatar
Lysandre committed
121
    "onnxconverter-common",
122
123
    "onnxruntime-tools>=1.4.2",
    "onnxruntime>=1.4.0",
Lysandre Debut's avatar
Lysandre Debut committed
124
    "optuna",
Vasudev Gupta's avatar
Vasudev Gupta committed
125
    "optax>=0.0.8",
126
    "packaging>=20.0",
127
128
129
    "parameterized",
    "protobuf",
    "psutil",
130
    "pyyaml>=5.1",
131
132
    "pydantic",
    "pytest",
Lysandre Debut's avatar
Lysandre Debut committed
133
    "pytest-timeout",
134
135
    "pytest-xdist",
    "python>=3.6.0",
Richard Liaw's avatar
Richard Liaw committed
136
    "ray[tune]",
137
138
139
    "recommonmark",
    "regex!=2019.12.17",
    "requests",
140
    "rouge-score",
Sylvain Gugger's avatar
Sylvain Gugger committed
141
    "sacrebleu>=1.4.12,<2.0.0",
142
    "sacremoses",
143
    "sagemaker>=2.31.0",
144
    "scikit-learn",
145
    "sentencepiece>=0.1.91,!=0.1.92",
146
    "sigopt",
147
    "librosa",
148
149
150
    "sphinx-copybutton",
    "sphinx-markdown-tables",
    "sphinx-rtd-theme==0.4.3",  # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
151
    "sphinx==3.2.1",
152
    "sphinxext-opengraph==0.4.1",
153
    "sphinx-intl",
154
    "starlette",
155
156
    "tensorflow-cpu>=2.3",
    "tensorflow>=2.3",
157
    "timeout-decorator",
NielsRogge's avatar
NielsRogge committed
158
    "timm",
159
    "tokenizers>=0.10.1,<0.11",
Lysandre Debut's avatar
Lysandre Debut committed
160
    "torch>=1.0",
Suraj Patil's avatar
Suraj Patil committed
161
    "torchaudio",
162
163
164
165
    "tqdm>=4.27",
    "unidic>=1.0.2",
    "unidic_lite>=1.0.7",
    "uvicorn",
166
]
167
168


169
170
171
172
173
174
# this is a lookup table with items like:
#
# tokenizers: "tokenizers==0.9.4"
# packaging: "packaging"
#
# some of the values are versioned whereas others aren't.
175
176
deps = {b: a for a, b in (re.findall(r"^(([^!=<>]+)(?:[!=<>].*)?$)", x)[0] for x in _deps)}

177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
# since we save this data in src/transformers/dependency_versions_table.py it can be easily accessed from
# anywhere. If you need to quickly access the data from this table in a shell, you can do so easily with:
#
# python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets
#
# Just pass the desired package names to that script as it's shown with 2 packages above.
#
# If transformers is not yet installed and the work is done from the cloned repo remember to add `PYTHONPATH=src` to the script above
#
# You can then feed this for example to `pip`:
#
# pip install -U $(python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets)
#

193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224

def deps_list(*pkgs):
    return [deps[pkg] for pkg in pkgs]


class DepsTableUpdateCommand(Command):
    """
    A custom distutils command that updates the dependency table.
    usage: python setup.py deps_table_update
    """

    description = "build runtime dependency table"
    user_options = [
        # format: (long option, short option, description).
        ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
    ]

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
        content = [
            "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
            "# 1. modify the `_deps` dict in setup.py",
            "# 2. run `make deps_table_update``",
            "deps = {",
            entries,
            "}",
225
            "",
226
227
228
        ]
        target = "src/transformers/dependency_versions_table.py"
        print(f"updating {target}")
Julien Plu's avatar
Julien Plu committed
229
        with open(target, "w", encoding="utf-8", newline="\n") as f:
230
231
232
233
234
235
236
237
            f.write("\n".join(content))


extras = {}

extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic")
extras["sklearn"] = deps_list("scikit-learn")

238
239
extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "keras2onnx", "keras")
extras["tf-cpu"] = deps_list("tensorflow-cpu", "onnxconverter-common", "keras2onnx", "keras")
240
241

extras["torch"] = deps_list("torch")
Stas Bekman's avatar
Stas Bekman committed
242
243

if os.name == "nt":  # windows
244
245
    extras["retrieval"] = deps_list("datasets")  # faiss is not supported on windows
    extras["flax"] = []  # jax is not supported on windows
246
else:
247
    extras["retrieval"] = deps_list("faiss-cpu", "datasets")
Vasudev Gupta's avatar
Vasudev Gupta committed
248
    extras["flax"] = deps_list("jax", "jaxlib", "flax", "optax")
249

250
251
extras["tokenizers"] = deps_list("tokenizers")
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
Lysandre Debut's avatar
Lysandre Debut committed
252
extras["onnx"] = deps_list("onnxconverter-common", "keras2onnx") + extras["onnxruntime"]
253
extras["modelcreation"] = deps_list("cookiecutter")
254

Philipp Schmid's avatar
Philipp Schmid committed
255
extras["sagemaker"] = deps_list("sagemaker")
256
257
extras["deepspeed"] = deps_list("deepspeed")
extras["fairscale"] = deps_list("fairscale")
Lysandre Debut's avatar
Lysandre Debut committed
258
extras["optuna"] = deps_list("optuna")
Richard Liaw's avatar
Richard Liaw committed
259
extras["ray"] = deps_list("ray[tune]")
260
extras["sigopt"] = deps_list("sigopt")
Lysandre Debut's avatar
Lysandre Debut committed
261

262
extras["integrations"] = extras["optuna"] + extras["ray"] + extras["sigopt"]
Philipp Schmid's avatar
Philipp Schmid committed
263

264
extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
265
extras["audio"] = deps_list("librosa")
266
267
268
269
extras["speech"] = deps_list("torchaudio") + extras["audio"]  # `pip install ".[speech]"` is deprecated and `pip install ".[torch-speech]"` should be used instead
extras["torch-speech"] = deps_list("torchaudio") + extras["audio"]
extras["tf-speech"] = extras["audio"]
extras["flax-speech"] = extras["audio"]
270
extras["vision"] = deps_list("Pillow")
NielsRogge's avatar
NielsRogge committed
271
extras["timm"] = deps_list("timm")
272
extras["codecarbon"] = deps_list("codecarbon")
Stas Bekman's avatar
Stas Bekman committed
273

274
275
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
extras["testing"] = (
Philipp Schmid's avatar
Philipp Schmid committed
276
    deps_list(
277
        "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-timeout", "black", "sacrebleu", "rouge-score", "nltk", "GitPython"
Philipp Schmid's avatar
Philipp Schmid committed
278
    )
279
280
281
    + extras["retrieval"]
    + extras["modelcreation"]
)
282

283
extras["quality"] = deps_list("black", "isort", "flake8")
Stas Bekman's avatar
Stas Bekman committed
284

285
286
287
288
289
290
extras["all"] = (
    extras["tf"]
    + extras["torch"]
    + extras["flax"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
291
    + extras["torch-speech"]
292
    + extras["vision"]
Lysandre Debut's avatar
Lysandre Debut committed
293
    + extras["integrations"]
NielsRogge's avatar
NielsRogge committed
294
    + extras["timm"]
295
    + extras["codecarbon"]
296
)
Stas Bekman's avatar
Stas Bekman committed
297

298
299
300
301
302
303
304
305
extras["docs_specific"] = deps_list(
    "docutils",
    "recommonmark",
    "sphinx",
    "sphinx-markdown-tables",
    "sphinx-rtd-theme",
    "sphinx-copybutton",
    "sphinxext-opengraph",
306
    "sphinx-intl",
307
308
309
310
)
# "docs" needs "all" to resolve all the references
extras["docs"] = extras["all"] + extras["docs_specific"]

311
312
313
314
315
extras["dev"] = (
    extras["all"]
    + extras["testing"]
    + extras["quality"]
    + extras["ja"]
316
    + extras["docs_specific"]
317
318
319
    + extras["sklearn"]
    + extras["modelcreation"]
)
Stas Bekman's avatar
Stas Bekman committed
320

321
322
extras["torchhub"] = deps_list(
    "filelock",
Sylvain Gugger's avatar
Sylvain Gugger committed
323
    "huggingface-hub",
324
325
326
327
328
329
330
331
332
333
334
335
    "importlib_metadata",
    "numpy",
    "packaging",
    "protobuf",
    "regex",
    "requests",
    "sacremoses",
    "sentencepiece",
    "torch",
    "tokenizers",
    "tqdm",
)
336

337
338
339
# when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
install_requires = [
    deps["dataclasses"] + ";python_version<'3.7'",  # dataclasses for Python versions that don't have it
340
    deps["importlib_metadata"] + ";python_version<'3.8'",  # importlib_metadata for Python versions that don't have it
341
    deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
342
    deps["huggingface-hub"],
343
344
    deps["numpy"],
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
Vasudev Gupta's avatar
Vasudev Gupta committed
345
    deps["pyyaml"],  # used for the model cards metadata
346
347
348
    deps["regex"],  # for OpenAI GPT
    deps["requests"],  # for downloading models over HTTPS
    deps["sacremoses"],  # for XLM
Sylvain Gugger's avatar
Sylvain Gugger committed
349
    deps["tokenizers"],
350
    deps["tqdm"],  # progress bars in model download and training scripts
351
352
]

thomwolf's avatar
thomwolf committed
353
setup(
354
    name="transformers",
Lysandre's avatar
Lysandre committed
355
    version="4.13.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
356
    author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Suraj Patil, Stas Bekman, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
thomwolf's avatar
thomwolf committed
357
    author_email="thomas@huggingface.co",
thomwolf's avatar
thomwolf committed
358
    description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
359
    long_description=open("README.md", "r", encoding="utf-8").read(),
thomwolf's avatar
thomwolf committed
360
    long_description_content_type="text/markdown",
361
362
    keywords="NLP deep learning transformer pytorch tensorflow BERT GPT GPT-2 google openai CMU",
    license="Apache",
363
    url="https://github.com/huggingface/transformers",
Aymeric Augustin's avatar
Aymeric Augustin committed
364
    package_dir={"": "src"},
365
    packages=find_packages("src"),
Will Frey's avatar
Will Frey committed
366
367
    package_data={"transformers": ["py.typed"]},
    zip_safe=False,
368
    extras_require=extras,
369
    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
370
    python_requires=">=3.6.0",
371
    install_requires=install_requires,
thomwolf's avatar
thomwolf committed
372
    classifiers=[
Aymeric Augustin's avatar
Aymeric Augustin committed
373
374
375
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
376
377
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
Aymeric Augustin's avatar
Aymeric Augustin committed
378
        "Operating System :: OS Independent",
379
        "Programming Language :: Python :: 3",
Aymeric Augustin's avatar
Aymeric Augustin committed
380
381
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
Anurag Kumar's avatar
Anurag Kumar committed
382
383
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
384
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
thomwolf's avatar
thomwolf committed
385
    ],
386
    cmdclass={"deps_table_update": DepsTableUpdateCommand},
thomwolf's avatar
thomwolf committed
387
)