"examples/summarization/t5/__init__.py" did not exist on "1484d67de9f4c8301874d62d821bbfbd3d6ce99b"
setup.py 11.7 KB
Newer Older
Sylvain Gugger's avatar
Sylvain Gugger committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
"""
Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/master/setup.py

To create the package for pypi.

Sylvain Gugger's avatar
Sylvain Gugger committed
20
21
1. Run `make pre-release` (or `make pre-patch` for a patch release) then run `make fix-copies` to fix the index of the
   documentation.
22

Philipp Schmid's avatar
Philipp Schmid committed
23
2. Run Tests for Amazon Sagemaker. The documentation is located in `./tests/sagemaker/README.md`, otherwise @philschmid.
24

Philipp Schmid's avatar
Philipp Schmid committed
25
3. Unpin specific versions from setup.py that use a git install.
Lysandre's avatar
Lysandre committed
26

Philipp Schmid's avatar
Philipp Schmid committed
27
4. Commit these changes with the message: "Release: VERSION"
28

Philipp Schmid's avatar
Philipp Schmid committed
29
5. Add a tag in git to mark the release: "git tag VERSION -m 'Adds tag VERSION for pypi' "
30
31
   Push the tag to git: git push --tags origin master

Philipp Schmid's avatar
Philipp Schmid committed
32
6. Build both the sources and the wheel. Do not change anything in setup.py between
33
34
   creating the wheel and the source distribution (obviously).

thomwolf's avatar
thomwolf committed
35
   For the wheel, run: "python setup.py bdist_wheel" in the top level directory.
36
   (this will build a wheel for the python version you use to build it).
37
38

   For the sources, run: "python setup.py sdist"
thomwolf's avatar
thomwolf committed
39
   You should now have a /dist directory with both .whl and .tar.gz source versions.
40

Philipp Schmid's avatar
Philipp Schmid committed
41
7. Check that everything looks correct by uploading the package to the pypi test server:
42
43
44

   twine upload dist/* -r pypitest
   (pypi suggest using twine as other methods upload files via plaintext.)
Lysandre's avatar
Lysandre committed
45
46
   You may have to specify the repository url, use the following command then:
   twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/
47
48

   Check that you can install it in a virtualenv by running:
49
   pip install -i https://testpypi.python.org/pypi transformers
50

Philipp Schmid's avatar
Philipp Schmid committed
51
8. Upload the final version to actual pypi:
52
53
   twine upload dist/* -r pypi

Philipp Schmid's avatar
Philipp Schmid committed
54
9. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
55

Philipp Schmid's avatar
Philipp Schmid committed
56
10. Run `make post-release` (or `make post-patch` for a patch release).
57
"""
Aymeric Augustin's avatar
Aymeric Augustin committed
58

Stas Bekman's avatar
Stas Bekman committed
59
import os
60
import re
61
import shutil
62
from distutils.core import Command
63
64
from pathlib import Path

thomwolf's avatar
thomwolf committed
65
66
from setuptools import find_packages, setup

67

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
stale_egg_info = Path(__file__).parent / "transformers.egg-info"
if stale_egg_info.exists():
    print(
        (
            "Warning: {} exists.\n\n"
            "If you recently updated transformers to 3.0 or later, this is expected,\n"
            "but it may prevent transformers from installing in editable mode.\n\n"
            "This directory is automatically generated by Python's packaging tools.\n"
            "I will remove it now.\n\n"
            "See https://github.com/pypa/pip/issues/5466 for details.\n"
        ).format(stale_egg_info)
    )
    shutil.rmtree(stale_egg_info)


84
85
86
87
# IMPORTANT:
# 1. all dependencies should be listed here with their version requirements if any
# 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
_deps = [
88
    "Pillow",
89
90
91
92
    "black>=20.8b1",
    "cookiecutter==1.7.2",
    "dataclasses",
    "datasets",
93
    "deepspeed>0.3.13",
Lysandre Debut's avatar
Lysandre Debut committed
94
    "docutils==0.16.0",
95
    "fairscale>0.3",
96
97
98
99
    "faiss-cpu",
    "fastapi",
    "filelock",
    "flake8>=3.8.3",
100
    "flax>=0.3.2",
101
    "fugashi>=1.0",
102
    "importlib_metadata",
103
104
    "ipadic>=1.0.0,<2.0",
    "isort>=5.5.4",
105
    "jax>=0.2.8",
106
    "jaxlib>=0.1.59",
107
    "keras2onnx",
108
    "nltk",
Sylvain Gugger's avatar
Sylvain Gugger committed
109
    "numpy>=1.17",
Lysandre's avatar
Lysandre committed
110
    "onnxconverter-common",
111
112
113
114
115
116
117
118
    "onnxruntime-tools>=1.4.2",
    "onnxruntime>=1.4.0",
    "packaging",
    "parameterized",
    "protobuf",
    "psutil",
    "pydantic",
    "pytest",
Lysandre Debut's avatar
Lysandre Debut committed
119
    "pytest-sugar",
120
121
122
123
124
    "pytest-xdist",
    "python>=3.6.0",
    "recommonmark",
    "regex!=2019.12.17",
    "requests",
125
126
    "rouge-score",
    "sacrebleu>=1.4.12",
127
    "sacremoses",
128
    "sagemaker>=2.31.0",
129
130
    "scikit-learn",
    "sentencepiece==0.1.91",
Patrick von Platen's avatar
Patrick von Platen committed
131
    "soundfile",
132
133
134
135
    "sphinx-copybutton",
    "sphinx-markdown-tables",
    "sphinx-rtd-theme==0.4.3",  # sphinx-rtd-theme==0.5.0 introduced big changes in the style.
    "sphinx==3.2.1",
136
    "sphinxext-opengraph==0.4.1",
137
    "starlette",
138
139
    "tensorflow-cpu>=2.3",
    "tensorflow>=2.3",
140
    "timeout-decorator",
141
    "tokenizers>=0.10.1,<0.11",
142
    "torch>=1.0",
Suraj Patil's avatar
Suraj Patil committed
143
    "torchaudio",
144
145
146
147
    "tqdm>=4.27",
    "unidic>=1.0.2",
    "unidic_lite>=1.0.7",
    "uvicorn",
148
]
149
150


151
152
153
154
155
156
# this is a lookup table with items like:
#
# tokenizers: "tokenizers==0.9.4"
# packaging: "packaging"
#
# some of the values are versioned whereas others aren't.
157
158
deps = {b: a for a, b in (re.findall(r"^(([^!=<>]+)(?:[!=<>].*)?$)", x)[0] for x in _deps)}

159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
# since we save this data in src/transformers/dependency_versions_table.py it can be easily accessed from
# anywhere. If you need to quickly access the data from this table in a shell, you can do so easily with:
#
# python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets
#
# Just pass the desired package names to that script as it's shown with 2 packages above.
#
# If transformers is not yet installed and the work is done from the cloned repo remember to add `PYTHONPATH=src` to the script above
#
# You can then feed this for example to `pip`:
#
# pip install -U $(python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets)
#

175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206

def deps_list(*pkgs):
    return [deps[pkg] for pkg in pkgs]


class DepsTableUpdateCommand(Command):
    """
    A custom distutils command that updates the dependency table.
    usage: python setup.py deps_table_update
    """

    description = "build runtime dependency table"
    user_options = [
        # format: (long option, short option, description).
        ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
    ]

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
        content = [
            "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
            "# 1. modify the `_deps` dict in setup.py",
            "# 2. run `make deps_table_update``",
            "deps = {",
            entries,
            "}",
207
            "",
208
209
210
        ]
        target = "src/transformers/dependency_versions_table.py"
        print(f"updating {target}")
Julien Plu's avatar
Julien Plu committed
211
        with open(target, "w", encoding="utf-8", newline="\n") as f:
212
213
214
215
216
217
218
219
220
221
222
223
            f.write("\n".join(content))


extras = {}

extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic")
extras["sklearn"] = deps_list("scikit-learn")

extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "keras2onnx")
extras["tf-cpu"] = deps_list("tensorflow-cpu", "onnxconverter-common", "keras2onnx")

extras["torch"] = deps_list("torch")
Stas Bekman's avatar
Stas Bekman committed
224
225

if os.name == "nt":  # windows
226
227
    extras["retrieval"] = deps_list("datasets")  # faiss is not supported on windows
    extras["flax"] = []  # jax is not supported on windows
228
else:
229
230
    extras["retrieval"] = deps_list("faiss-cpu", "datasets")
    extras["flax"] = deps_list("jax", "jaxlib", "flax")
231

232
233
extras["tokenizers"] = deps_list("tokenizers")
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
Lysandre Debut's avatar
Lysandre Debut committed
234
extras["onnx"] = deps_list("onnxconverter-common", "keras2onnx") + extras["onnxruntime"]
235
extras["modelcreation"] = deps_list("cookiecutter")
236

Philipp Schmid's avatar
Philipp Schmid committed
237
extras["sagemaker"] = deps_list("sagemaker")
238
239
extras["deepspeed"] = deps_list("deepspeed")
extras["fairscale"] = deps_list("fairscale")
Philipp Schmid's avatar
Philipp Schmid committed
240

241
extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
Suraj Patil's avatar
Suraj Patil committed
242
extras["speech"] = deps_list("soundfile", "torchaudio")
243
extras["vision"] = deps_list("Pillow")
Stas Bekman's avatar
Stas Bekman committed
244

245
246
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
extras["testing"] = (
Philipp Schmid's avatar
Philipp Schmid committed
247
    deps_list(
248
        "pytest", "pytest-xdist", "timeout-decorator", "parameterized", "psutil", "datasets", "pytest-sugar", "black", "sacrebleu", "rouge-score", "nltk"
Philipp Schmid's avatar
Philipp Schmid committed
249
    )
250
251
252
    + extras["retrieval"]
    + extras["modelcreation"]
)
253

254
extras["quality"] = deps_list("black", "isort", "flake8")
Stas Bekman's avatar
Stas Bekman committed
255

256
257
258
259
260
261
262
263
264
extras["all"] = (
    extras["tf"]
    + extras["torch"]
    + extras["flax"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["speech"]
    + extras["vision"]
)
Stas Bekman's avatar
Stas Bekman committed
265

266
267
268
269
270
271
272
273
274
275
276
277
extras["docs_specific"] = deps_list(
    "docutils",
    "recommonmark",
    "sphinx",
    "sphinx-markdown-tables",
    "sphinx-rtd-theme",
    "sphinx-copybutton",
    "sphinxext-opengraph",
)
# "docs" needs "all" to resolve all the references
extras["docs"] = extras["all"] + extras["docs_specific"]

278
279
280
281
282
extras["dev"] = (
    extras["all"]
    + extras["testing"]
    + extras["quality"]
    + extras["ja"]
283
    + extras["docs_specific"]
284
285
286
    + extras["sklearn"]
    + extras["modelcreation"]
)
Stas Bekman's avatar
Stas Bekman committed
287

288
289
290
291
292
293
294
295
296
297
298
299
300
301
extras["torchhub"] = deps_list(
    "filelock",
    "importlib_metadata",
    "numpy",
    "packaging",
    "protobuf",
    "regex",
    "requests",
    "sacremoses",
    "sentencepiece",
    "torch",
    "tokenizers",
    "tqdm",
)
302

303
304
305
# when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
install_requires = [
    deps["dataclasses"] + ";python_version<'3.7'",  # dataclasses for Python versions that don't have it
306
    deps["importlib_metadata"] + ";python_version<'3.8'",  # importlib_metadata for Python versions that don't have it
307
    deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
308
309
    deps["numpy"],
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
310
311
312
    deps["regex"],  # for OpenAI GPT
    deps["requests"],  # for downloading models over HTTPS
    deps["sacremoses"],  # for XLM
Sylvain Gugger's avatar
Sylvain Gugger committed
313
    deps["tokenizers"],
314
    deps["tqdm"],  # progress bars in model download and training scripts
315
316
]

thomwolf's avatar
thomwolf committed
317
setup(
318
    name="transformers",
Lysandre's avatar
Lysandre committed
319
    version="4.6.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
sgugger's avatar
sgugger committed
320
    author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Sam Shleifer, Patrick von Platen, Sylvain Gugger, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors",
thomwolf's avatar
thomwolf committed
321
    author_email="thomas@huggingface.co",
thomwolf's avatar
thomwolf committed
322
    description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch",
323
    long_description=open("README.md", "r", encoding="utf-8").read(),
thomwolf's avatar
thomwolf committed
324
    long_description_content_type="text/markdown",
325
326
    keywords="NLP deep learning transformer pytorch tensorflow BERT GPT GPT-2 google openai CMU",
    license="Apache",
327
    url="https://github.com/huggingface/transformers",
Aymeric Augustin's avatar
Aymeric Augustin committed
328
    package_dir={"": "src"},
329
    packages=find_packages("src"),
330
    extras_require=extras,
331
    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
332
    python_requires=">=3.6.0",
333
    install_requires=install_requires,
thomwolf's avatar
thomwolf committed
334
    classifiers=[
Aymeric Augustin's avatar
Aymeric Augustin committed
335
336
337
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
338
339
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
Aymeric Augustin's avatar
Aymeric Augustin committed
340
        "Operating System :: OS Independent",
341
        "Programming Language :: Python :: 3",
Aymeric Augustin's avatar
Aymeric Augustin committed
342
343
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
344
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
thomwolf's avatar
thomwolf committed
345
    ],
346
    cmdclass={"deps_table_update": DepsTableUpdateCommand},
thomwolf's avatar
thomwolf committed
347
)