setup.py 15.7 KB
Newer Older
1
# Copyright 2021 The HuggingFace Team. All rights reserved.
Sylvain Gugger's avatar
Sylvain Gugger committed
2
3
4
5
6
7
8
9
10
11
12
13
14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
"""
16
Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/main/setup.py
17
18
19

To create the package for pypi.

Sylvain Gugger's avatar
Sylvain Gugger committed
20
21
1. Run `make pre-release` (or `make pre-patch` for a patch release) then run `make fix-copies` to fix the index of the
   documentation.
22

23
24
   If releasing on a special branch, copy the updated README.md on the main branch for your the commit you will make
   for the post-release and run `make fix-copies` on the main branch as well.
25

Philipp Schmid's avatar
Philipp Schmid committed
26
2. Run Tests for Amazon Sagemaker. The documentation is located in `./tests/sagemaker/README.md`, otherwise @philschmid.
27

Philipp Schmid's avatar
Philipp Schmid committed
28
3. Unpin specific versions from setup.py that use a git install.
Lysandre's avatar
Lysandre committed
29

30
4. Checkout the release branch (v<RELEASE>-release, for example v4.19-release), and commit these changes with the
Lysandre Debut's avatar
Lysandre Debut committed
31
   message: "Release: <VERSION>" and push.
32

33
5. Wait for the tests on main to be completed and be green (otherwise revert and fix bugs)
34

Patrick von Platen's avatar
Patrick von Platen committed
35
6. Add a tag in git to mark the release: "git tag v<VERSION> -m 'Adds tag v<VERSION> for pypi' "
Lysandre Debut's avatar
Lysandre Debut committed
36
   Push the tag to git: git push --tags origin v<RELEASE>-release
37

38
7. Build both the sources and the wheel. Do not change anything in setup.py between
39
40
   creating the wheel and the source distribution (obviously).

41
42
43
44
   Clean up your build and dist folders (to avoid re-uploading oldies):
   rm -rf dist
   rm -rf build

thomwolf's avatar
thomwolf committed
45
   For the wheel, run: "python setup.py bdist_wheel" in the top level directory.
46
   (this will build a wheel for the python version you use to build it).
47
48

   For the sources, run: "python setup.py sdist"
thomwolf's avatar
thomwolf committed
49
   You should now have a /dist directory with both .whl and .tar.gz source versions.
50

51
8. Check that everything looks correct by uploading the package to the pypi test server:
52

53
   twine upload dist/* -r testpypi
54
   (pypi suggest using twine as other methods upload files via plaintext.)
Lysandre's avatar
Lysandre committed
55
   You may have to specify the repository url, use the following command then:
56
   twine upload dist/* -r testpypi --repository-url=https://test.pypi.org/legacy/
57
58

   Check that you can install it in a virtualenv by running:
59
   pip install -i https://testpypi.python.org/pypi transformers
60

61
   Check you can run the following commands:
62
   python -c "from transformers import pipeline; classifier = pipeline('text-classification'); print(classifier('What a nice release'))"
63
64
   python -c "from transformers import *"

65
66
   If making a patch release, double check the bug you are patching is indeed resolved.

67
9. Upload the final version to actual pypi:
68
69
   twine upload dist/* -r pypi

70
10. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
71

72
11. Run `make post-release` then run `make fix-copies`. If you were on a branch for the release,
73
    you need to go back to main before executing this.
74
"""
Aymeric Augustin's avatar
Aymeric Augustin committed
75

Stas Bekman's avatar
Stas Bekman committed
76
import os
77
import re
78
79
80
import shutil
from pathlib import Path

81
from setuptools import Command, find_packages, setup
thomwolf's avatar
thomwolf committed
82

83

84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
stale_egg_info = Path(__file__).parent / "transformers.egg-info"
if stale_egg_info.exists():
    print(
        (
            "Warning: {} exists.\n\n"
            "If you recently updated transformers to 3.0 or later, this is expected,\n"
            "but it may prevent transformers from installing in editable mode.\n\n"
            "This directory is automatically generated by Python's packaging tools.\n"
            "I will remove it now.\n\n"
            "See https://github.com/pypa/pip/issues/5466 for details.\n"
        ).format(stale_egg_info)
    )
    shutil.rmtree(stale_egg_info)


100
101
102
103
# IMPORTANT:
# 1. all dependencies should be listed here with their version requirements if any
# 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
_deps = [
104
    "Pillow",
105
    "accelerate>=0.17.0",
106
    "av==9.2.0",  # Latest version of PyAV (10.0.0) has issues with audio stream.
Sylvain Gugger's avatar
Sylvain Gugger committed
107
    "beautifulsoup4",
108
    "black~=23.1",
109
    "codecarbon==1.2.0",
Yih-Dar's avatar
Yih-Dar committed
110
    "cookiecutter==1.7.3",
111
    "dataclasses",
112
    "datasets!=2.5.0",
113
    "decord==0.6.0",
114
    "deepspeed>=0.8.3",
115
    "dill<0.3.5",
116
    "evaluate>=0.2.0",
117
    "fairscale>0.3",
118
119
120
    "faiss-cpu",
    "fastapi",
    "filelock",
121
    "flax>=0.4.1,<=0.6.9",
122
    "ftfy",
123
    "fugashi>=1.0",
124
    "GitPython<3.1.19",
125
    "hf-doc-builder>=0.3.0",
126
    "huggingface-hub>=0.11.0,<1.0",
127
    "importlib_metadata",
128
129
    "ipadic>=1.0.0,<2.0",
    "isort>=5.5.4",
130
131
    "jax>=0.2.8,!=0.3.2,<=0.3.6",
    "jaxlib>=0.1.65,<=0.3.6",
132
    "jieba",
133
    "kenlm",
134
    "keras-nlp>=0.3.1",
Sylvain Gugger's avatar
Sylvain Gugger committed
135
    "librosa",
136
    "nltk",
137
    "natten>=0.14.6",
Sylvain Gugger's avatar
Sylvain Gugger committed
138
    "numba<0.57.0",  # Can be removed once unpinned.
Sylvain Gugger's avatar
Sylvain Gugger committed
139
    "numpy>=1.17",
Lysandre's avatar
Lysandre committed
140
    "onnxconverter-common",
141
142
    "onnxruntime-tools>=1.4.2",
    "onnxruntime>=1.4.0",
Lysandre Debut's avatar
Lysandre Debut committed
143
    "optuna",
144
    "optax>=0.0.8,<=0.1.4",
145
    "packaging>=20.0",
146
    "parameterized",
147
    "phonemizer",
148
    "protobuf<=3.20.2",
149
    "psutil",
150
    "pyyaml>=5.1",
151
152
    "pydantic",
    "pytest",
Lysandre Debut's avatar
Lysandre Debut committed
153
    "pytest-timeout",
154
    "pytest-xdist",
Lysandre Debut's avatar
Lysandre Debut committed
155
    "python>=3.7.0",
Richard Liaw's avatar
Richard Liaw committed
156
    "ray[tune]",
157
158
    "regex!=2019.12.17",
    "requests",
Sylvain Gugger's avatar
Sylvain Gugger committed
159
    "rhoknp>=1.1.0",
160
    "rjieba",
161
    "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
Sylvain Gugger's avatar
Sylvain Gugger committed
162
    "ruff>=0.0.241,<=0.0.259",
Sylvain Gugger's avatar
Sylvain Gugger committed
163
    "sacrebleu>=1.4.12,<2.0.0",
164
    "sacremoses",
165
    "safetensors>=0.2.1",
166
    "sagemaker>=2.31.0",
167
    "scikit-learn",
168
    "sentencepiece>=0.1.91,!=0.1.92",
169
    "sigopt",
170
    "starlette",
Sylvain Gugger's avatar
Sylvain Gugger committed
171
172
    "sudachipy>=0.6.6",
    "sudachidict_core>=20220729",
173
    # TensorFlow pin. When changing this value, update examples/tensorflow/_tests_requirements.txt accordingly
174
175
176
    "tensorflow-cpu>=2.4,<2.13",
    "tensorflow>=2.4,<2.13",
    "tensorflow-text<2.13",
177
    "tf2onnx",
178
    "timeout-decorator",
NielsRogge's avatar
NielsRogge committed
179
    "timm",
180
    "tokenizers>=0.11.1,!=0.11.3,<0.14",
181
    "torch>=1.9,!=1.12.0",
Suraj Patil's avatar
Suraj Patil committed
182
    "torchaudio",
NielsRogge's avatar
NielsRogge committed
183
    "torchvision",
184
    "pyctcdecode>=0.4.0",
185
186
187
188
    "tqdm>=4.27",
    "unidic>=1.0.2",
    "unidic_lite>=1.0.7",
    "uvicorn",
189
]
190
191


192
193
194
195
196
197
# this is a lookup table with items like:
#
# tokenizers: "tokenizers==0.9.4"
# packaging: "packaging"
#
# some of the values are versioned whereas others aren't.
198
deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)}
199

200
201
202
203
204
205
206
207
208
209
210
211
212
# since we save this data in src/transformers/dependency_versions_table.py it can be easily accessed from
# anywhere. If you need to quickly access the data from this table in a shell, you can do so easily with:
#
# python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets
#
# Just pass the desired package names to that script as it's shown with 2 packages above.
#
# If transformers is not yet installed and the work is done from the cloned repo remember to add `PYTHONPATH=src` to the script above
#
# You can then feed this for example to `pip`:
#
# pip install -U $(python -c 'import sys; from transformers.dependency_versions_table import deps; \
Stas Bekman's avatar
Stas Bekman committed
213
# print(" ".join([deps[x] for x in sys.argv[1:]]))' tokenizers datasets)
214
215
#

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

def deps_list(*pkgs):
    return [deps[pkg] for pkg in pkgs]


class DepsTableUpdateCommand(Command):
    """
    A custom distutils command that updates the dependency table.
    usage: python setup.py deps_table_update
    """

    description = "build runtime dependency table"
    user_options = [
        # format: (long option, short option, description).
        ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
    ]

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
        content = [
            "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
            "# 1. modify the `_deps` dict in setup.py",
            "# 2. run `make deps_table_update``",
            "deps = {",
            entries,
            "}",
248
            "",
249
250
251
        ]
        target = "src/transformers/dependency_versions_table.py"
        print(f"updating {target}")
Julien Plu's avatar
Julien Plu committed
252
        with open(target, "w", encoding="utf-8", newline="\n") as f:
253
254
            f.write("\n".join(content))

255

256
257
extras = {}

Hao Wang's avatar
Hao Wang committed
258
extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic", "sudachipy", "sudachidict_core", "rhoknp")
259
260
extras["sklearn"] = deps_list("scikit-learn")

261
262
extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp")
extras["tf-cpu"] = deps_list("tensorflow-cpu", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp")
263

264
extras["torch"] = deps_list("torch", "accelerate")
265
extras["accelerate"] = deps_list("accelerate")
Stas Bekman's avatar
Stas Bekman committed
266
267

if os.name == "nt":  # windows
268
269
    extras["retrieval"] = deps_list("datasets")  # faiss is not supported on windows
    extras["flax"] = []  # jax is not supported on windows
270
else:
271
    extras["retrieval"] = deps_list("faiss-cpu", "datasets")
Vasudev Gupta's avatar
Vasudev Gupta committed
272
    extras["flax"] = deps_list("jax", "jaxlib", "flax", "optax")
273

274
extras["tokenizers"] = deps_list("tokenizers")
275
extras["ftfy"] = deps_list("ftfy")
276
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
277
extras["onnx"] = deps_list("onnxconverter-common", "tf2onnx") + extras["onnxruntime"]
278
extras["modelcreation"] = deps_list("cookiecutter")
279

Philipp Schmid's avatar
Philipp Schmid committed
280
extras["sagemaker"] = deps_list("sagemaker")
281
extras["deepspeed"] = deps_list("deepspeed") + extras["accelerate"]
282
extras["fairscale"] = deps_list("fairscale")
Lysandre Debut's avatar
Lysandre Debut committed
283
extras["optuna"] = deps_list("optuna")
Richard Liaw's avatar
Richard Liaw committed
284
extras["ray"] = deps_list("ray[tune]")
285
extras["sigopt"] = deps_list("sigopt")
Lysandre Debut's avatar
Lysandre Debut committed
286

287
extras["integrations"] = extras["optuna"] + extras["ray"] + extras["sigopt"]
Philipp Schmid's avatar
Philipp Schmid committed
288

289
extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
Sylvain Gugger's avatar
Sylvain Gugger committed
290
291
# numba can be removed here once unpinned
extras["audio"] = deps_list("librosa", "pyctcdecode", "phonemizer", "kenlm", "numba")
292
293
# `pip install ".[speech]"` is deprecated and `pip install ".[torch-speech]"` should be used instead
extras["speech"] = deps_list("torchaudio") + extras["audio"]
294
295
296
extras["torch-speech"] = deps_list("torchaudio") + extras["audio"]
extras["tf-speech"] = extras["audio"]
extras["flax-speech"] = extras["audio"]
297
extras["vision"] = deps_list("Pillow")
NielsRogge's avatar
NielsRogge committed
298
extras["timm"] = deps_list("timm")
NielsRogge's avatar
NielsRogge committed
299
extras["torch-vision"] = deps_list("torchvision") + extras["vision"]
300
extras["natten"] = deps_list("natten")
301
extras["codecarbon"] = deps_list("codecarbon")
302
extras["video"] = deps_list("decord", "av")
303

304
305
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
extras["testing"] = (
Philipp Schmid's avatar
Philipp Schmid committed
306
    deps_list(
307
308
309
310
311
312
        "pytest",
        "pytest-xdist",
        "timeout-decorator",
        "parameterized",
        "psutil",
        "datasets",
313
        "dill",
314
        "evaluate",
315
316
317
318
319
320
        "pytest-timeout",
        "black",
        "sacrebleu",
        "rouge-score",
        "nltk",
        "GitPython",
321
        "hf-doc-builder",
322
        "protobuf",  # Can be removed once we can unpin protobuf
323
        "sacremoses",
324
        "rjieba",
325
        "safetensors",
326
        "beautifulsoup4",
Philipp Schmid's avatar
Philipp Schmid committed
327
    )
328
329
330
    + extras["retrieval"]
    + extras["modelcreation"]
)
331

332
extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]
333

334
extras["quality"] = deps_list("black", "datasets", "isort", "ruff", "GitPython", "hf-doc-builder")
Stas Bekman's avatar
Stas Bekman committed
335

336
337
338
339
340
341
extras["all"] = (
    extras["tf"]
    + extras["torch"]
    + extras["flax"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
342
    + extras["torch-speech"]
343
    + extras["vision"]
Lysandre Debut's avatar
Lysandre Debut committed
344
    + extras["integrations"]
NielsRogge's avatar
NielsRogge committed
345
    + extras["timm"]
NielsRogge's avatar
NielsRogge committed
346
    + extras["torch-vision"]
347
    + extras["codecarbon"]
348
    + extras["accelerate"]
349
    + extras["video"]
350
)
Stas Bekman's avatar
Stas Bekman committed
351

352
# Might need to add doc-builder and some specific deps in the future
353
extras["docs_specific"] = ["hf-doc-builder"]
354

355
356
357
# "docs" needs "all" to resolve all the references
extras["docs"] = extras["all"] + extras["docs_specific"]

358
extras["dev-torch"] = (
359
360
    extras["testing"]
    + extras["torch"]
361
362
363
364
365
366
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["torch-speech"]
    + extras["vision"]
    + extras["integrations"]
    + extras["timm"]
NielsRogge's avatar
NielsRogge committed
367
    + extras["torch-vision"]
368
369
370
371
372
373
374
375
376
    + extras["codecarbon"]
    + extras["quality"]
    + extras["ja"]
    + extras["docs_specific"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnxruntime"]
)
extras["dev-tensorflow"] = (
377
378
379
380
381
382
383
384
385
386
387
    extras["testing"]
    + extras["tf"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["vision"]
    + extras["quality"]
    + extras["docs_specific"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnx"]
    + extras["tf-speech"]
388
)
389
390
391
392
393
extras["dev"] = (
    extras["all"]
    + extras["testing"]
    + extras["quality"]
    + extras["ja"]
394
    + extras["docs_specific"]
395
396
397
    + extras["sklearn"]
    + extras["modelcreation"]
)
Stas Bekman's avatar
Stas Bekman committed
398

399
400
extras["torchhub"] = deps_list(
    "filelock",
Sylvain Gugger's avatar
Sylvain Gugger committed
401
    "huggingface-hub",
402
403
404
405
406
407
408
409
410
411
412
    "importlib_metadata",
    "numpy",
    "packaging",
    "protobuf",
    "regex",
    "requests",
    "sentencepiece",
    "torch",
    "tokenizers",
    "tqdm",
)
413

414
415
# when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
install_requires = [
416
    deps["importlib_metadata"] + ";python_version<'3.8'",  # importlib_metadata for Python versions that don't have it
417
    deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
418
    deps["huggingface-hub"],
419
420
    deps["numpy"],
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
Vasudev Gupta's avatar
Vasudev Gupta committed
421
    deps["pyyaml"],  # used for the model cards metadata
422
423
    deps["regex"],  # for OpenAI GPT
    deps["requests"],  # for downloading models over HTTPS
Sylvain Gugger's avatar
Sylvain Gugger committed
424
    deps["tokenizers"],
425
    deps["tqdm"],  # progress bars in model download and training scripts
426
427
]

thomwolf's avatar
thomwolf committed
428
setup(
429
    name="transformers",
Sylvain Gugger's avatar
Sylvain Gugger committed
430
    version="4.29.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
431
432
433
434
435
436
437
438
439
440
441
442
443
    author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)",
    author_email="transformers@huggingface.co",
    description="State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow",
    long_description=open("README.md", "r", encoding="utf-8").read(),
    long_description_content_type="text/markdown",
    keywords="NLP vision speech deep learning transformer pytorch tensorflow jax BERT GPT-2 Wav2Vec2 ViT",
    license="Apache 2.0 License",
    url="https://github.com/huggingface/transformers",
    package_dir={"": "src"},
    packages=find_packages("src"),
    include_package_data=True,
    package_data={"transformers": ["*.cu", "*.cpp", "*.cuh", "*.h", "*.pyx"]},
    zip_safe=False,
444
    extras_require=extras,
445
446
    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
    python_requires=">=3.7.0",
447
    install_requires=install_requires,
448
449
450
451
452
453
454
455
456
457
458
459
460
461
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ],
462
    cmdclass={"deps_table_update": DepsTableUpdateCommand},
thomwolf's avatar
thomwolf committed
463
)