setup.py 15.6 KB
Newer Older
1
# Copyright 2021 The HuggingFace Team. All rights reserved.
Sylvain Gugger's avatar
Sylvain Gugger committed
2
3
4
5
6
7
8
9
10
11
12
13
14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
"""
16
Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/main/setup.py
17
18
19

To create the package for pypi.

Sylvain Gugger's avatar
Sylvain Gugger committed
20
21
1. Run `make pre-release` (or `make pre-patch` for a patch release) then run `make fix-copies` to fix the index of the
   documentation.
22

23
24
   If releasing on a special branch, copy the updated README.md on the main branch for your the commit you will make
   for the post-release and run `make fix-copies` on the main branch as well.
25

Philipp Schmid's avatar
Philipp Schmid committed
26
2. Run Tests for Amazon Sagemaker. The documentation is located in `./tests/sagemaker/README.md`, otherwise @philschmid.
27

Philipp Schmid's avatar
Philipp Schmid committed
28
3. Unpin specific versions from setup.py that use a git install.
Lysandre's avatar
Lysandre committed
29

30
4. Checkout the release branch (v<RELEASE>-release, for example v4.19-release), and commit these changes with the
Lysandre Debut's avatar
Lysandre Debut committed
31
   message: "Release: <VERSION>" and push.
32

33
5. Wait for the tests on main to be completed and be green (otherwise revert and fix bugs)
34

Patrick von Platen's avatar
Patrick von Platen committed
35
6. Add a tag in git to mark the release: "git tag v<VERSION> -m 'Adds tag v<VERSION> for pypi' "
Lysandre Debut's avatar
Lysandre Debut committed
36
   Push the tag to git: git push --tags origin v<RELEASE>-release
37

38
7. Build both the sources and the wheel. Do not change anything in setup.py between
39
40
   creating the wheel and the source distribution (obviously).

41
42
43
44
   Clean up your build and dist folders (to avoid re-uploading oldies):
   rm -rf dist
   rm -rf build

thomwolf's avatar
thomwolf committed
45
   For the wheel, run: "python setup.py bdist_wheel" in the top level directory.
46
   (this will build a wheel for the python version you use to build it).
47
48

   For the sources, run: "python setup.py sdist"
thomwolf's avatar
thomwolf committed
49
   You should now have a /dist directory with both .whl and .tar.gz source versions.
50

51
8. Check that everything looks correct by uploading the package to the pypi test server:
52

53
   twine upload dist/* -r testpypi
54
   (pypi suggest using twine as other methods upload files via plaintext.)
Lysandre's avatar
Lysandre committed
55
   You may have to specify the repository url, use the following command then:
56
   twine upload dist/* -r testpypi --repository-url=https://test.pypi.org/legacy/
57
58

   Check that you can install it in a virtualenv by running:
59
   pip install -i https://testpypi.python.org/pypi transformers
60

61
   Check you can run the following commands:
62
   python -c "from transformers import pipeline; classifier = pipeline('text-classification'); print(classifier('What a nice release'))"
63
64
   python -c "from transformers import *"

65
66
   If making a patch release, double check the bug you are patching is indeed resolved.

67
9. Upload the final version to actual pypi:
68
69
   twine upload dist/* -r pypi

70
10. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
71

72
11. Run `make post-release` then run `make fix-copies`. If you were on a branch for the release,
73
    you need to go back to main before executing this.
74
"""
Aymeric Augustin's avatar
Aymeric Augustin committed
75

Stas Bekman's avatar
Stas Bekman committed
76
import os
77
import re
78
import shutil
79
from distutils.core import Command
80
81
from pathlib import Path

thomwolf's avatar
thomwolf committed
82
83
from setuptools import find_packages, setup

84

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
stale_egg_info = Path(__file__).parent / "transformers.egg-info"
if stale_egg_info.exists():
    print(
        (
            "Warning: {} exists.\n\n"
            "If you recently updated transformers to 3.0 or later, this is expected,\n"
            "but it may prevent transformers from installing in editable mode.\n\n"
            "This directory is automatically generated by Python's packaging tools.\n"
            "I will remove it now.\n\n"
            "See https://github.com/pypa/pip/issues/5466 for details.\n"
        ).format(stale_egg_info)
    )
    shutil.rmtree(stale_egg_info)


101
102
103
104
# IMPORTANT:
# 1. all dependencies should be listed here with their version requirements if any
# 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
_deps = [
105
    "Pillow",
106
    "accelerate>=0.10.0",
107
    "av==9.2.0",  # Latest version of PyAV (10.0.0) has issues with audio stream.
Sylvain Gugger's avatar
Sylvain Gugger committed
108
    "beautifulsoup4",
109
    "black~=23.1",
110
    "codecarbon==1.2.0",
Yih-Dar's avatar
Yih-Dar committed
111
    "cookiecutter==1.7.3",
112
    "dataclasses",
113
    "datasets!=2.5.0",
114
    "decord==0.6.0",
115
    "deepspeed>=0.8.3",
116
    "dill<0.3.5",
117
    "evaluate>=0.2.0",
118
    "fairscale>0.3",
119
120
121
    "faiss-cpu",
    "fastapi",
    "filelock",
122
    "flax>=0.4.1",
123
    "ftfy",
124
    "fugashi>=1.0",
125
    "GitPython<3.1.19",
126
    "hf-doc-builder>=0.3.0",
127
    "huggingface-hub>=0.11.0,<1.0",
128
    "importlib_metadata",
129
130
    "ipadic>=1.0.0,<2.0",
    "isort>=5.5.4",
131
132
    "jax>=0.2.8,!=0.3.2,<=0.3.6",
    "jaxlib>=0.1.65,<=0.3.6",
133
    "jieba",
134
    "kenlm",
135
    "keras-nlp>=0.3.1",
Sylvain Gugger's avatar
Sylvain Gugger committed
136
    "librosa",
137
    "nltk",
138
    "natten>=0.14.6",
Sylvain Gugger's avatar
Sylvain Gugger committed
139
    "numpy>=1.17",
Lysandre's avatar
Lysandre committed
140
    "onnxconverter-common",
141
142
    "onnxruntime-tools>=1.4.2",
    "onnxruntime>=1.4.0",
Lysandre Debut's avatar
Lysandre Debut committed
143
    "optuna",
Vasudev Gupta's avatar
Vasudev Gupta committed
144
    "optax>=0.0.8",
145
    "packaging>=20.0",
146
    "parameterized",
147
    "phonemizer",
148
    "protobuf<=3.20.2",
149
    "psutil",
150
    "pyyaml>=5.1",
151
152
    "pydantic",
    "pytest",
Lysandre Debut's avatar
Lysandre Debut committed
153
    "pytest-timeout",
154
    "pytest-xdist",
Lysandre Debut's avatar
Lysandre Debut committed
155
    "python>=3.7.0",
Richard Liaw's avatar
Richard Liaw committed
156
    "ray[tune]",
157
158
    "regex!=2019.12.17",
    "requests",
Sylvain Gugger's avatar
Sylvain Gugger committed
159
    "rhoknp>=1.1.0",
160
    "rjieba",
161
    "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
Sylvain Gugger's avatar
Sylvain Gugger committed
162
    "ruff>=0.0.241,<=0.0.259",
Sylvain Gugger's avatar
Sylvain Gugger committed
163
    "sacrebleu>=1.4.12,<2.0.0",
164
    "sacremoses",
165
    "safetensors>=0.2.1",
166
    "sagemaker>=2.31.0",
167
    "scikit-learn",
168
    "sentencepiece>=0.1.91,!=0.1.92",
169
    "sigopt",
170
    "starlette",
Sylvain Gugger's avatar
Sylvain Gugger committed
171
172
    "sudachipy>=0.6.6",
    "sudachidict_core>=20220729",
173
    # TensorFlow pin. When changing this value, update examples/tensorflow/_tests_requirements.txt accordingly
174
175
176
    "tensorflow-cpu>=2.4,<2.13",
    "tensorflow>=2.4,<2.13",
    "tensorflow-text<2.13",
177
    "tf2onnx",
178
    "timeout-decorator",
NielsRogge's avatar
NielsRogge committed
179
    "timm",
180
    "tokenizers>=0.11.1,!=0.11.3,<0.14",
181
    "torch>=1.9,!=1.12.0",
Suraj Patil's avatar
Suraj Patil committed
182
    "torchaudio",
NielsRogge's avatar
NielsRogge committed
183
    "torchvision",
184
    "pyctcdecode>=0.4.0",
185
186
187
188
    "tqdm>=4.27",
    "unidic>=1.0.2",
    "unidic_lite>=1.0.7",
    "uvicorn",
189
]
190
191


192
193
194
195
196
197
# this is a lookup table with items like:
#
# tokenizers: "tokenizers==0.9.4"
# packaging: "packaging"
#
# some of the values are versioned whereas others aren't.
198
deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)}
199

200
201
202
203
204
205
206
207
208
209
210
211
212
# since we save this data in src/transformers/dependency_versions_table.py it can be easily accessed from
# anywhere. If you need to quickly access the data from this table in a shell, you can do so easily with:
#
# python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets
#
# Just pass the desired package names to that script as it's shown with 2 packages above.
#
# If transformers is not yet installed and the work is done from the cloned repo remember to add `PYTHONPATH=src` to the script above
#
# You can then feed this for example to `pip`:
#
# pip install -U $(python -c 'import sys; from transformers.dependency_versions_table import deps; \
Stas Bekman's avatar
Stas Bekman committed
213
# print(" ".join([deps[x] for x in sys.argv[1:]]))' tokenizers datasets)
214
215
#

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

def deps_list(*pkgs):
    return [deps[pkg] for pkg in pkgs]


class DepsTableUpdateCommand(Command):
    """
    A custom distutils command that updates the dependency table.
    usage: python setup.py deps_table_update
    """

    description = "build runtime dependency table"
    user_options = [
        # format: (long option, short option, description).
        ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
    ]

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
        content = [
            "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
            "# 1. modify the `_deps` dict in setup.py",
            "# 2. run `make deps_table_update``",
            "deps = {",
            entries,
            "}",
248
            "",
249
250
251
        ]
        target = "src/transformers/dependency_versions_table.py"
        print(f"updating {target}")
Julien Plu's avatar
Julien Plu committed
252
        with open(target, "w", encoding="utf-8", newline="\n") as f:
253
254
255
256
            f.write("\n".join(content))

extras = {}

Hao Wang's avatar
Hao Wang committed
257
extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic", "sudachipy", "sudachidict_core", "rhoknp")
258
259
extras["sklearn"] = deps_list("scikit-learn")

260
261
extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp")
extras["tf-cpu"] = deps_list("tensorflow-cpu", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp")
262
263

extras["torch"] = deps_list("torch")
264
extras["accelerate"] = deps_list("accelerate")
Stas Bekman's avatar
Stas Bekman committed
265
266

if os.name == "nt":  # windows
267
268
    extras["retrieval"] = deps_list("datasets")  # faiss is not supported on windows
    extras["flax"] = []  # jax is not supported on windows
269
else:
270
    extras["retrieval"] = deps_list("faiss-cpu", "datasets")
Vasudev Gupta's avatar
Vasudev Gupta committed
271
    extras["flax"] = deps_list("jax", "jaxlib", "flax", "optax")
272

273
extras["tokenizers"] = deps_list("tokenizers")
274
extras["ftfy"] = deps_list("ftfy")
275
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
276
extras["onnx"] = deps_list("onnxconverter-common", "tf2onnx") + extras["onnxruntime"]
277
extras["modelcreation"] = deps_list("cookiecutter")
278

Philipp Schmid's avatar
Philipp Schmid committed
279
extras["sagemaker"] = deps_list("sagemaker")
280
extras["deepspeed"] = deps_list("deepspeed") + extras["accelerate"]
281
extras["fairscale"] = deps_list("fairscale")
Lysandre Debut's avatar
Lysandre Debut committed
282
extras["optuna"] = deps_list("optuna")
Richard Liaw's avatar
Richard Liaw committed
283
extras["ray"] = deps_list("ray[tune]")
284
extras["sigopt"] = deps_list("sigopt")
Lysandre Debut's avatar
Lysandre Debut committed
285

286
extras["integrations"] = extras["optuna"] + extras["ray"] + extras["sigopt"]
Philipp Schmid's avatar
Philipp Schmid committed
287

288
extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
289
extras["audio"] = deps_list("librosa", "pyctcdecode", "phonemizer", "kenlm")
290
291
# `pip install ".[speech]"` is deprecated and `pip install ".[torch-speech]"` should be used instead
extras["speech"] = deps_list("torchaudio") + extras["audio"]
292
293
294
extras["torch-speech"] = deps_list("torchaudio") + extras["audio"]
extras["tf-speech"] = extras["audio"]
extras["flax-speech"] = extras["audio"]
295
extras["vision"] = deps_list("Pillow")
NielsRogge's avatar
NielsRogge committed
296
extras["timm"] = deps_list("timm")
NielsRogge's avatar
NielsRogge committed
297
extras["torch-vision"] = deps_list("torchvision") + extras["vision"]
298
extras["natten"] = deps_list("natten")
299
extras["codecarbon"] = deps_list("codecarbon")
300
extras["video"] = deps_list("decord", "av")
301

302
303
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
extras["testing"] = (
Philipp Schmid's avatar
Philipp Schmid committed
304
    deps_list(
305
306
307
308
309
310
        "pytest",
        "pytest-xdist",
        "timeout-decorator",
        "parameterized",
        "psutil",
        "datasets",
311
        "dill",
312
        "evaluate",
313
314
315
316
317
318
        "pytest-timeout",
        "black",
        "sacrebleu",
        "rouge-score",
        "nltk",
        "GitPython",
319
        "hf-doc-builder",
320
        "protobuf",  # Can be removed once we can unpin protobuf
321
        "sacremoses",
322
        "rjieba",
323
        "safetensors",
324
        "beautifulsoup4",
Philipp Schmid's avatar
Philipp Schmid committed
325
    )
326
327
328
    + extras["retrieval"]
    + extras["modelcreation"]
)
329

330
extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]
331

332
extras["quality"] = deps_list("black", "datasets", "isort", "ruff", "GitPython", "hf-doc-builder")
Stas Bekman's avatar
Stas Bekman committed
333

334
335
336
337
338
339
extras["all"] = (
    extras["tf"]
    + extras["torch"]
    + extras["flax"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
340
    + extras["torch-speech"]
341
    + extras["vision"]
Lysandre Debut's avatar
Lysandre Debut committed
342
    + extras["integrations"]
NielsRogge's avatar
NielsRogge committed
343
    + extras["timm"]
NielsRogge's avatar
NielsRogge committed
344
    + extras["torch-vision"]
345
    + extras["codecarbon"]
346
    + extras["accelerate"]
347
    + extras["video"]
348
)
Stas Bekman's avatar
Stas Bekman committed
349

350
# Might need to add doc-builder and some specific deps in the future
351
extras["docs_specific"] = ["hf-doc-builder"]
352

353
354
355
# "docs" needs "all" to resolve all the references
extras["docs"] = extras["all"] + extras["docs_specific"]

356
extras["dev-torch"] = (
357
358
    extras["testing"]
    + extras["torch"]
359
360
361
362
363
364
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["torch-speech"]
    + extras["vision"]
    + extras["integrations"]
    + extras["timm"]
NielsRogge's avatar
NielsRogge committed
365
    + extras["torch-vision"]
366
367
368
369
370
371
372
373
374
    + extras["codecarbon"]
    + extras["quality"]
    + extras["ja"]
    + extras["docs_specific"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnxruntime"]
)
extras["dev-tensorflow"] = (
375
376
377
378
379
380
381
382
383
384
385
    extras["testing"]
    + extras["tf"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["vision"]
    + extras["quality"]
    + extras["docs_specific"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnx"]
    + extras["tf-speech"]
386
)
387
388
389
390
391
extras["dev"] = (
    extras["all"]
    + extras["testing"]
    + extras["quality"]
    + extras["ja"]
392
    + extras["docs_specific"]
393
394
395
    + extras["sklearn"]
    + extras["modelcreation"]
)
Stas Bekman's avatar
Stas Bekman committed
396

397
398
extras["torchhub"] = deps_list(
    "filelock",
Sylvain Gugger's avatar
Sylvain Gugger committed
399
    "huggingface-hub",
400
401
402
403
404
405
406
407
408
409
410
    "importlib_metadata",
    "numpy",
    "packaging",
    "protobuf",
    "regex",
    "requests",
    "sentencepiece",
    "torch",
    "tokenizers",
    "tqdm",
)
411

412
413
# when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
install_requires = [
414
    deps["importlib_metadata"] + ";python_version<'3.8'",  # importlib_metadata for Python versions that don't have it
415
    deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
416
    deps["huggingface-hub"],
417
418
    deps["numpy"],
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
Vasudev Gupta's avatar
Vasudev Gupta committed
419
    deps["pyyaml"],  # used for the model cards metadata
420
421
    deps["regex"],  # for OpenAI GPT
    deps["requests"],  # for downloading models over HTTPS
Sylvain Gugger's avatar
Sylvain Gugger committed
422
    deps["tokenizers"],
423
    deps["tqdm"],  # progress bars in model download and training scripts
424
425
]

thomwolf's avatar
thomwolf committed
426
setup(
427
    name="transformers",
Sylvain Gugger's avatar
Sylvain Gugger committed
428
    version="4.28.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
Sylvain Gugger's avatar
Sylvain Gugger committed
429
430
431
    author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)",
    author_email="transformers@huggingface.co",
    description="State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow",
432
    long_description=open("README.md", "r", encoding="utf-8").read(),
thomwolf's avatar
thomwolf committed
433
    long_description_content_type="text/markdown",
Stas Bekman's avatar
Stas Bekman committed
434
435
    keywords="NLP vision speech deep learning transformer pytorch tensorflow jax BERT GPT-2 Wav2Vec2 ViT",
    license="Apache 2.0 License",
436
    url="https://github.com/huggingface/transformers",
Aymeric Augustin's avatar
Aymeric Augustin committed
437
    package_dir={"": "src"},
438
    packages=find_packages("src"),
439
440
    include_package_data=True,
    package_data={"transformers": ["*.cu", "*.cpp", "*.cuh", "*.h", "*.pyx"]},
Will Frey's avatar
Will Frey committed
441
    zip_safe=False,
442
    extras_require=extras,
443
    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
Sylvain Gugger's avatar
Sylvain Gugger committed
444
    python_requires=">=3.7.0",
445
    install_requires=install_requires,
thomwolf's avatar
thomwolf committed
446
    classifiers=[
Aymeric Augustin's avatar
Aymeric Augustin committed
447
448
449
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
450
451
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
Aymeric Augustin's avatar
Aymeric Augustin committed
452
        "Operating System :: OS Independent",
453
        "Programming Language :: Python :: 3",
Aymeric Augustin's avatar
Aymeric Augustin committed
454
        "Programming Language :: Python :: 3.7",
Anurag Kumar's avatar
Anurag Kumar committed
455
456
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
Stas Bekman's avatar
Stas Bekman committed
457
        "Programming Language :: Python :: 3.10",
458
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
thomwolf's avatar
thomwolf committed
459
    ],
460
    cmdclass={"deps_table_update": DepsTableUpdateCommand},
thomwolf's avatar
thomwolf committed
461
)