setup.py 16 KB
Newer Older
1
# Copyright 2021 The HuggingFace Team. All rights reserved.
Sylvain Gugger's avatar
Sylvain Gugger committed
2
3
4
5
6
7
8
9
10
11
12
13
14
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
"""
16
Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/main/setup.py
17
18
19

To create the package for pypi.

Sylvain Gugger's avatar
Sylvain Gugger committed
20
21
1. Create the release branch named: v<RELEASE>-release, for example v4.19-release. For a patch release checkout the
   current release branch.
22

23
24
   If releasing on a special branch, copy the updated README.md on the main branch for your the commit you will make
   for the post-release and run `make fix-copies` on the main branch as well.
25

Sylvain Gugger's avatar
Sylvain Gugger committed
26
27
2. Run `make pre-release` (or `make pre-patch` for a patch release) and commit these changes with the message:
   "Release: <VERSION>" and push.
28

Sylvain Gugger's avatar
Sylvain Gugger committed
29
30
3. Go back to the main branch and run `make post-release` then `make fix-copies`. Commit these changes with the
   message "v<NEXT_VERSION>.dev.0" and push to main.
Lysandre's avatar
Lysandre committed
31

Sylvain Gugger's avatar
Sylvain Gugger committed
32
# If you were just cutting the branch in preparation for a release, you can stop here for now.
33

Sylvain Gugger's avatar
Sylvain Gugger committed
34
4. Wait for the tests on the release branch to be completed and be green (otherwise revert and fix bugs)
35

Sylvain Gugger's avatar
Sylvain Gugger committed
36
5. On the release branch, add a tag in git to mark the release: "git tag v<VERSION> -m 'Adds tag v<VERSION> for pypi' "
Lysandre Debut's avatar
Lysandre Debut committed
37
   Push the tag to git: git push --tags origin v<RELEASE>-release
38

Sylvain Gugger's avatar
Sylvain Gugger committed
39
6. Build both the sources and the wheel. Do not change anything in setup.py between
40
41
   creating the wheel and the source distribution (obviously).

42
43
   Run `make build-release`. This will build the release and do some sanity checks for you. If this ends with an error
   message, you need to fix things before going further.
44

thomwolf's avatar
thomwolf committed
45
   You should now have a /dist directory with both .whl and .tar.gz source versions.
46

Sylvain Gugger's avatar
Sylvain Gugger committed
47
7. Check that everything looks correct by uploading the package to the pypi test server:
48

49
   twine upload dist/* -r testpypi
50
   (pypi suggest using twine as other methods upload files via plaintext.)
Lysandre's avatar
Lysandre committed
51
   You may have to specify the repository url, use the following command then:
52
   twine upload dist/* -r testpypi --repository-url=https://test.pypi.org/legacy/
53
54

   Check that you can install it in a virtualenv by running:
55
   pip install -i https://testpypi.python.org/pypi transformers
56

57
   Check you can run the following commands:
58
   python -c "from transformers import pipeline; classifier = pipeline('text-classification'); print(classifier('What a nice release'))"
59
   python -c "from transformers import *"
60
   python utils/check_build.py --check_lib
61

62
63
   If making a patch release, double check the bug you are patching is indeed resolved.

Sylvain Gugger's avatar
Sylvain Gugger committed
64
8. Upload the final version to actual pypi:
65
66
   twine upload dist/* -r pypi

Sylvain Gugger's avatar
Sylvain Gugger committed
67
9. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory.
68
"""
Aymeric Augustin's avatar
Aymeric Augustin committed
69

Stas Bekman's avatar
Stas Bekman committed
70
import os
71
import re
72
73
74
import shutil
from pathlib import Path

75
from setuptools import Command, find_packages, setup
thomwolf's avatar
thomwolf committed
76

77

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
# Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
stale_egg_info = Path(__file__).parent / "transformers.egg-info"
if stale_egg_info.exists():
    print(
        (
            "Warning: {} exists.\n\n"
            "If you recently updated transformers to 3.0 or later, this is expected,\n"
            "but it may prevent transformers from installing in editable mode.\n\n"
            "This directory is automatically generated by Python's packaging tools.\n"
            "I will remove it now.\n\n"
            "See https://github.com/pypa/pip/issues/5466 for details.\n"
        ).format(stale_egg_info)
    )
    shutil.rmtree(stale_egg_info)


94
95
96
97
# IMPORTANT:
# 1. all dependencies should be listed here with their version requirements if any
# 2. once modified, run: `make deps_table_update` to update src/transformers/dependency_versions_table.py
_deps = [
98
    "Pillow>=10.0.1,<=15.0",
99
    "accelerate>=0.21.0",
100
    "av==9.2.0",  # Latest version of PyAV (10.0.0) has issues with audio stream.
Sylvain Gugger's avatar
Sylvain Gugger committed
101
    "beautifulsoup4",
102
    "codecarbon==1.2.0",
Yih-Dar's avatar
Yih-Dar committed
103
    "cookiecutter==1.7.3",
104
    "dataclasses",
105
    "datasets!=2.5.0",
106
    "decord==0.6.0",
107
    "deepspeed>=0.9.3",
Lysandre Debut's avatar
Lysandre Debut committed
108
    "diffusers",
109
    "dill<0.3.5",
110
    "evaluate>=0.2.0",
111
112
113
    "faiss-cpu",
    "fastapi",
    "filelock",
114
    "flax>=0.4.1,<=0.7.0",
115
    "fsspec<2023.10.0",
116
    "ftfy",
117
    "fugashi>=1.0",
118
    "GitPython<3.1.19",
119
    "huggingface-hub>=0.19.3,<1.0",
120
    "importlib_metadata",
121
122
    "ipadic>=1.0.0,<2.0",
    "isort>=5.5.4",
123
124
    "jax>=0.4.1,<=0.4.13",
    "jaxlib>=0.4.1,<=0.4.13",
125
    "jieba",
126
    "kenlm",
Matt's avatar
Matt committed
127
    # Keras pin - this is to make sure Keras 3 doesn't destroy us. Remove or change when we have proper support.
128
    "keras>2.9,<2.16",
129
    "keras-nlp>=0.3.1",
Sylvain Gugger's avatar
Sylvain Gugger committed
130
    "librosa",
131
    "nltk",
Joao Gante's avatar
Joao Gante committed
132
    "natten>=0.14.6,<0.15.0",
Sylvain Gugger's avatar
Sylvain Gugger committed
133
    "numpy>=1.17",
Lysandre's avatar
Lysandre committed
134
    "onnxconverter-common",
135
136
    "onnxruntime-tools>=1.4.2",
    "onnxruntime>=1.4.0",
Lysandre Debut's avatar
Lysandre Debut committed
137
    "opencv-python",
Lysandre Debut's avatar
Lysandre Debut committed
138
    "optuna",
139
    "optax>=0.0.8,<=0.1.4",
140
    "packaging>=20.0",
141
    "parameterized",
142
    "phonemizer",
Yih-Dar's avatar
Yih-Dar committed
143
    "protobuf",
144
    "psutil",
145
    "pyyaml>=5.1",
Yih-Dar's avatar
Yih-Dar committed
146
    "pydantic",
147
    "pytest>=7.2.0,<8.0.0",
Lysandre Debut's avatar
Lysandre Debut committed
148
    "pytest-timeout",
149
    "pytest-xdist",
150
    "python>=3.8.0",
151
    "ray[tune]>=2.7.0",
152
153
    "regex!=2019.12.17",
    "requests",
Sylvain Gugger's avatar
Sylvain Gugger committed
154
    "rhoknp>=1.1.0,<1.3.1",
155
    "rjieba",
156
    "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
Yih-Dar's avatar
Yih-Dar committed
157
    "ruff==0.1.5",
Sylvain Gugger's avatar
Sylvain Gugger committed
158
    "sacrebleu>=1.4.12,<2.0.0",
159
    "sacremoses",
160
    "safetensors>=0.4.1",
161
    "sagemaker>=2.31.0",
162
    "scikit-learn",
Joao Gante's avatar
Joao Gante committed
163
    "scipy<1.13.0",  # SciPy >= 1.13.0 is not supported with the current jax pin (`jax>=0.4.1,<=0.4.13`)
164
    "sentencepiece>=0.1.91,!=0.1.92",
165
    "sigopt",
166
    "starlette",
Sylvain Gugger's avatar
Sylvain Gugger committed
167
168
    "sudachipy>=0.6.6",
    "sudachidict_core>=20220729",
169
    "tensorboard",
170
    # TensorFlow pin. When changing this value, update examples/tensorflow/_tests_requirements.txt accordingly
171
172
    "tensorflow-cpu>2.9,<2.16",
    "tensorflow>2.9,<2.16",
Matt's avatar
Matt committed
173
    "tensorflow-text<2.16",
174
    "tensorflow-probability<2.16",
175
    "tf2onnx",
176
    "timeout-decorator",
NielsRogge's avatar
NielsRogge committed
177
    "timm",
178
    "tokenizers>=0.19,<0.20",
Yih-Dar's avatar
Yih-Dar committed
179
180
181
    "torch",
    "torchaudio",
    "torchvision",
182
    "pyctcdecode>=0.4.0",
183
184
185
    "tqdm>=4.27",
    "unidic>=1.0.2",
    "unidic_lite>=1.0.7",
Sylvain Gugger's avatar
Sylvain Gugger committed
186
    "urllib3<2.0.0",
187
    "uvicorn",
188
    "pytest-rich",
189
]
190
191


192
193
194
195
196
197
# this is a lookup table with items like:
#
# tokenizers: "tokenizers==0.9.4"
# packaging: "packaging"
#
# some of the values are versioned whereas others aren't.
198
deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)}
199

200
201
202
203
204
205
206
207
208
209
210
211
212
# since we save this data in src/transformers/dependency_versions_table.py it can be easily accessed from
# anywhere. If you need to quickly access the data from this table in a shell, you can do so easily with:
#
# python -c 'import sys; from transformers.dependency_versions_table import deps; \
# print(" ".join([ deps[x] for x in sys.argv[1:]]))' tokenizers datasets
#
# Just pass the desired package names to that script as it's shown with 2 packages above.
#
# If transformers is not yet installed and the work is done from the cloned repo remember to add `PYTHONPATH=src` to the script above
#
# You can then feed this for example to `pip`:
#
# pip install -U $(python -c 'import sys; from transformers.dependency_versions_table import deps; \
Stas Bekman's avatar
Stas Bekman committed
213
# print(" ".join([deps[x] for x in sys.argv[1:]]))' tokenizers datasets)
214
215
#

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

def deps_list(*pkgs):
    return [deps[pkg] for pkg in pkgs]


class DepsTableUpdateCommand(Command):
    """
    A custom distutils command that updates the dependency table.
    usage: python setup.py deps_table_update
    """

    description = "build runtime dependency table"
    user_options = [
        # format: (long option, short option, description).
        ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
    ]

    def initialize_options(self):
        pass

    def finalize_options(self):
        pass

    def run(self):
        entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
        content = [
            "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
            "# 1. modify the `_deps` dict in setup.py",
            "# 2. run `make deps_table_update``",
            "deps = {",
            entries,
            "}",
248
            "",
249
250
251
        ]
        target = "src/transformers/dependency_versions_table.py"
        print(f"updating {target}")
Julien Plu's avatar
Julien Plu committed
252
        with open(target, "w", encoding="utf-8", newline="\n") as f:
253
254
            f.write("\n".join(content))

255

256
257
extras = {}

Hao Wang's avatar
Hao Wang committed
258
extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic", "sudachipy", "sudachidict_core", "rhoknp")
259
260
extras["sklearn"] = deps_list("scikit-learn")

261
extras["tf"] = deps_list("tensorflow", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp")
262
extras["tf-cpu"] = deps_list("keras", "tensorflow-cpu", "onnxconverter-common", "tf2onnx", "tensorflow-text", "keras-nlp", "tensorflow-probability")
263

264
extras["torch"] = deps_list("torch", "accelerate")
265
extras["accelerate"] = deps_list("accelerate")
Stas Bekman's avatar
Stas Bekman committed
266
267

if os.name == "nt":  # windows
268
269
    extras["retrieval"] = deps_list("datasets")  # faiss is not supported on windows
    extras["flax"] = []  # jax is not supported on windows
270
else:
271
    extras["retrieval"] = deps_list("faiss-cpu", "datasets")
Joao Gante's avatar
Joao Gante committed
272
    extras["flax"] = deps_list("jax", "jaxlib", "flax", "optax", "scipy")
273

274
extras["tokenizers"] = deps_list("tokenizers")
275
extras["ftfy"] = deps_list("ftfy")
276
extras["onnxruntime"] = deps_list("onnxruntime", "onnxruntime-tools")
277
extras["onnx"] = deps_list("onnxconverter-common", "tf2onnx") + extras["onnxruntime"]
278
extras["modelcreation"] = deps_list("cookiecutter")
279

Philipp Schmid's avatar
Philipp Schmid committed
280
extras["sagemaker"] = deps_list("sagemaker")
281
extras["deepspeed"] = deps_list("deepspeed") + extras["accelerate"]
Lysandre Debut's avatar
Lysandre Debut committed
282
extras["optuna"] = deps_list("optuna")
Richard Liaw's avatar
Richard Liaw committed
283
extras["ray"] = deps_list("ray[tune]")
284
extras["sigopt"] = deps_list("sigopt")
Lysandre Debut's avatar
Lysandre Debut committed
285

286
extras["integrations"] = extras["optuna"] + extras["ray"] + extras["sigopt"]
Philipp Schmid's avatar
Philipp Schmid committed
287

288
extras["serving"] = deps_list("pydantic", "uvicorn", "fastapi", "starlette")
Sanchit Gandhi's avatar
Sanchit Gandhi committed
289
extras["audio"] = deps_list("librosa", "pyctcdecode", "phonemizer", "kenlm")
290
291
# `pip install ".[speech]"` is deprecated and `pip install ".[torch-speech]"` should be used instead
extras["speech"] = deps_list("torchaudio") + extras["audio"]
292
293
294
extras["torch-speech"] = deps_list("torchaudio") + extras["audio"]
extras["tf-speech"] = extras["audio"]
extras["flax-speech"] = extras["audio"]
295
extras["vision"] = deps_list("Pillow")
NielsRogge's avatar
NielsRogge committed
296
extras["timm"] = deps_list("timm")
NielsRogge's avatar
NielsRogge committed
297
extras["torch-vision"] = deps_list("torchvision") + extras["vision"]
298
extras["natten"] = deps_list("natten")
299
extras["codecarbon"] = deps_list("codecarbon")
300
extras["video"] = deps_list("decord", "av")
301

302
303
extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
extras["testing"] = (
Philipp Schmid's avatar
Philipp Schmid committed
304
    deps_list(
305
        "pytest",
306
        "pytest-rich",
307
308
309
310
311
        "pytest-xdist",
        "timeout-decorator",
        "parameterized",
        "psutil",
        "datasets",
312
        "dill",
313
        "evaluate",
314
        "pytest-timeout",
315
        "ruff",
316
317
318
319
        "sacrebleu",
        "rouge-score",
        "nltk",
        "GitPython",
320
        "sacremoses",
321
        "rjieba",
322
        "beautifulsoup4",
323
        "tensorboard",
324
        "pydantic",
325
        "sentencepiece",
Philipp Schmid's avatar
Philipp Schmid committed
326
    )
327
328
329
    + extras["retrieval"]
    + extras["modelcreation"]
)
330

331
extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]
332

333
extras["quality"] = deps_list("datasets", "isort", "ruff", "GitPython", "urllib3")
Stas Bekman's avatar
Stas Bekman committed
334

335
336
337
338
339
340
extras["all"] = (
    extras["tf"]
    + extras["torch"]
    + extras["flax"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
341
    + extras["torch-speech"]
342
    + extras["vision"]
Lysandre Debut's avatar
Lysandre Debut committed
343
    + extras["integrations"]
NielsRogge's avatar
NielsRogge committed
344
    + extras["timm"]
NielsRogge's avatar
NielsRogge committed
345
    + extras["torch-vision"]
346
    + extras["codecarbon"]
347
    + extras["accelerate"]
348
    + extras["video"]
349
)
Stas Bekman's avatar
Stas Bekman committed
350

351

352
extras["dev-torch"] = (
353
354
    extras["testing"]
    + extras["torch"]
355
356
357
358
359
360
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["torch-speech"]
    + extras["vision"]
    + extras["integrations"]
    + extras["timm"]
NielsRogge's avatar
NielsRogge committed
361
    + extras["torch-vision"]
362
363
364
365
366
367
368
369
    + extras["codecarbon"]
    + extras["quality"]
    + extras["ja"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnxruntime"]
)
extras["dev-tensorflow"] = (
370
371
372
373
374
375
376
377
378
379
    extras["testing"]
    + extras["tf"]
    + extras["sentencepiece"]
    + extras["tokenizers"]
    + extras["vision"]
    + extras["quality"]
    + extras["sklearn"]
    + extras["modelcreation"]
    + extras["onnx"]
    + extras["tf-speech"]
380
)
381
382
383
384
385
386
387
388
extras["dev"] = (
    extras["all"]
    + extras["testing"]
    + extras["quality"]
    + extras["ja"]
    + extras["sklearn"]
    + extras["modelcreation"]
)
Stas Bekman's avatar
Stas Bekman committed
389

390
391
extras["torchhub"] = deps_list(
    "filelock",
Sylvain Gugger's avatar
Sylvain Gugger committed
392
    "huggingface-hub",
393
394
395
396
397
398
399
400
401
402
403
    "importlib_metadata",
    "numpy",
    "packaging",
    "protobuf",
    "regex",
    "requests",
    "sentencepiece",
    "torch",
    "tokenizers",
    "tqdm",
)
404

Lysandre Debut's avatar
Lysandre Debut committed
405
extras["agents"] = deps_list(
Sylvain Gugger's avatar
Style  
Sylvain Gugger committed
406
    "diffusers", "accelerate", "datasets", "torch", "sentencepiece", "opencv-python", "Pillow"
Lysandre Debut's avatar
Lysandre Debut committed
407
408
)

409
410
# when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
install_requires = [
411
    deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
412
    deps["huggingface-hub"],
413
414
    deps["numpy"],
    deps["packaging"],  # utilities from PyPA to e.g., compare versions
Vasudev Gupta's avatar
Vasudev Gupta committed
415
    deps["pyyaml"],  # used for the model cards metadata
416
417
    deps["regex"],  # for OpenAI GPT
    deps["requests"],  # for downloading models over HTTPS
Sylvain Gugger's avatar
Sylvain Gugger committed
418
    deps["tokenizers"],
419
    deps["safetensors"],
420
    deps["tqdm"],  # progress bars in model download and training scripts
421
422
]

thomwolf's avatar
thomwolf committed
423
setup(
424
    name="transformers",
Lysandre's avatar
Lysandre committed
425
    version="4.41.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
426
427
428
429
430
431
432
433
434
435
436
    author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)",
    author_email="transformers@huggingface.co",
    description="State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow",
    long_description=open("README.md", "r", encoding="utf-8").read(),
    long_description_content_type="text/markdown",
    keywords="NLP vision speech deep learning transformer pytorch tensorflow jax BERT GPT-2 Wav2Vec2 ViT",
    license="Apache 2.0 License",
    url="https://github.com/huggingface/transformers",
    package_dir={"": "src"},
    packages=find_packages("src"),
    include_package_data=True,
437
    package_data={"": ["**/*.cu", "**/*.cpp", "**/*.cuh", "**/*.h", "**/*.pyx"]},
438
    zip_safe=False,
439
    extras_require=extras,
440
    entry_points={"console_scripts": ["transformers-cli=transformers.commands.transformers_cli:main"]},
441
    python_requires=">=3.8.0",
442
    install_requires=list(install_requires),
443
444
445
446
447
448
449
450
451
452
453
454
455
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Education",
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: Apache Software License",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Programming Language :: Python :: 3.10",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ],
456
    cmdclass={"deps_table_update": DepsTableUpdateCommand},
thomwolf's avatar
thomwolf committed
457
)
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472

extras["tests_torch"] = deps_list()
extras["tests_tf"] = deps_list()
extras["tests_flax"] = deps_list()
extras["tests_torch_and_tf"] = deps_list()
extras["tests_torch_and_flax"] = deps_list()
extras["tests_hub"] = deps_list()
extras["tests_pipelines_torch"] = deps_list()
extras["tests_pipelines_tf"] = deps_list()
extras["tests_onnx"] = deps_list()
extras["tests_examples_torch"] = deps_list()
extras["tests_examples_tf"] = deps_list()
extras["tests_custom_tokenizers"] = deps_list()
extras["tests_exotic_models"] = deps_list()
extras["consistency"] = deps_list()