Update tokenizers requirement (#10077)

f285e4c3 · Anthony MOI · GitHub · ddaafd78 · f285e4c3 · f285e4c3
Unverified Commit f285e4c3 authored Feb 08, 2021 by Anthony MOI Committed by GitHub Feb 08, 2021
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 2 deletions

setup.py setup.py +1 -1

src/transformers/dependency_versions_table.py src/transformers/dependency_versions_table.py +1 -1

No files found.
--- a/setup.py
+++ b/setup.py
@@ -132,7 +132,7 @@ _deps = [
    "tensorflow-cpu>=2.3",
    "tensorflow>=2.3",
    "timeout-decorator",
-    "tokenizers==0.10.1rc1",
+    "tokenizers>=0.10.1,<0.11",
    "torch>=1.0",
    "tqdm>=4.27",
    "unidic>=1.0.2",

--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -45,7 +45,7 @@ deps = {
    "tensorflow-cpu": "tensorflow-cpu>=2.3",
    "tensorflow": "tensorflow>=2.3",
    "timeout-decorator": "timeout-decorator",
-    "tokenizers": "tokenizers==0.10.1rc1",
+    "tokenizers": "tokenizers>=0.10.1,<0.11",
    "torch": "torch>=1.0",
    "tqdm": "tqdm>=4.27",
    "unidic": "unidic>=1.0.2",