Add RemBert to AutoTokenizer (#13224)

2772d3e7 · Lysandre Debut · GitHub · f1bb6f08 · 2772d3e7
Unverified Commit 2772d3e7 authored Aug 23, 2021 by Lysandre Debut Committed by GitHub Aug 23, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 7 additions and 0 deletions

src/transformers/models/auto/tokenization_auto.py src/transformers/models/auto/tokenization_auto.py +7 -0

No files found.
--- a/src/transformers/models/auto/tokenization_auto.py
+++ b/src/transformers/models/auto/tokenization_auto.py
@@ -199,6 +199,13 @@ else:
                    "MBart50TokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
+            (
+                "rembert",
+                (
+                    "RemBertTokenizer" if is_sentencepiece_available() else None,
+                    "RemBertTokenizerFast" if is_tokenizers_available() else None,
+                ),
+            ),
        ]
    )