Add XLMProphetNetTokenizer to tokenization auto (#8245)

9a23af4a · Lysandre Debut · GitHub · 5b178f3c · 9a23af4a
Unverified Commit 9a23af4a authored Nov 02, 2020 by Lysandre Debut Committed by GitHub Nov 02, 2020
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 0 deletions

src/transformers/tokenization_auto.py src/transformers/tokenization_auto.py +1 -0

No files found.
--- a/src/transformers/tokenization_auto.py
+++ b/src/transformers/tokenization_auto.py
@@ -113,6 +113,7 @@ else:
    T5Tokenizer = None
    XLMRobertaTokenizer = None
    XLNetTokenizer = None
+    XLMProphetNetTokenizer = None

 if is_tokenizers_available():
    from .tokenization_albert_fast import AlbertTokenizerFast