Add CLIP tokenizer to AutoTokenizer (#13258)

68b69072 · Lysandre Debut · GitHub · 3bbe68f8 · 68b69072
Unverified Commit 68b69072 authored Aug 25, 2021 by Lysandre Debut Committed by GitHub Aug 25, 2021
Hide whitespace changes
Inline Side-by-side

Showing with 7 additions and 0 deletions

src/transformers/models/auto/tokenization_auto.py src/transformers/models/auto/tokenization_auto.py +7 -0

No files found.
--- a/src/transformers/models/auto/tokenization_auto.py
+++ b/src/transformers/models/auto/tokenization_auto.py
@@ -206,6 +206,13 @@ else:
                    "RemBertTokenizerFast" if is_tokenizers_available() else None,
                ),
            ),
+            (
+                "clip",
+                (
+                    "CLIPTokenizer",
+                    "CLIPTokenizerFast" if is_tokenizers_available() else None,
+                ),
+            ),
        ]
    )