Skip to content

GitLab

Unverified Commit dcb0e614 authored by Patrick von Platen's avatar Patrick von Platen Committed by GitHub
Browse files

push (#11667)

parent 05a93067
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 0 deletions
src/transformers/models/auto/tokenization_auto.py
View file @ dcb0e614
...@@ -63,6 +63,7 @@ from .configuration_auto import ( ...@@ -63,6 +63,7 @@ from .configuration_auto import (
BertConfig, BertConfig,
BertGenerationConfig, BertGenerationConfig,
BigBirdConfig, BigBirdConfig,
BigBirdPegasusConfig,
BlenderbotConfig, BlenderbotConfig,
BlenderbotSmallConfig, BlenderbotSmallConfig,
CamembertConfig, CamembertConfig,
...@@ -275,6 +276,7 @@ TOKENIZER_MAPPING = OrderedDict( ...@@ -275,6 +276,7 @@ TOKENIZER_MAPPING = OrderedDict(
(Wav2Vec2Config, (Wav2Vec2CTCTokenizer, None)), (Wav2Vec2Config, (Wav2Vec2CTCTokenizer, None)),
(GPTNeoConfig, (GPT2Tokenizer, GPT2TokenizerFast)), (GPTNeoConfig, (GPT2Tokenizer, GPT2TokenizerFast)),
(LukeConfig, (LukeTokenizer, None)), (LukeConfig, (LukeTokenizer, None)),
(BigBirdPegasusConfig, (PegasusTokenizer, PegasusTokenizerFast)),
] ]
) )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment