Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "4b557132ce955d58fd84572c03e79f43bdc91450"
Unverified Commit 95792a94 authored by rmroczkowski's avatar rmroczkowski Committed by GitHub
Browse files

Herbert tokenizer auto load (#7968)

parent 4abb7ffc
Show whitespace changes
Inline Side-by-side
Showing with 4 additions and 0 deletions
src/transformers/tokenization_auto.py
View file @ 95792a94
...@@ -73,6 +73,7 @@ from .tokenization_flaubert import FlaubertTokenizer ...@@ -73,6 +73,7 @@ from .tokenization_flaubert import FlaubertTokenizer
from .tokenization_fsmt import FSMTTokenizer from .tokenization_fsmt import FSMTTokenizer
from .tokenization_funnel import FunnelTokenizer from .tokenization_funnel import FunnelTokenizer
from .tokenization_gpt2 import GPT2Tokenizer from .tokenization_gpt2 import GPT2Tokenizer
from .tokenization_herbert import HerbertTokenizer
from .tokenization_layoutlm import LayoutLMTokenizer from .tokenization_layoutlm import LayoutLMTokenizer
from .tokenization_longformer import LongformerTokenizer from .tokenization_longformer import LongformerTokenizer
from .tokenization_lxmert import LxmertTokenizer from .tokenization_lxmert import LxmertTokenizer
...@@ -123,6 +124,7 @@ if is_tokenizers_available(): ...@@ -123,6 +124,7 @@ if is_tokenizers_available():
from .tokenization_electra_fast import ElectraTokenizerFast from .tokenization_electra_fast import ElectraTokenizerFast
from .tokenization_funnel_fast import FunnelTokenizerFast from .tokenization_funnel_fast import FunnelTokenizerFast
from .tokenization_gpt2_fast import GPT2TokenizerFast from .tokenization_gpt2_fast import GPT2TokenizerFast
from .tokenization_herbert_fast import HerbertTokenizerFast
from .tokenization_layoutlm_fast import LayoutLMTokenizerFast from .tokenization_layoutlm_fast import LayoutLMTokenizerFast
from .tokenization_longformer_fast import LongformerTokenizerFast from .tokenization_longformer_fast import LongformerTokenizerFast
from .tokenization_lxmert_fast import LxmertTokenizerFast from .tokenization_lxmert_fast import LxmertTokenizerFast
...@@ -147,6 +149,7 @@ else: ...@@ -147,6 +149,7 @@ else:
ElectraTokenizerFast = None ElectraTokenizerFast = None
FunnelTokenizerFast = None FunnelTokenizerFast = None
GPT2TokenizerFast = None GPT2TokenizerFast = None
HerbertTokenizerFast = None
LayoutLMTokenizerFast = None LayoutLMTokenizerFast = None
LongformerTokenizerFast = None LongformerTokenizerFast = None
LxmertTokenizerFast = None LxmertTokenizerFast = None
...@@ -191,6 +194,7 @@ TOKENIZER_MAPPING = OrderedDict( ...@@ -191,6 +194,7 @@ TOKENIZER_MAPPING = OrderedDict(
(LayoutLMConfig, (LayoutLMTokenizer, LayoutLMTokenizerFast)), (LayoutLMConfig, (LayoutLMTokenizer, LayoutLMTokenizerFast)),
(DPRConfig, (DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast)), (DPRConfig, (DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast)),
(SqueezeBertConfig, (SqueezeBertTokenizer, SqueezeBertTokenizerFast)), (SqueezeBertConfig, (SqueezeBertTokenizer, SqueezeBertTokenizerFast)),
(BertConfig, (HerbertTokenizer, HerbertTokenizerFast)),
(BertConfig, (BertTokenizer, BertTokenizerFast)), (BertConfig, (BertTokenizer, BertTokenizerFast)),
(OpenAIGPTConfig, (OpenAIGPTTokenizer, OpenAIGPTTokenizerFast)), (OpenAIGPTConfig, (OpenAIGPTTokenizer, OpenAIGPTTokenizerFast)),
(GPT2Config, (GPT2Tokenizer, GPT2TokenizerFast)), (GPT2Config, (GPT2Tokenizer, GPT2TokenizerFast)),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment