Unverified Commit 280a811e authored by Li-Huai (Allan) Lin's avatar Li-Huai (Allan) Lin Committed by GitHub
Browse files

Use `AlbertConverter` for FNet instead of using FNet's own converter (#14365)

* Add normalizer to FNetConverter

* Style

* Directly use AlbertConverter
parent 55f49c5f
...@@ -526,18 +526,6 @@ class AlbertConverter(SpmConverter): ...@@ -526,18 +526,6 @@ class AlbertConverter(SpmConverter):
) )
class FNetConverter(SpmConverter):
def post_processor(self):
return processors.TemplateProcessing(
single="[CLS]:0 $A:0 [SEP]:0",
pair="[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1",
special_tokens=[
("[CLS]", self.original_tokenizer.convert_tokens_to_ids("[CLS]")),
("[SEP]", self.original_tokenizer.convert_tokens_to_ids("[SEP]")),
],
)
class BarthezConverter(SpmConverter): class BarthezConverter(SpmConverter):
def unk_id(self, proto): def unk_id(self, proto):
unk_id = 3 unk_id = 3
...@@ -938,7 +926,7 @@ SLOW_TO_FAST_CONVERTERS = { ...@@ -938,7 +926,7 @@ SLOW_TO_FAST_CONVERTERS = {
"DPRQuestionEncoderTokenizer": BertConverter, "DPRQuestionEncoderTokenizer": BertConverter,
"DPRContextEncoderTokenizer": BertConverter, "DPRContextEncoderTokenizer": BertConverter,
"ElectraTokenizer": BertConverter, "ElectraTokenizer": BertConverter,
"FNetTokenizer": FNetConverter, "FNetTokenizer": AlbertConverter,
"FunnelTokenizer": FunnelConverter, "FunnelTokenizer": FunnelConverter,
"GPT2Tokenizer": GPT2Converter, "GPT2Tokenizer": GPT2Converter,
"HerbertTokenizer": HerbertConverter, "HerbertTokenizer": HerbertConverter,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment