Skip to content

GitLab

Commit 2d5d86e0 authored by thomwolf's avatar thomwolf
Browse files

fix #2031

parent af077b15
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 1 deletion
transformers/tokenization_albert.py
View file @ 2d5d86e0
...@@ -66,7 +66,7 @@ class AlbertTokenizer(PreTrainedTokenizer): ...@@ -66,7 +66,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
def __init__(self, vocab_file, def __init__(self, vocab_file,
do_lower_case=True, remove_space=True, keep_accents=False, do_lower_case=True, remove_space=True, keep_accents=False,
bos_token="[CLS]", eos_token="[SEP]", unk_token="<unk>", sep_token="[SEP]", bos_token="[CLS]", eos_token="[SEP]", unk_token="<unk>", sep_token="[SEP]",
pad_token="<pad>", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): pad_token="<pad>", cls_token="[CLS]", mask_token="[MASK]", **kwargs):
super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token,
unk_token=unk_token, sep_token=sep_token, unk_token=unk_token, sep_token=sep_token,
pad_token=pad_token, cls_token=cls_token, pad_token=pad_token, cls_token=cls_token,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment