Skip to content

GitLab

Commit 50e6daf8 authored by thomwolf's avatar thomwolf
Browse files

fix Roberta tokenizer __init__

parent 0517e7a1
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 1 deletion
pytorch_transformers/tokenization_roberta.py
View file @ 50e6daf8
...@@ -72,7 +72,8 @@ class RobertaTokenizer(GPT2Tokenizer): ...@@ -72,7 +72,8 @@ class RobertaTokenizer(GPT2Tokenizer):
def __init__(self, vocab_file, merges_file, errors='replace', bos_token="<s>", eos_token="</s>", sep_token="</s>", def __init__(self, vocab_file, merges_file, errors='replace', bos_token="<s>", eos_token="</s>", sep_token="</s>",
cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>', **kwargs): cls_token="<s>", unk_token="<unk>", pad_token='<pad>', mask_token='<mask>', **kwargs):
super(RobertaTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, super(RobertaTokenizer, self).__init__(vocab_file=vocab_file, merges_file=merges_file, errors=errors,
bos_token=bos_token, eos_token=eos_token, unk_token=unk_token,
sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, sep_token=sep_token, cls_token=cls_token, pad_token=pad_token,
mask_token=mask_token, **kwargs) mask_token=mask_token, **kwargs)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment