Commit 07e21307 authored by epwalsh's avatar epwalsh
Browse files

fix adding special tokens

parent caf1d116
...@@ -676,7 +676,7 @@ class PreTrainedTokenizer(object): ...@@ -676,7 +676,7 @@ class PreTrainedTokenizer(object):
all_toks = [] all_toks = []
set_attr = self.special_tokens_map set_attr = self.special_tokens_map
for attr_value in set_attr.values(): for attr_value in set_attr.values():
all_toks = all_toks + (attr_value if isinstance(attr_value, (list, tuple)) else [attr_value]) all_toks = all_toks + (list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value])
all_toks = list(set(all_toks)) all_toks = list(set(all_toks))
return all_toks return all_toks
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment