Commit cb6d54bf authored by Lysandre's avatar Lysandre
Browse files

Numpy compatibility for sentence piece

convert to int earlier
parent ceae85ad
......@@ -1227,6 +1227,7 @@ class PreTrainedTokenizer(object):
return self._convert_id_to_token(ids)
tokens = []
for index in ids:
index = int(index)
if skip_special_tokens and index in self.all_special_ids:
continue
if index in self.added_tokens_decoder:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment