Skip to content

GitLab

Unverified Commit a557836a authored by Thomas Wolf's avatar Thomas Wolf Committed by GitHub
Browse files

Merge pull request #2191 from huggingface/fix_sp_np 

Numpy compatibility for sentence piece
parents ceae85ad cb6d54bf
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 0 deletions
transformers/tokenization_utils.py
View file @ a557836a
...@@ -1227,6 +1227,7 @@ class PreTrainedTokenizer(object): ...@@ -1227,6 +1227,7 @@ class PreTrainedTokenizer(object):
return self._convert_id_to_token(ids) return self._convert_id_to_token(ids)
tokens = [] tokens = []
for index in ids: for index in ids:
index = int(index)
if skip_special_tokens and index in self.all_special_ids: if skip_special_tokens and index in self.all_special_ids:
continue continue
if index in self.added_tokens_decoder: if index in self.added_tokens_decoder:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment