Unverified Commit 4e244b88 authored by Taras Tsugrii's avatar Taras Tsugrii Committed by GitHub
Browse files

Replace appends with list comprehension. (#23359)

It's more idiomatic and significantly more efficient because
1) it avoids repeated `append` call that Python has to resolve on each iteration
2) can preallocate the size of the final list avoiding resizing
parent 918a06e2
...@@ -249,10 +249,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase): ...@@ -249,10 +249,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
if isinstance(tokens, str): if isinstance(tokens, str):
return self._convert_token_to_id_with_added_voc(tokens) return self._convert_token_to_id_with_added_voc(tokens)
ids = [] return [self._convert_token_to_id_with_added_voc(token) for token in tokens]
for token in tokens:
ids.append(self._convert_token_to_id_with_added_voc(token))
return ids
def _convert_token_to_id_with_added_voc(self, token: str) -> int: def _convert_token_to_id_with_added_voc(self, token: str) -> int:
index = self._tokenizer.token_to_id(token) index = self._tokenizer.token_to_id(token)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment