Override get_vocab for fast tokenizer. (#4717)

f6d5046a · Funtowicz Morgan · GitHub · 88762a2f · f6d5046a
Unverified Commit f6d5046a authored Jun 02, 2020 by Funtowicz Morgan Committed by GitHub Jun 02, 2020
Show whitespace changes
Inline Side-by-side

Showing with 3 additions and 0 deletions

src/transformers/tokenization_utils.py src/transformers/tokenization_utils.py +3 -0

No files found.
--- a/src/transformers/tokenization_utils.py
+++ b/src/transformers/tokenization_utils.py
@@ -2368,6 +2368,9 @@ class PreTrainedTokenizerFast(PreTrainedTokenizer):
    def _convert_id_to_token(self, index: int) -> Optional[str]:
        return self._tokenizer.id_to_token(int(index))

+    def get_vocab(self):
+        return self._tokenizer.get_vocab(True)
+
    def convert_tokens_to_string(self, tokens: List[int], skip_special_tokens: bool = False) -> str:
        return self._tokenizer.decode(tokens, skip_special_tokens)