Declare __len__ method in PreTrainedTokenizerBase (#21210)

Declare len method in PreTrainedTokenizerBase (#21210)
7419d807 · Thomas Wang · GitHub · ef530175 · 7419d807
Unverified Commit 7419d807 authored Jan 20, 2023 by Thomas Wang Committed by GitHub Jan 20, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 0 deletions

src/transformers/tokenization_utils_base.py src/transformers/tokenization_utils_base.py +3 -0

No files found.
--- a/src/transformers/tokenization_utils_base.py
+++ b/src/transformers/tokenization_utils_base.py
@@ -1578,6 +1578,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
            f" special_tokens={self.special_tokens_map_extended})"
        )

+    def __len__(self) -> int:
+        raise NotImplementedError()
+
    def get_vocab(self) -> Dict[str, int]:
        """
        Returns the vocabulary as a dictionary of token to index.