[`LlamaTokenizer`] make unk_token_length a property (#25689)

make unk_token_length a property

[`LlamaTokenizer`] make unk_token_length a property (#25689)
make unk_token_length a property
6e6da5e4 · Arthur · GitHub · b85b8806 · 6e6da5e4
Unverified Commit 6e6da5e4 authored Aug 24, 2023 by Arthur Committed by GitHub Aug 24, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 1 deletion

src/transformers/models/llama/tokenization_llama.py src/transformers/models/llama/tokenization_llama.py +4 -1

No files found.
--- a/src/transformers/models/llama/tokenization_llama.py
+++ b/src/transformers/models/llama/tokenization_llama.py
@@ -154,7 +154,10 @@ class LlamaTokenizer(PreTrainedTokenizer):
        self.use_default_system_prompt = use_default_system_prompt

        self.sp_model = self.get_spm_processor()
-        self.unk_token_length = len(self.sp_model.encode(str(self.unk_token)))
+
+    @property
+    def unk_token_length(self):
+        return len(self.sp_model.encode(str(self.unk_token)))

    # Copied from transformers.models.t5.tokenization_t5.T5Tokenizer.get_spm_processor
    def get_spm_processor(self):