"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "a98f6a1da012ca7847e4dceb3ffcedfd75a77b08"
Unverified Commit a39ebbf8 authored by Arthur's avatar Arthur Committed by GitHub
Browse files

[`CodeLlama`] Fix CI (#25890)

* Fix coellama

* style
parent 3b39b906
...@@ -151,7 +151,6 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast): ...@@ -151,7 +151,6 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast):
self.update_post_processor() self.update_post_processor()
self.vocab_file = vocab_file self.vocab_file = vocab_file
self.can_save_slow_tokenizer = False if not self.vocab_file else True
self._prefix_token = prefix_token self._prefix_token = prefix_token
self._middle_token = middle_token self._middle_token = middle_token
...@@ -159,6 +158,10 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast): ...@@ -159,6 +158,10 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast):
self._eot_token = eot_token self._eot_token = eot_token
self.fill_token = fill_token self.fill_token = fill_token
@property
def can_save_slow_tokenizer(self) -> bool:
return os.path.isfile(self.vocab_file) if self.vocab_file else False
def update_post_processor(self): def update_post_processor(self):
""" """
Updates the underlying post processor with the current `bos_token` and `eos_token`. Updates the underlying post processor with the current `bos_token` and `eos_token`.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment