Unverified Commit 1d568dfa authored by Arthur's avatar Arthur Committed by GitHub
Browse files

legacy to init the slow tokenizer when converting from slow was wrong (#30972)

parent 1432f641
......@@ -151,9 +151,6 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
self.legacy = legacy
if add_prefix_space is not None:
logger.warning_once(
"You set `add_prefix_space`. The tokenizer needs to be converted from the slow tokenizers"
)
kwargs["from_slow"] = True
super().__init__(
......@@ -166,6 +163,7 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
use_default_system_prompt=use_default_system_prompt,
legacy=legacy,
**kwargs,
)
self._add_bos_token = add_bos_token
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment