Unverified Commit b026b5ca authored by Mayank Mishra's avatar Mayank Mishra Committed by GitHub
Browse files

Fix tokenizer export for LLamaTokenizerFast (#27222)

* fix tokenizer

* fix tokenizer
parent cc3e4781
...@@ -149,6 +149,8 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast): ...@@ -149,6 +149,8 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast):
unk_token=unk_token, unk_token=unk_token,
bos_token=bos_token, bos_token=bos_token,
eos_token=eos_token, eos_token=eos_token,
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
prefix_token=prefix_token, prefix_token=prefix_token,
middle_token=middle_token, middle_token=middle_token,
suffix_token=suffix_token, suffix_token=suffix_token,
......
...@@ -128,6 +128,8 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast): ...@@ -128,6 +128,8 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
unk_token=unk_token, unk_token=unk_token,
bos_token=bos_token, bos_token=bos_token,
eos_token=eos_token, eos_token=eos_token,
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
use_default_system_prompt=use_default_system_prompt, use_default_system_prompt=use_default_system_prompt,
**kwargs, **kwargs,
) )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment