Unverified Commit b026b5ca authored by Mayank Mishra's avatar Mayank Mishra Committed by GitHub
Browse files

Fix tokenizer export for LLamaTokenizerFast (#27222)

* fix tokenizer

* fix tokenizer
parent cc3e4781
......@@ -149,6 +149,8 @@ class CodeLlamaTokenizerFast(PreTrainedTokenizerFast):
unk_token=unk_token,
bos_token=bos_token,
eos_token=eos_token,
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
prefix_token=prefix_token,
middle_token=middle_token,
suffix_token=suffix_token,
......
......@@ -128,6 +128,8 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
unk_token=unk_token,
bos_token=bos_token,
eos_token=eos_token,
add_bos_token=add_bos_token,
add_eos_token=add_eos_token,
use_default_system_prompt=use_default_system_prompt,
**kwargs,
)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment