Unverified Commit 330d83fd authored by Koichi Yasuoka's avatar Koichi Yasuoka Committed by GitHub
Browse files

Typo in "end_of_word_suffix" (#13477)

But does it really work?
parent 2a15e8cc
...@@ -666,7 +666,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase): ...@@ -666,7 +666,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
kwargs["continuing_subword_prefix"] = tokenizer_json["model"]["continuing_subword_prefix"] kwargs["continuing_subword_prefix"] = tokenizer_json["model"]["continuing_subword_prefix"]
if ( if (
tokenizer_json["model"]["type"] == "BPE" tokenizer_json["model"]["type"] == "BPE"
and "end_of_work_suffix" not in kwargs and "end_of_word_suffix" not in kwargs
and tokenizer_json["model"]["end_of_word_suffix"] is not None and tokenizer_json["model"]["end_of_word_suffix"] is not None
): ):
kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"] kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment