"vscode:/vscode.git/clone" did not exist on "cde0c750af2fae6848ed3ee8be381b4f1230ecd0"
Unverified Commit 330d83fd authored by Koichi Yasuoka's avatar Koichi Yasuoka Committed by GitHub
Browse files

Typo in "end_of_word_suffix" (#13477)

But does it really work?
parent 2a15e8cc
......@@ -666,7 +666,7 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
kwargs["continuing_subword_prefix"] = tokenizer_json["model"]["continuing_subword_prefix"]
if (
tokenizer_json["model"]["type"] == "BPE"
and "end_of_work_suffix" not in kwargs
and "end_of_word_suffix" not in kwargs
and tokenizer_json["model"]["end_of_word_suffix"] is not None
):
kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment