Skip to content

GitLab

"examples/trials/git@developer.sourcefind.cn:OpenDAS/nni.git" did not exist on "fb3c596bcede58610226c913bd31923c3539b4c2"
Unverified Commit 2f0f9038 authored by Patrick von Platen's avatar Patrick von Platen Committed by GitHub
Browse files

[Barthez Tokenizer] Fix saving (#15815)

parent ca57b450
Hide whitespace changes
Inline Side-by-side
Showing with 5 additions and 1 deletion
src/transformers/models/barthez/tokenization_barthez.py
View file @ 2f0f9038
...@@ -288,7 +288,11 @@ class BarthezTokenizer(PreTrainedTokenizer): ...@@ -288,7 +288,11 @@ class BarthezTokenizer(PreTrainedTokenizer):
save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
) )
if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
copyfile(self.vocab_file, out_vocab_file) copyfile(self.vocab_file, out_vocab_file)
elif not os.path.isfile(self.vocab_file):
with open(out_vocab_file, "wb") as fi:
content_spiece_model = self.sp_model.serialized_model_proto()
fi.write(content_spiece_model)
return (out_vocab_file,) return (out_vocab_file,)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment