Unverified Commit 18170ee5 authored by Chen Xin's avatar Chen Xin Committed by GitHub
Browse files

fix Tokenizer load error when the path of the being-converted model is not writable (#669)

parent 013000d1
# Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) OpenMMLab. All rights reserved.
import json import json
import os
import os.path as osp import os.path as osp
from typing import Optional, Sequence, Union from typing import Optional, Sequence, Union
...@@ -125,6 +126,7 @@ class HuggingFaceTokenizer: ...@@ -125,6 +126,7 @@ class HuggingFaceTokenizer:
# save tokenizer.json to reuse # save tokenizer.json to reuse
if not osp.exists(backend_tokenizer_file) and model_file_exists: if not osp.exists(backend_tokenizer_file) and model_file_exists:
if hasattr(self.model, 'backend_tokenizer'): if hasattr(self.model, 'backend_tokenizer'):
if os.access(model_dir, os.W_OK):
self.model.backend_tokenizer.save(backend_tokenizer_file) self.model.backend_tokenizer.save(backend_tokenizer_file)
if self.model.eos_token_id is None: if self.model.eos_token_id is None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment