Unverified Commit 18170ee5 authored by Chen Xin's avatar Chen Xin Committed by GitHub
Browse files

fix Tokenizer load error when the path of the being-converted model is not writable (#669)

parent 013000d1
# Copyright (c) OpenMMLab. All rights reserved.
import json
import os
import os.path as osp
from typing import Optional, Sequence, Union
......@@ -125,7 +126,8 @@ class HuggingFaceTokenizer:
# save tokenizer.json to reuse
if not osp.exists(backend_tokenizer_file) and model_file_exists:
if hasattr(self.model, 'backend_tokenizer'):
self.model.backend_tokenizer.save(backend_tokenizer_file)
if os.access(model_dir, os.W_OK):
self.model.backend_tokenizer.save(backend_tokenizer_file)
if self.model.eos_token_id is None:
generation_config_file = osp.join(model_dir,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment