"git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "a55809241fee13f4ef4020bc6e087cf3acb6d485"
Unverified Commit 0cffa424 authored by Jabin Huang's avatar Jabin Huang Committed by GitHub
Browse files

Updata tokenization_auto.py (#6870)

Updata tokenization_auto.py to handle Inherited tokenizer
parent 03fb8e79
...@@ -244,7 +244,7 @@ class AutoTokenizer: ...@@ -244,7 +244,7 @@ class AutoTokenizer:
config = config.encoder config = config.encoder
for config_class, (tokenizer_class_py, tokenizer_class_fast) in TOKENIZER_MAPPING.items(): for config_class, (tokenizer_class_py, tokenizer_class_fast) in TOKENIZER_MAPPING.items():
if isinstance(config, config_class): if type(config) is config_class:
if tokenizer_class_fast and use_fast: if tokenizer_class_fast and use_fast:
return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
else: else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment