Unverified Commit 0cffa424 authored by Jabin Huang's avatar Jabin Huang Committed by GitHub
Browse files

Updata tokenization_auto.py (#6870)

Updata tokenization_auto.py to handle Inherited tokenizer
parent 03fb8e79
......@@ -244,7 +244,7 @@ class AutoTokenizer:
config = config.encoder
for config_class, (tokenizer_class_py, tokenizer_class_fast) in TOKENIZER_MAPPING.items():
if isinstance(config, config_class):
if type(config) is config_class:
if tokenizer_class_fast and use_fast:
return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment