Unverified Commit a947386c authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Better warning when loading a tokenizer with AutoTokenizer w/o SnetencePiece (#8881)

parent 9c18f156
...@@ -369,7 +369,13 @@ class AutoTokenizer: ...@@ -369,7 +369,13 @@ class AutoTokenizer:
if tokenizer_class_fast and (use_fast or tokenizer_class_py is None): if tokenizer_class_fast and (use_fast or tokenizer_class_py is None):
return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
else: else:
return tokenizer_class_py.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) if tokenizer_class_py is not None:
return tokenizer_class_py.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
else:
raise ValueError(
"This tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed "
"in order to use this tokenizer."
)
raise ValueError( raise ValueError(
"Unrecognized configuration class {} to build an AutoTokenizer.\n" "Unrecognized configuration class {} to build an AutoTokenizer.\n"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment