Unverified Commit 4a433e32 authored by Arthur's avatar Arthur Committed by GitHub
Browse files

Add-warning-tokenizer (#20826)

* add fast not use warning

* update
parent 76d02fea
......@@ -576,9 +576,14 @@ class AutoTokenizer:
tokenizer_class_name, tokenizer_fast_class_name = tokenizer_class_tuple
if use_fast and tokenizer_fast_class_name is not None:
tokenizer_class = tokenizer_class_from_name(tokenizer_fast_class_name)
if use_fast:
if tokenizer_fast_class_name is not None:
tokenizer_class = tokenizer_class_from_name(tokenizer_fast_class_name)
else:
logger.warning(
"`use_fast` is set to `True` but the tokenizer class does not have a fast version. "
" Falling back to the slow version."
)
if tokenizer_class is None:
tokenizer_class = tokenizer_class_from_name(tokenizer_class_name)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment