Unverified Commit 5d4d62d0 authored by Matt's avatar Matt Committed by GitHub
Browse files

Correctly resolve trust_remote_code=None for AutoTokenizer (#28419)

* Correctly resolve trust_remote_code=None for AutoTokenizer

* Second attempt at a proper resolution
parent 55090585
...@@ -770,7 +770,13 @@ class AutoTokenizer: ...@@ -770,7 +770,13 @@ class AutoTokenizer:
tokenizer_auto_map = config.auto_map["AutoTokenizer"] tokenizer_auto_map = config.auto_map["AutoTokenizer"]
has_remote_code = tokenizer_auto_map is not None has_remote_code = tokenizer_auto_map is not None
has_local_code = config_tokenizer_class is not None or type(config) in TOKENIZER_MAPPING has_local_code = type(config) in TOKENIZER_MAPPING or (
config_tokenizer_class is not None
and (
tokenizer_class_from_name(config_tokenizer_class) is not None
or tokenizer_class_from_name(config_tokenizer_class + "Fast") is not None
)
)
trust_remote_code = resolve_trust_remote_code( trust_remote_code = resolve_trust_remote_code(
trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code
) )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment