Unverified Commit 1cd7c6f1 authored by YouJiacheng's avatar YouJiacheng Committed by GitHub
Browse files

Fix from_pretrained kwargs passing (#18387)

Fix #18385
I don't know whether `use_auth_token`, `cache_dir` and `local_files_only` should be passed to `(cls.slow_tokenizer_class)._from_pretrained`, but I guess it should.
parent 96b5d7db
...@@ -1809,6 +1809,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): ...@@ -1809,6 +1809,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
*init_inputs, *init_inputs,
use_auth_token=use_auth_token, use_auth_token=use_auth_token,
cache_dir=cache_dir, cache_dir=cache_dir,
local_files_only=local_files_only,
**kwargs, **kwargs,
) )
...@@ -1821,6 +1822,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): ...@@ -1821,6 +1822,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
*init_inputs, *init_inputs,
use_auth_token=None, use_auth_token=None,
cache_dir=None, cache_dir=None,
local_files_only=False,
**kwargs **kwargs
): ):
# We instantiate fast tokenizers based on a slow tokenizer if we don't have access to the tokenizer.json # We instantiate fast tokenizers based on a slow tokenizer if we don't have access to the tokenizer.json
...@@ -1833,6 +1835,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): ...@@ -1833,6 +1835,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
pretrained_model_name_or_path, pretrained_model_name_or_path,
copy.deepcopy(init_configuration), copy.deepcopy(init_configuration),
*init_inputs, *init_inputs,
use_auth_token=use_auth_token,
cache_dir=cache_dir,
local_files_only=local_files_only,
**(copy.deepcopy(kwargs)), **(copy.deepcopy(kwargs)),
) )
else: else:
...@@ -1864,6 +1869,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin): ...@@ -1864,6 +1869,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
pretrained_model_name_or_path, pretrained_model_name_or_path,
use_auth_token=use_auth_token, use_auth_token=use_auth_token,
cache_dir=cache_dir, cache_dir=cache_dir,
local_files_only=local_files_only,
) )
config_tokenizer_class = config.tokenizer_class config_tokenizer_class = config.tokenizer_class
except (OSError, ValueError, KeyError): except (OSError, ValueError, KeyError):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment