Unverified Commit f4f4e6b2 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Use existing functionality for #13251 (#13333)

parent d5064953
......@@ -37,6 +37,7 @@ from .configuration_auto import (
CONFIG_MAPPING_NAMES,
AutoConfig,
config_class_to_model_type,
model_type_to_module_name,
replace_list_option_in_docstrings,
)
......@@ -230,10 +231,9 @@ def tokenizer_class_from_name(class_name: str):
if class_name in tokenizers:
break
if module_name == "openai-gpt":
module_name = "openai"
module_name = model_type_to_module_name(module_name)
module = importlib.import_module(f".{module_name.replace('-', '_')}", "transformers.models")
module = importlib.import_module(f".{module_name}", "transformers.models")
return getattr(module, class_name)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment