Skip to content

GitLab

Unverified Commit f4f4e6b2 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Use existing functionality for #13251 (#13333)

parent d5064953
Hide whitespace changes
Inline Side-by-side
Showing with 3 additions and 3 deletions
src/transformers/models/auto/tokenization_auto.py
View file @ f4f4e6b2
...@@ -37,6 +37,7 @@ from .configuration_auto import ( ...@@ -37,6 +37,7 @@ from .configuration_auto import (
CONFIG_MAPPING_NAMES, CONFIG_MAPPING_NAMES,
AutoConfig, AutoConfig,
config_class_to_model_type, config_class_to_model_type,
model_type_to_module_name,
replace_list_option_in_docstrings, replace_list_option_in_docstrings,
) )
...@@ -230,10 +231,9 @@ def tokenizer_class_from_name(class_name: str): ...@@ -230,10 +231,9 @@ def tokenizer_class_from_name(class_name: str):
if class_name in tokenizers: if class_name in tokenizers:
break break
if module_name == "openai-gpt": module_name = model_type_to_module_name(module_name)
module_name = "openai"
module = importlib.import_module(f".{module_name.replace('-', '_')}", "transformers.models") module = importlib.import_module(f".{module_name}", "transformers.models")
return getattr(module, class_name) return getattr(module, class_name)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment