Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
df56c843
Unverified
Commit
df56c843
authored
Nov 17, 2022
by
Yih-Dar
Committed by
GitHub
Nov 17, 2022
Browse files
[Tiny model creation] deal with `ImageProcessor` (#20298)
Co-authored-by:
ydshieh
<
ydshieh@users.noreply.github.com
>
parent
4bb07647
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
5 additions
and
2 deletions
+5
-2
utils/create_dummy_models.py
utils/create_dummy_models.py
+5
-2
No files found.
utils/create_dummy_models.py
View file @
df56c843
...
@@ -41,6 +41,7 @@ from transformers import (
...
@@ -41,6 +41,7 @@ from transformers import (
)
)
from
transformers.feature_extraction_utils
import
FeatureExtractionMixin
from
transformers.feature_extraction_utils
import
FeatureExtractionMixin
from
transformers.file_utils
import
is_tf_available
,
is_torch_available
from
transformers.file_utils
import
is_tf_available
,
is_torch_available
from
transformers.image_processing_utils
import
BaseImageProcessor
from
transformers.models.auto.configuration_auto
import
AutoConfig
,
model_type_to_module_name
from
transformers.models.auto.configuration_auto
import
AutoConfig
,
model_type_to_module_name
from
transformers.models.fsmt
import
configuration_fsmt
from
transformers.models.fsmt
import
configuration_fsmt
from
transformers.processing_utils
import
ProcessorMixin
,
transformers_module
from
transformers.processing_utils
import
ProcessorMixin
,
transformers_module
...
@@ -383,6 +384,8 @@ def convert_processors(processors, tiny_config, output_folder, result):
...
@@ -383,6 +384,8 @@ def convert_processors(processors, tiny_config, output_folder, result):
for
processor
in
processors
:
for
processor
in
processors
:
if
isinstance
(
processor
,
PreTrainedTokenizerBase
):
if
isinstance
(
processor
,
PreTrainedTokenizerBase
):
tokenizers
.
append
(
processor
)
tokenizers
.
append
(
processor
)
elif
isinstance
(
processor
,
BaseImageProcessor
):
feature_extractors
.
append
(
processor
)
elif
isinstance
(
processor
,
FeatureExtractionMixin
):
elif
isinstance
(
processor
,
FeatureExtractionMixin
):
feature_extractors
.
append
(
processor
)
feature_extractors
.
append
(
processor
)
elif
isinstance
(
processor
,
ProcessorMixin
):
elif
isinstance
(
processor
,
ProcessorMixin
):
...
@@ -848,8 +851,8 @@ def build(config_class, models_to_create, output_dir):
...
@@ -848,8 +851,8 @@ def build(config_class, models_to_create, output_dir):
error
=
f
"Failed to convert the processors:
{
e
}
"
error
=
f
"Failed to convert the processors:
{
e
}
"
result
[
"warnings"
].
append
(
error
)
result
[
"warnings"
].
append
(
error
)
if
len
(
result
[
"
processor
"
]
)
==
0
:
if
len
(
processor
s
)
==
0
:
error
=
f
"No processor
could be converted
for
{
config_class
.
__name__
}
."
error
=
f
"No processor
is returned by `convert_processors`
for
{
config_class
.
__name__
}
."
fill_result_with_error
(
result
,
error
,
models_to_create
)
fill_result_with_error
(
result
,
error
,
models_to_create
)
logger
.
error
(
result
[
"error"
])
logger
.
error
(
result
[
"error"
])
return
result
return
result
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment