Skip to content

GitLab

Unverified Commit 0a6b9048 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Init pickle (#12567) 

* Try to pickle transformers

* Deal with special objs better

* Make picklable
parent b29c3945
Show whitespace changes
Inline Side-by-side
Showing with 40 additions and 312 deletions
src/transformers/models/cpm/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule
from ...file_utils import _LazyModule
_import_structure = {
......@@ -30,19 +30,6 @@ if TYPE_CHECKING:
from .tokenization_cpm import CpmTokenizer
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/ctrl/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tf_available, is_torch_available
from ...file_utils import _LazyModule, is_tf_available, is_torch_available
_import_structure = {
......@@ -68,19 +68,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/deberta/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tokenizers_available, is_torch_available
from ...file_utils import _LazyModule, is_tokenizers_available, is_torch_available
_import_structure = {
......@@ -60,19 +60,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/deberta_v2/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_torch_available
from ...file_utils import _LazyModule, is_torch_available
_import_structure = {
......@@ -54,19 +54,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/deit/__init__.py
View file @ 0a6b9048
......@@ -17,7 +17,7 @@
# limitations under the License.
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_torch_available, is_vision_available
from ...file_utils import _LazyModule, is_torch_available, is_vision_available
_import_structure = {
......@@ -54,19 +54,6 @@ if TYPE_CHECKING:
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/detr/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_timm_available, is_vision_available
from ...file_utils import _LazyModule, is_timm_available, is_vision_available
_import_structure = {
......@@ -54,19 +54,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/distilbert/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tf_available, is_tokenizers_available, is_torch_available
from ...file_utils import _LazyModule, is_tf_available, is_tokenizers_available, is_torch_available
_import_structure = {
......@@ -88,19 +88,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/dpr/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tf_available, is_tokenizers_available, is_torch_available
from ...file_utils import _LazyModule, is_tf_available, is_tokenizers_available, is_torch_available
_import_structure = {
......@@ -109,19 +109,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/electra/__init__.py
View file @ 0a6b9048
......@@ -18,13 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import (
_BaseLazyModule,
is_flax_available,
is_tf_available,
is_tokenizers_available,
is_torch_available,
)
from ...file_utils import _LazyModule, is_flax_available, is_tf_available, is_tokenizers_available, is_torch_available
_import_structure = {
......@@ -122,19 +116,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/encoder_decoder/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_torch_available
from ...file_utils import _LazyModule, is_torch_available
_import_structure = {
......@@ -36,19 +36,6 @@ if TYPE_CHECKING:
from .modeling_encoder_decoder import EncoderDecoderModel
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/flaubert/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tf_available, is_torch_available
from ...file_utils import _LazyModule, is_tf_available, is_torch_available
_import_structure = {
......@@ -80,19 +80,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/fsmt/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_torch_available
from ...file_utils import _LazyModule, is_torch_available
_import_structure = {
......@@ -38,19 +38,6 @@ if TYPE_CHECKING:
from .modeling_fsmt import FSMTForConditionalGeneration, FSMTModel, PretrainedFSMTModel
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/funnel/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tf_available, is_tokenizers_available, is_torch_available
from ...file_utils import _LazyModule, is_tf_available, is_tokenizers_available, is_torch_available
_import_structure = {
......@@ -97,19 +97,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/gpt2/__init__.py
View file @ 0a6b9048
......@@ -18,13 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import (
_BaseLazyModule,
is_flax_available,
is_tf_available,
is_tokenizers_available,
is_torch_available,
)
from ...file_utils import _LazyModule, is_flax_available, is_tf_available, is_tokenizers_available, is_torch_available
_import_structure = {
......@@ -93,19 +87,6 @@ if TYPE_CHECKING:
from .modeling_flax_gpt2 import FlaxGPT2LMHeadModel, FlaxGPT2Model, FlaxGPT2PreTrainedModel
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/gpt_neo/__init__.py
View file @ 0a6b9048
......@@ -17,7 +17,7 @@
# limitations under the License.
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_flax_available, is_torch_available
from ...file_utils import _LazyModule, is_flax_available, is_torch_available
_import_structure = {
......@@ -60,19 +60,6 @@ if TYPE_CHECKING:
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/herbert/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tokenizers_available
from ...file_utils import _LazyModule, is_tokenizers_available
_import_structure = {
......@@ -36,19 +36,6 @@ if TYPE_CHECKING:
from .tokenization_herbert_fast import HerbertTokenizerFast
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/hubert/__init__.py
View file @ 0a6b9048
......@@ -17,7 +17,7 @@
# limitations under the License.
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_torch_available
from ...file_utils import _LazyModule, is_torch_available
_import_structure = {
......@@ -46,19 +46,6 @@ if TYPE_CHECKING:
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/ibert/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_torch_available
from ...file_utils import _LazyModule, is_torch_available
_import_structure = {
......@@ -53,19 +53,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/layoutlm/__init__.py
View file @ 0a6b9048
......@@ -18,7 +18,7 @@
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tf_available, is_tokenizers_available, is_torch_available
from ...file_utils import _LazyModule, is_tf_available, is_tokenizers_available, is_torch_available
from .configuration_layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig
from .tokenization_layoutlm import LayoutLMTokenizer
......@@ -81,19 +81,6 @@ if TYPE_CHECKING:
)
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
src/transformers/models/led/__init__.py
View file @ 0a6b9048
......@@ -17,7 +17,7 @@
# limitations under the License.
from typing import TYPE_CHECKING
from ...file_utils import _BaseLazyModule, is_tf_available, is_tokenizers_available, is_torch_available
from ...file_utils import _LazyModule, is_tf_available, is_tokenizers_available, is_torch_available
_import_structure = {
......@@ -64,19 +64,6 @@ if TYPE_CHECKING:
from .modeling_tf_led import TFLEDForConditionalGeneration, TFLEDModel, TFLEDPreTrainedModel
else:
import importlib
import os
import sys
class _LazyModule(_BaseLazyModule):
"""
Module class that surfaces all objects but only performs associated imports when the objects are requested.
"""
__file__ = globals()["__file__"]
__path__ = [os.path.dirname(__file__)]
def _get_module(self, module_name: str):
return importlib.import_module("." + module_name, self.__name__)
sys.modules[__name__] = _LazyModule(__name__, _import_structure)
sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment