[BIG] pytorch-transformers => transformers

pytorch_transformers/tests/modeling_tf_transfo_xl_test.py → transformers/tests/modeling_tf_transfo_xl_test.py

@@ -24,11 +24,11 @@ import pytest
 from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor)
 from .configuration_common_test import ConfigTester
 
-from pytorch_transformers import TransfoXLConfig, is_tf_available
+from transformers import TransfoXLConfig, is_tf_available
 
 if is_tf_available():
     import tensorflow as tf
-    from pytorch_transformers.modeling_tf_transfo_xl import (TFTransfoXLModel,
+    from transformers.modeling_tf_transfo_xl import (TFTransfoXLModel,
                                                              TFTransfoXLLMHeadModel,
                                                              TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP)
 else:
@@ -206,7 +206,7 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester):
 
     @pytest.mark.slow
     def test_model_from_pretrained(self):
-        cache_dir = "/tmp/pytorch_transformers_test/"
+        cache_dir = "/tmp/transformers_test/"
         for model_name in list(TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir)
             shutil.rmtree(cache_dir)

pytorch_transformers/tests/modeling_tf_xlm_test.py → transformers/tests/modeling_tf_xlm_test.py

@@ -20,11 +20,11 @@ import unittest
 import shutil
 import pytest
 
-from pytorch_transformers import is_tf_available
+from transformers import is_tf_available
 
 if is_tf_available():
     import tensorflow as tf
-    from pytorch_transformers import (XLMConfig, TFXLMModel,
+    from transformers import (XLMConfig, TFXLMModel,
                                       TFXLMWithLMHeadModel,
                                       TFXLMForSequenceClassification,
                                       TFXLMForQuestionAnsweringSimple,
@@ -253,7 +253,7 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester):
 
     @pytest.mark.slow
     def test_model_from_pretrained(self):
-        cache_dir = "/tmp/pytorch_transformers_test/"
+        cache_dir = "/tmp/transformers_test/"
         for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir)
             shutil.rmtree(cache_dir)

pytorch_transformers/tests/modeling_tf_xlnet_test.py → transformers/tests/modeling_tf_xlnet_test.py

@@ -23,12 +23,12 @@ import random
 import shutil
 import pytest
 
-from pytorch_transformers import XLNetConfig, is_tf_available
+from transformers import XLNetConfig, is_tf_available
 
 if is_tf_available():
     import tensorflow as tf
 
-    from pytorch_transformers.modeling_tf_xlnet import (TFXLNetModel, TFXLNetLMHeadModel,
+    from transformers.modeling_tf_xlnet import (TFXLNetModel, TFXLNetLMHeadModel,
                                                         TFXLNetForSequenceClassification,
                                                         TFXLNetForQuestionAnsweringSimple,
                                                         TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP)
@@ -291,7 +291,7 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester):
 
     @pytest.mark.slow
     def test_model_from_pretrained(self):
-        cache_dir = "/tmp/pytorch_transformers_test/"
+        cache_dir = "/tmp/transformers_test/"
         for model_name in list(TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = TFXLNetModel.from_pretrained(model_name, cache_dir=cache_dir)
             shutil.rmtree(cache_dir)

pytorch_transformers/tests/modeling_transfo_xl_test.py → transformers/tests/modeling_transfo_xl_test.py

@@ -21,12 +21,12 @@ import random
 import shutil
 import pytest
 
-from pytorch_transformers import is_torch_available
+from transformers import is_torch_available
 
 if is_torch_available():
     import torch
-    from pytorch_transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel)
-    from pytorch_transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP
+    from transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel)
+    from transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
 
@@ -206,7 +206,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester):
 
     @pytest.mark.slow
     def test_model_from_pretrained(self):
-        cache_dir = "/tmp/pytorch_transformers_test/"
+        cache_dir = "/tmp/transformers_test/"
         for model_name in list(TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = TransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir)
             shutil.rmtree(cache_dir)

pytorch_transformers/tests/modeling_xlm_test.py → transformers/tests/modeling_xlm_test.py

@@ -20,12 +20,12 @@ import unittest
 import shutil
 import pytest
 
-from pytorch_transformers import is_torch_available
+from transformers import is_torch_available
 
 if is_torch_available():
-    from pytorch_transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering,
+    from transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering,
                                       XLMForSequenceClassification, XLMForQuestionAnsweringSimple)
-    from pytorch_transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP
+    from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
 
@@ -314,7 +314,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester):
 
     @pytest.mark.slow
     def test_model_from_pretrained(self):
-        cache_dir = "/tmp/pytorch_transformers_test/"
+        cache_dir = "/tmp/transformers_test/"
         for model_name in list(XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir)
             shutil.rmtree(cache_dir)

pytorch_transformers/tests/modeling_xlnet_test.py → transformers/tests/modeling_xlnet_test.py

@@ -23,13 +23,13 @@ import random
 import shutil
 import pytest
 
-from pytorch_transformers import is_torch_available
+from transformers import is_torch_available
 
 if is_torch_available():
     import torch
 
-    from pytorch_transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering)
-    from pytorch_transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP
+    from transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering)
+    from transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP
 else:
     pytestmark = pytest.mark.skip("Require Torch")
 
@@ -317,7 +317,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester):
 
     @pytest.mark.slow
     def test_model_from_pretrained(self):
-        cache_dir = "/tmp/pytorch_transformers_test/"
+        cache_dir = "/tmp/transformers_test/"
         for model_name in list(XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]:
             model = XLNetModel.from_pretrained(model_name, cache_dir=cache_dir)
             shutil.rmtree(cache_dir)

pytorch_transformers/tests/optimization_test.py → transformers/tests/optimization_test.py

@@ -20,12 +20,12 @@ import unittest
 import os
 import pytest
 
-from pytorch_transformers import is_torch_available
+from transformers import is_torch_available
 
 if is_torch_available():
     import torch
 
-    from pytorch_transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule,
+    from transformers import (AdamW, ConstantLRSchedule, WarmupConstantSchedule,
                                     WarmupCosineSchedule, WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule)
 else:
     pytestmark = pytest.mark.skip("Require Torch")

pytorch_transformers/tests/tokenization_auto_test.py → transformers/tests/tokenization_auto_test.py

@@ -21,8 +21,8 @@ import shutil
 import pytest
 import logging
 
-from pytorch_transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer
-from pytorch_transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP
+from transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer
+from transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP
 
 
 class AutoTokenizerTest(unittest.TestCase):

pytorch_transformers/tests/tokenization_bert_test.py → transformers/tests/tokenization_bert_test.py

@@ -18,7 +18,7 @@ import os
 import unittest
 from io import open
 
-from pytorch_transformers.tokenization_bert import (BasicTokenizer,
+from transformers.tokenization_bert import (BasicTokenizer,
                                                     BertTokenizer,
                                                     WordpieceTokenizer,
                                                     _is_control, _is_punctuation,

pytorch_transformers/tests/tokenization_distilbert_test.py → transformers/tests/tokenization_distilbert_test.py

@@ -18,7 +18,7 @@ import os
 import unittest
 from io import open
 
-from pytorch_transformers.tokenization_distilbert import (DistilBertTokenizer)
+from transformers.tokenization_distilbert import (DistilBertTokenizer)
 
 from .tokenization_tests_commons import CommonTestCases
 from .tokenization_bert_test import BertTokenizationTest

pytorch_transformers/tests/tokenization_gpt2_test.py → transformers/tests/tokenization_gpt2_test.py

@@ -19,7 +19,7 @@ import unittest
 import json
 from io import open
 
-from pytorch_transformers.tokenization_gpt2 import GPT2Tokenizer, VOCAB_FILES_NAMES
+from transformers.tokenization_gpt2 import GPT2Tokenizer, VOCAB_FILES_NAMES
 
 from .tokenization_tests_commons import CommonTestCases
 

pytorch_transformers/tests/tokenization_openai_test.py → transformers/tests/tokenization_openai_test.py

@@ -18,7 +18,7 @@ import os
 import unittest
 import json
 
-from pytorch_transformers.tokenization_openai import OpenAIGPTTokenizer, VOCAB_FILES_NAMES
+from transformers.tokenization_openai import OpenAIGPTTokenizer, VOCAB_FILES_NAMES
 
 from .tokenization_tests_commons import CommonTestCases
 

pytorch_transformers/tests/tokenization_roberta_test.py → transformers/tests/tokenization_roberta_test.py

@@ -19,7 +19,7 @@ import json
 import unittest
 from io import open
 
-from pytorch_transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES
+from transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES
 from .tokenization_tests_commons import CommonTestCases
 
 

pytorch_transformers/tests/tokenization_transfo_xl_test.py → transformers/tests/tokenization_transfo_xl_test.py

@@ -19,11 +19,11 @@ import unittest
 import pytest
 from io import open
 
-from pytorch_transformers import is_torch_available
+from transformers import is_torch_available
 
 if is_torch_available():
     import torch
-    from pytorch_transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES
+    from transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES
 else:
     pytestmark = pytest.mark.skip("Require Torch")  # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save
 

pytorch_transformers/tests/tokenization_utils_test.py → transformers/tests/tokenization_utils_test.py

@@ -19,8 +19,8 @@ from __future__ import print_function
 import unittest
 import six
 
-from pytorch_transformers import PreTrainedTokenizer
-from pytorch_transformers.tokenization_gpt2 import GPT2Tokenizer
+from transformers import PreTrainedTokenizer
+from transformers.tokenization_gpt2 import GPT2Tokenizer
 
 class TokenizerUtilsTest(unittest.TestCase):
     def check_tokenizer_from_pretrained(self, tokenizer_class):

pytorch_transformers/tests/tokenization_xlm_test.py → transformers/tests/tokenization_xlm_test.py

@@ -18,7 +18,7 @@ import os
 import unittest
 import json
 
-from pytorch_transformers.tokenization_xlm import XLMTokenizer, VOCAB_FILES_NAMES
+from transformers.tokenization_xlm import XLMTokenizer, VOCAB_FILES_NAMES
 
 from .tokenization_tests_commons import CommonTestCases
 

pytorch_transformers/tests/tokenization_xlnet_test.py → transformers/tests/tokenization_xlnet_test.py

@@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import os
 import unittest
 
-from pytorch_transformers.tokenization_xlnet import (XLNetTokenizer, SPIECE_UNDERLINE)
+from transformers.tokenization_xlnet import (XLNetTokenizer, SPIECE_UNDERLINE)
 
 from .tokenization_tests_commons import CommonTestCases
 

pytorch_transformers/tokenization_auto.py → transformers/tokenization_auto.py

@@ -30,7 +30,7 @@ from .tokenization_distilbert import DistilBertTokenizer
 logger = logging.getLogger(__name__)
 
 class AutoTokenizer(object):
-    r""":class:`~pytorch_transformers.AutoTokenizer` is a generic tokenizer class
+    r""":class:`~transformers.AutoTokenizer` is a generic tokenizer class
         that will be instantiated as one of the tokenizer classes of the library
         when created with the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)`
         class method.
@@ -75,7 +75,7 @@ class AutoTokenizer(object):
             pretrained_model_name_or_path: either:
 
                 - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``.
-                - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~pytorch_transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``.
+                - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``.
                 - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``.
 
             cache_dir: (`optional`) string:
@@ -90,7 +90,7 @@ class AutoTokenizer(object):
 
             inputs: (`optional`) positional arguments: will be passed to the Tokenizer ``__init__`` method.
 
-            kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~pytorch_transformers.PreTrainedTokenizer` for details.
+            kwargs: (`optional`) keyword arguments: will be passed to the Tokenizer ``__init__`` method. Can be used to set special tokens like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. See parameters in the doc string of :class:`~transformers.PreTrainedTokenizer` for details.
 
         Examples::
 

pytorch_transformers/tokenization_bert.py → transformers/tokenization_bert.py

@@ -103,7 +103,7 @@ def whitespace_tokenize(text):
 class BertTokenizer(PreTrainedTokenizer):
     r"""
     Constructs a BertTokenizer.
-    :class:`~pytorch_transformers.BertTokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece
+    :class:`~transformers.BertTokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece
 
     Args:
         vocab_file: Path to a one-wordpiece-per-line vocabulary file