Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
7ae642b7
Commit
7ae642b7
authored
Sep 05, 2019
by
thomwolf
Browse files
update conversion scripts
parent
69bff899
Changes
9
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
20 additions
and
9 deletions
+20
-9
pytorch_transformers/__init__.py
pytorch_transformers/__init__.py
+11
-0
pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py
pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py
+1
-1
pytorch_transformers/convert_openai_checkpoint_to_pytorch.py
pytorch_transformers/convert_openai_checkpoint_to_pytorch.py
+1
-1
pytorch_transformers/convert_pytorch_checkpoint_to_tf.py
pytorch_transformers/convert_pytorch_checkpoint_to_tf.py
+1
-1
pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py
...rch_transformers/convert_roberta_checkpoint_to_pytorch.py
+2
-2
pytorch_transformers/convert_tf_checkpoint_to_pytorch.py
pytorch_transformers/convert_tf_checkpoint_to_pytorch.py
+1
-1
pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py
..._transformers/convert_transfo_xl_checkpoint_to_pytorch.py
+1
-1
pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py
pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py
+1
-1
pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py
pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py
+1
-1
No files found.
pytorch_transformers/__init__.py
View file @
7ae642b7
__version__
=
"1.2.0"
# Work around to update TensorFlow's absl.logging threshold which alters the
# default Python logging output behavior when present.
# see: https://github.com/abseil/abseil-py/issues/99
# and: https://github.com/tensorflow/tensorflow/issues/26691#issuecomment-500369493
try
:
import
absl.logging
absl
.
logging
.
set_verbosity
(
'info'
)
absl
.
logging
.
set_stderrthreshold
(
'info'
)
absl
.
logging
.
_warn_preinit_stderr
=
False
except
:
pass
# Tokenizer
from
.tokenization_utils
import
(
PreTrainedTokenizer
)
...
...
pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py
View file @
7ae642b7
...
...
@@ -21,7 +21,7 @@ from io import open
import
torch
from
pytorch_transformers
.modeling_gpt2
import
(
CONFIG_NAME
,
WEIGHTS_NAME
,
from
pytorch_transformers
import
(
CONFIG_NAME
,
WEIGHTS_NAME
,
GPT2Config
,
GPT2Model
,
load_tf_weights_in_gpt2
)
...
...
pytorch_transformers/convert_openai_checkpoint_to_pytorch.py
View file @
7ae642b7
...
...
@@ -21,7 +21,7 @@ from io import open
import
torch
from
pytorch_transformers
.modeling_openai
import
(
CONFIG_NAME
,
WEIGHTS_NAME
,
from
pytorch_transformers
import
(
CONFIG_NAME
,
WEIGHTS_NAME
,
OpenAIGPTConfig
,
OpenAIGPTModel
,
load_tf_weights_in_openai_gpt
)
...
...
pytorch_transformers/convert_pytorch_checkpoint_to_tf.py
View file @
7ae642b7
...
...
@@ -20,7 +20,7 @@ import argparse
import
torch
import
numpy
as
np
import
tensorflow
as
tf
from
pytorch_transformers
.modeling
import
BertModel
from
pytorch_transformers
import
BertModel
def
convert_pytorch_checkpoint_to_tf
(
model
:
BertModel
,
ckpt_dir
:
str
,
model_name
:
str
):
...
...
pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py
View file @
7ae642b7
...
...
@@ -23,12 +23,12 @@ import torch
from
fairseq.models.roberta
import
RobertaModel
as
FairseqRobertaModel
from
fairseq.modules
import
TransformerSentenceEncoderLayer
from
pytorch_transformers
.modeling_bert
import
(
BertConfig
,
BertEncoder
,
from
pytorch_transformers
import
(
BertConfig
,
BertEncoder
,
BertIntermediate
,
BertLayer
,
BertModel
,
BertOutput
,
BertSelfAttention
,
BertSelfOutput
)
from
pytorch_transformers
.modeling_roberta
import
(
RobertaEmbeddings
,
from
pytorch_transformers
import
(
RobertaEmbeddings
,
RobertaForMaskedLM
,
RobertaForSequenceClassification
,
RobertaModel
)
...
...
pytorch_transformers/convert_tf_checkpoint_to_pytorch.py
View file @
7ae642b7
...
...
@@ -21,7 +21,7 @@ from __future__ import print_function
import
argparse
import
torch
from
pytorch_transformers
.modeling_bert
import
BertConfig
,
BertForPreTraining
,
load_tf_weights_in_bert
from
pytorch_transformers
import
BertConfig
,
BertForPreTraining
,
load_tf_weights_in_bert
import
logging
logging
.
basicConfig
(
level
=
logging
.
INFO
)
...
...
pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py
View file @
7ae642b7
...
...
@@ -26,7 +26,7 @@ import torch
import
pytorch_transformers.tokenization_transfo_xl
as
data_utils
from
pytorch_transformers
import
CONFIG_NAME
,
WEIGHTS_NAME
from
pytorch_transformers
.modeling_transfo_xl
import
(
TransfoXLConfig
,
TransfoXLLMHeadModel
,
from
pytorch_transformers
import
(
TransfoXLConfig
,
TransfoXLLMHeadModel
,
load_tf_weights_in_transfo_xl
)
from
pytorch_transformers.tokenization_transfo_xl
import
(
CORPUS_NAME
,
VOCAB_FILES_NAMES
)
...
...
pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py
View file @
7ae642b7
...
...
@@ -23,7 +23,7 @@ from io import open
import
torch
import
numpy
from
pytorch_transformers
.modeling_utils
import
CONFIG_NAME
,
WEIGHTS_NAME
from
pytorch_transformers
import
CONFIG_NAME
,
WEIGHTS_NAME
from
pytorch_transformers.tokenization_xlm
import
VOCAB_FILES_NAMES
import
logging
...
...
pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py
View file @
7ae642b7
...
...
@@ -22,7 +22,7 @@ import os
import
argparse
import
torch
from
pytorch_transformers
.modeling_xlnet
import
(
CONFIG_NAME
,
WEIGHTS_NAME
,
from
pytorch_transformers
import
(
CONFIG_NAME
,
WEIGHTS_NAME
,
XLNetConfig
,
XLNetLMHeadModel
,
XLNetForQuestionAnswering
,
XLNetForSequenceClassification
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment