Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
7fc628d9
Commit
7fc628d9
authored
Jan 08, 2020
by
Julien Plu
Committed by
Lysandre Debut
Jan 29, 2020
Browse files
Apply style
parent
64ca8556
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
14 additions
and
9 deletions
+14
-9
src/transformers/__init__.py
src/transformers/__init__.py
+12
-1
src/transformers/convert_pytorch_checkpoint_to_tf2.py
src/transformers/convert_pytorch_checkpoint_to_tf2.py
+0
-8
templates/adding_a_new_example_script/run_xxx.py
templates/adding_a_new_example_script/run_xxx.py
+1
-0
templates/adding_a_new_example_script/utils_xxx.py
templates/adding_a_new_example_script/utils_xxx.py
+1
-0
No files found.
src/transformers/__init__.py
View file @
7fc628d9
...
@@ -31,6 +31,7 @@ from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, Open
...
@@ -31,6 +31,7 @@ from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, Open
from
.configuration_roberta
import
ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
,
RobertaConfig
from
.configuration_roberta
import
ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP
,
RobertaConfig
from
.configuration_t5
import
T5_PRETRAINED_CONFIG_ARCHIVE_MAP
,
T5Config
from
.configuration_t5
import
T5_PRETRAINED_CONFIG_ARCHIVE_MAP
,
T5Config
from
.configuration_transfo_xl
import
TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP
,
TransfoXLConfig
from
.configuration_transfo_xl
import
TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP
,
TransfoXLConfig
# Configurations
# Configurations
from
.configuration_utils
import
PretrainedConfig
from
.configuration_utils
import
PretrainedConfig
from
.configuration_xlm
import
XLM_PRETRAINED_CONFIG_ARCHIVE_MAP
,
XLMConfig
from
.configuration_xlm
import
XLM_PRETRAINED_CONFIG_ARCHIVE_MAP
,
XLMConfig
...
@@ -55,6 +56,7 @@ from .data import (
...
@@ -55,6 +56,7 @@ from .data import (
xnli_processors
,
xnli_processors
,
xnli_tasks_num_labels
,
xnli_tasks_num_labels
,
)
)
# Files and general utilities
# Files and general utilities
from
.file_utils
import
(
from
.file_utils
import
(
CONFIG_NAME
,
CONFIG_NAME
,
...
@@ -71,8 +73,10 @@ from .file_utils import (
...
@@ -71,8 +73,10 @@ from .file_utils import (
is_tf_available
,
is_tf_available
,
is_torch_available
,
is_torch_available
,
)
)
# Model Cards
# Model Cards
from
.modelcard
import
ModelCard
from
.modelcard
import
ModelCard
# TF 2.0 <=> PyTorch conversion utilities
# TF 2.0 <=> PyTorch conversion utilities
from
.modeling_tf_pytorch_utils
import
(
from
.modeling_tf_pytorch_utils
import
(
convert_tf_weight_name_to_pt_weight_name
,
convert_tf_weight_name_to_pt_weight_name
,
...
@@ -83,6 +87,7 @@ from .modeling_tf_pytorch_utils import (
...
@@ -83,6 +87,7 @@ from .modeling_tf_pytorch_utils import (
load_tf2_model_in_pytorch_model
,
load_tf2_model_in_pytorch_model
,
load_tf2_weights_in_pytorch_model
,
load_tf2_weights_in_pytorch_model
,
)
)
# Pipelines
# Pipelines
from
.pipelines
import
(
from
.pipelines
import
(
CsvPipelineDataFormat
,
CsvPipelineDataFormat
,
...
@@ -108,6 +113,7 @@ from .tokenization_openai import OpenAIGPTTokenizer
...
@@ -108,6 +113,7 @@ from .tokenization_openai import OpenAIGPTTokenizer
from
.tokenization_roberta
import
RobertaTokenizer
from
.tokenization_roberta
import
RobertaTokenizer
from
.tokenization_t5
import
T5Tokenizer
from
.tokenization_t5
import
T5Tokenizer
from
.tokenization_transfo_xl
import
TransfoXLCorpus
,
TransfoXLTokenizer
from
.tokenization_transfo_xl
import
TransfoXLCorpus
,
TransfoXLTokenizer
# Tokenizers
# Tokenizers
from
.tokenization_utils
import
PreTrainedTokenizer
from
.tokenization_utils
import
PreTrainedTokenizer
from
.tokenization_xlm
import
XLMTokenizer
from
.tokenization_xlm
import
XLMTokenizer
...
@@ -377,7 +383,12 @@ if is_tf_available():
...
@@ -377,7 +383,12 @@ if is_tf_available():
TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
,
TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP
,
)
)
from
.modeling_tf_t5
import
TFT5PreTrainedModel
,
TFT5Model
,
TFT5WithLMHeadModel
,
TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP
from
.modeling_tf_t5
import
(
TFT5PreTrainedModel
,
TFT5Model
,
TFT5WithLMHeadModel
,
TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP
,
)
# Optimization
# Optimization
from
.optimization_tf
import
WarmUp
,
create_optimizer
,
AdamWeightDecay
,
GradientAccumulator
from
.optimization_tf
import
WarmUp
,
create_optimizer
,
AdamWeightDecay
,
GradientAccumulator
...
...
src/transformers/convert_pytorch_checkpoint_to_tf2.py
View file @
7fc628d9
...
@@ -54,7 +54,6 @@ from transformers import (
...
@@ -54,7 +54,6 @@ from transformers import (
TFT5WithLMHeadModel
,
TFT5WithLMHeadModel
,
TFTransfoXLLMHeadModel
,
TFTransfoXLLMHeadModel
,
TFXLMRobertaForMaskedLM
,
TFXLMRobertaForMaskedLM
,
TFXLMRobertaForSequenceClassification
,
TFXLMWithLMHeadModel
,
TFXLMWithLMHeadModel
,
TFXLNetLMHeadModel
,
TFXLNetLMHeadModel
,
TransfoXLConfig
,
TransfoXLConfig
,
...
@@ -266,13 +265,6 @@ MODEL_CLASSES = {
...
@@ -266,13 +265,6 @@ MODEL_CLASSES = {
DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP
,
DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP
,
DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
,
DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
,
),
),
"distilbert-base-uncased-distilled-squad"
:
(
DistilBertConfig
,
TFDistilBertForQuestionAnswering
,
DistilBertForQuestionAnswering
,
DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP
,
DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
,
),
"ctrl"
:
(
"ctrl"
:
(
CTRLConfig
,
CTRLConfig
,
TFCTRLLMHeadModel
,
TFCTRLLMHeadModel
,
...
...
templates/adding_a_new_example_script/run_xxx.py
View file @
7fc628d9
...
@@ -52,6 +52,7 @@ from utils_squad import (
...
@@ -52,6 +52,7 @@ from utils_squad import (
write_predictions
,
write_predictions
,
write_predictions_extended
,
write_predictions_extended
,
)
)
# The follwing import is the official SQuAD evaluation script (2.0).
# The follwing import is the official SQuAD evaluation script (2.0).
# You can remove it from the dependencies if you are using this script outside of the library
# You can remove it from the dependencies if you are using this script outside of the library
# We've added it here for automated tests (see examples/test_examples.py file)
# We've added it here for automated tests (see examples/test_examples.py file)
...
...
templates/adding_a_new_example_script/utils_xxx.py
View file @
7fc628d9
...
@@ -21,6 +21,7 @@ import logging
...
@@ -21,6 +21,7 @@ import logging
import
math
import
math
from
transformers.tokenization_bert
import
BasicTokenizer
,
whitespace_tokenize
from
transformers.tokenization_bert
import
BasicTokenizer
,
whitespace_tokenize
# Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method)
# Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method)
from
utils_squad_evaluate
import
find_all_best_thresh_v2
,
get_raw_scores
,
make_qid_to_has_ans
from
utils_squad_evaluate
import
find_all_best_thresh_v2
,
get_raw_scores
,
make_qid_to_has_ans
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment