Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
7959d835
Unverified
Commit
7959d835
authored
Apr 26, 2021
by
Sylvain Gugger
Committed by
GitHub
Apr 26, 2021
Browse files
Give each test a different repo name (#11453)
parent
b03b2a65
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
30 additions
and
26 deletions
+30
-26
tests/test_configuration_common.py
tests/test_configuration_common.py
+6
-6
tests/test_modeling_tf_common.py
tests/test_modeling_tf_common.py
+6
-6
tests/test_tokenization_common.py
tests/test_tokenization_common.py
+9
-7
tests/test_trainer.py
tests/test_trainer.py
+9
-7
No files found.
tests/test_configuration_common.py
View file @
7959d835
...
@@ -99,12 +99,12 @@ class ConfigPushToHubTester(unittest.TestCase):
...
@@ -99,12 +99,12 @@ class ConfigPushToHubTester(unittest.TestCase):
@
classmethod
@
classmethod
def
tearDownClass
(
cls
):
def
tearDownClass
(
cls
):
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
config
"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
config
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
...
@@ -113,9 +113,9 @@ class ConfigPushToHubTester(unittest.TestCase):
...
@@ -113,9 +113,9 @@ class ConfigPushToHubTester(unittest.TestCase):
vocab_size
=
99
,
hidden_size
=
32
,
num_hidden_layers
=
5
,
num_attention_heads
=
4
,
intermediate_size
=
37
vocab_size
=
99
,
hidden_size
=
32
,
num_hidden_layers
=
5
,
num_attention_heads
=
4
,
intermediate_size
=
37
)
)
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
model
"
,
use_auth_token
=
self
.
_token
)
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
config
"
,
use_auth_token
=
self
.
_token
)
new_config
=
BertConfig
.
from_pretrained
(
f
"
{
USER
}
/test-
model
"
)
new_config
=
BertConfig
.
from_pretrained
(
f
"
{
USER
}
/test-
config
"
)
for
k
,
v
in
config
.
__dict__
.
items
():
for
k
,
v
in
config
.
__dict__
.
items
():
if
k
!=
"transformers_version"
:
if
k
!=
"transformers_version"
:
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
...
@@ -129,12 +129,12 @@ class ConfigPushToHubTester(unittest.TestCase):
...
@@ -129,12 +129,12 @@ class ConfigPushToHubTester(unittest.TestCase):
config
.
save_pretrained
(
config
.
save_pretrained
(
tmp_dir
,
tmp_dir
,
push_to_hub
=
True
,
push_to_hub
=
True
,
repo_name
=
"test-
model
-org"
,
repo_name
=
"test-
config
-org"
,
use_auth_token
=
self
.
_token
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
organization
=
"valid_org"
,
)
)
new_config
=
BertConfig
.
from_pretrained
(
"valid_org/test-
model
-org"
)
new_config
=
BertConfig
.
from_pretrained
(
"valid_org/test-
config
-org"
)
for
k
,
v
in
config
.
__dict__
.
items
():
for
k
,
v
in
config
.
__dict__
.
items
():
if
k
!=
"transformers_version"
:
if
k
!=
"transformers_version"
:
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
tests/test_modeling_tf_common.py
View file @
7959d835
...
@@ -1357,12 +1357,12 @@ class TFModelPushToHubTester(unittest.TestCase):
...
@@ -1357,12 +1357,12 @@ class TFModelPushToHubTester(unittest.TestCase):
@
classmethod
@
classmethod
def
tearDownClass
(
cls
):
def
tearDownClass
(
cls
):
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model
-tf
"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model-
tf-
org"
,
organization
=
"valid_org"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
...
@@ -1374,9 +1374,9 @@ class TFModelPushToHubTester(unittest.TestCase):
...
@@ -1374,9 +1374,9 @@ class TFModelPushToHubTester(unittest.TestCase):
# Make sure model is properly initialized
# Make sure model is properly initialized
_
=
model
(
model
.
dummy_inputs
)
_
=
model
(
model
.
dummy_inputs
)
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model"
,
use_auth_token
=
self
.
_token
)
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model
-tf
"
,
use_auth_token
=
self
.
_token
)
new_model
=
TFBertModel
.
from_pretrained
(
f
"
{
USER
}
/test-model"
)
new_model
=
TFBertModel
.
from_pretrained
(
f
"
{
USER
}
/test-model
-tf
"
)
models_equal
=
True
models_equal
=
True
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
...
@@ -1392,12 +1392,12 @@ class TFModelPushToHubTester(unittest.TestCase):
...
@@ -1392,12 +1392,12 @@ class TFModelPushToHubTester(unittest.TestCase):
model
.
save_pretrained
(
model
.
save_pretrained
(
tmp_dir
,
tmp_dir
,
push_to_hub
=
True
,
push_to_hub
=
True
,
repo_name
=
"test-model-org"
,
repo_name
=
"test-model-
tf-
org"
,
use_auth_token
=
self
.
_token
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
organization
=
"valid_org"
,
)
)
new_model
=
TFBertModel
.
from_pretrained
(
"valid_org/test-model-org"
)
new_model
=
TFBertModel
.
from_pretrained
(
"valid_org/test-model-
tf-
org"
)
models_equal
=
True
models_equal
=
True
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
...
...
tests/test_tokenization_common.py
View file @
7959d835
...
@@ -2874,7 +2874,7 @@ class TokenizerTesterMixin:
...
@@ -2874,7 +2874,7 @@ class TokenizerTesterMixin:
@
is_staging_test
@
is_staging_test
class
Token
z
ierPushToHubTester
(
unittest
.
TestCase
):
class
Tokeni
z
erPushToHubTester
(
unittest
.
TestCase
):
vocab_tokens
=
[
"[UNK]"
,
"[CLS]"
,
"[SEP]"
,
"[PAD]"
,
"[MASK]"
,
"bla"
,
"blou"
]
vocab_tokens
=
[
"[UNK]"
,
"[CLS]"
,
"[SEP]"
,
"[PAD]"
,
"[MASK]"
,
"bla"
,
"blou"
]
@
classmethod
@
classmethod
...
@@ -2885,12 +2885,12 @@ class TokenzierPushToHubTester(unittest.TestCase):
...
@@ -2885,12 +2885,12 @@ class TokenzierPushToHubTester(unittest.TestCase):
@
classmethod
@
classmethod
def
tearDownClass
(
cls
):
def
tearDownClass
(
cls
):
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
tokenizer
"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
tokenizer
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
...
@@ -2900,9 +2900,11 @@ class TokenzierPushToHubTester(unittest.TestCase):
...
@@ -2900,9 +2900,11 @@ class TokenzierPushToHubTester(unittest.TestCase):
with
open
(
vocab_file
,
"w"
,
encoding
=
"utf-8"
)
as
vocab_writer
:
with
open
(
vocab_file
,
"w"
,
encoding
=
"utf-8"
)
as
vocab_writer
:
vocab_writer
.
write
(
""
.
join
([
x
+
"
\n
"
for
x
in
self
.
vocab_tokens
]))
vocab_writer
.
write
(
""
.
join
([
x
+
"
\n
"
for
x
in
self
.
vocab_tokens
]))
tokenizer
=
BertTokenizer
(
vocab_file
)
tokenizer
=
BertTokenizer
(
vocab_file
)
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model"
,
use_auth_token
=
self
.
_token
)
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-tokenizer"
,
use_auth_token
=
self
.
_token
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
f
"
{
USER
}
/test-
model
"
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
f
"
{
USER
}
/test-
tokenizer
"
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
def
test_push_to_hub_in_organization
(
self
):
def
test_push_to_hub_in_organization
(
self
):
...
@@ -2914,10 +2916,10 @@ class TokenzierPushToHubTester(unittest.TestCase):
...
@@ -2914,10 +2916,10 @@ class TokenzierPushToHubTester(unittest.TestCase):
tokenizer
.
save_pretrained
(
tokenizer
.
save_pretrained
(
tmp_dir
,
tmp_dir
,
push_to_hub
=
True
,
push_to_hub
=
True
,
repo_name
=
"test-
model
-org"
,
repo_name
=
"test-
tokenizer
-org"
,
use_auth_token
=
self
.
_token
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
organization
=
"valid_org"
,
)
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
"valid_org/test-
model
-org"
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
"valid_org/test-
tokenizer
-org"
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
tests/test_trainer.py
View file @
7959d835
...
@@ -1099,12 +1099,12 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
...
@@ -1099,12 +1099,12 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
@
classmethod
@
classmethod
def
tearDownClass
(
cls
):
def
tearDownClass
(
cls
):
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
trainer
"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
try
:
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
trainer
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
except
HTTPError
:
pass
pass
...
@@ -1112,14 +1112,14 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
...
@@ -1112,14 +1112,14 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
.
save_model
()
trainer
.
save_model
()
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-
model
"
,
use_auth_token
=
self
.
_token
)
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-
trainer
"
,
use_auth_token
=
self
.
_token
)
# Extract repo_name from the url
# Extract repo_name from the url
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
self
.
assertTrue
(
re_search
is
not
None
)
self
.
assertTrue
(
re_search
is
not
None
)
repo_name
=
re_search
.
groups
()[
0
]
repo_name
=
re_search
.
groups
()[
0
]
self
.
assertEqual
(
repo_name
,
f
"
{
USER
}
/test-
model
"
)
self
.
assertEqual
(
repo_name
,
f
"
{
USER
}
/test-
trainer
"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
repo_name
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
repo_name
)
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
...
@@ -1129,15 +1129,17 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
...
@@ -1129,15 +1129,17 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
.
save_model
()
trainer
.
save_model
()
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-model-org"
,
organization
=
"valid_org"
,
use_auth_token
=
self
.
_token
)
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-trainer-org"
,
organization
=
"valid_org"
,
use_auth_token
=
self
.
_token
)
# Extract repo_name from the url
# Extract repo_name from the url
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
self
.
assertTrue
(
re_search
is
not
None
)
self
.
assertTrue
(
re_search
is
not
None
)
repo_name
=
re_search
.
groups
()[
0
]
repo_name
=
re_search
.
groups
()[
0
]
self
.
assertEqual
(
repo_name
,
"valid_org/test-
model
-org"
)
self
.
assertEqual
(
repo_name
,
"valid_org/test-
trainer
-org"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
"valid_org/test-
model
-org"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
"valid_org/test-
trainer
-org"
)
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
self
.
assertEqual
(
model
.
b
.
item
(),
trainer
.
model
.
b
.
item
())
self
.
assertEqual
(
model
.
b
.
item
(),
trainer
.
model
.
b
.
item
())
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment