Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
7959d835
Unverified
Commit
7959d835
authored
Apr 26, 2021
by
Sylvain Gugger
Committed by
GitHub
Apr 26, 2021
Browse files
Give each test a different repo name (#11453)
parent
b03b2a65
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
30 additions
and
26 deletions
+30
-26
tests/test_configuration_common.py
tests/test_configuration_common.py
+6
-6
tests/test_modeling_tf_common.py
tests/test_modeling_tf_common.py
+6
-6
tests/test_tokenization_common.py
tests/test_tokenization_common.py
+9
-7
tests/test_trainer.py
tests/test_trainer.py
+9
-7
No files found.
tests/test_configuration_common.py
View file @
7959d835
...
...
@@ -99,12 +99,12 @@ class ConfigPushToHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
config
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
config
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -113,9 +113,9 @@ class ConfigPushToHubTester(unittest.TestCase):
vocab_size
=
99
,
hidden_size
=
32
,
num_hidden_layers
=
5
,
num_attention_heads
=
4
,
intermediate_size
=
37
)
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
model
"
,
use_auth_token
=
self
.
_token
)
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
config
"
,
use_auth_token
=
self
.
_token
)
new_config
=
BertConfig
.
from_pretrained
(
f
"
{
USER
}
/test-
model
"
)
new_config
=
BertConfig
.
from_pretrained
(
f
"
{
USER
}
/test-
config
"
)
for
k
,
v
in
config
.
__dict__
.
items
():
if
k
!=
"transformers_version"
:
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
...
...
@@ -129,12 +129,12 @@ class ConfigPushToHubTester(unittest.TestCase):
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
model
-org"
,
repo_name
=
"test-
config
-org"
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
)
new_config
=
BertConfig
.
from_pretrained
(
"valid_org/test-
model
-org"
)
new_config
=
BertConfig
.
from_pretrained
(
"valid_org/test-
config
-org"
)
for
k
,
v
in
config
.
__dict__
.
items
():
if
k
!=
"transformers_version"
:
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
tests/test_modeling_tf_common.py
View file @
7959d835
...
...
@@ -1357,12 +1357,12 @@ class TFModelPushToHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model
-tf
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model-
tf-
org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -1374,9 +1374,9 @@ class TFModelPushToHubTester(unittest.TestCase):
# Make sure model is properly initialized
_
=
model
(
model
.
dummy_inputs
)
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model"
,
use_auth_token
=
self
.
_token
)
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model
-tf
"
,
use_auth_token
=
self
.
_token
)
new_model
=
TFBertModel
.
from_pretrained
(
f
"
{
USER
}
/test-model"
)
new_model
=
TFBertModel
.
from_pretrained
(
f
"
{
USER
}
/test-model
-tf
"
)
models_equal
=
True
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
...
...
@@ -1392,12 +1392,12 @@ class TFModelPushToHubTester(unittest.TestCase):
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model-org"
,
repo_name
=
"test-model-
tf-
org"
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
)
new_model
=
TFBertModel
.
from_pretrained
(
"valid_org/test-model-org"
)
new_model
=
TFBertModel
.
from_pretrained
(
"valid_org/test-model-
tf-
org"
)
models_equal
=
True
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
...
...
tests/test_tokenization_common.py
View file @
7959d835
...
...
@@ -2874,7 +2874,7 @@ class TokenizerTesterMixin:
@
is_staging_test
class
Token
z
ierPushToHubTester
(
unittest
.
TestCase
):
class
Tokeni
z
erPushToHubTester
(
unittest
.
TestCase
):
vocab_tokens
=
[
"[UNK]"
,
"[CLS]"
,
"[SEP]"
,
"[PAD]"
,
"[MASK]"
,
"bla"
,
"blou"
]
@
classmethod
...
...
@@ -2885,12 +2885,12 @@ class TokenzierPushToHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
tokenizer
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
tokenizer
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -2900,9 +2900,11 @@ class TokenzierPushToHubTester(unittest.TestCase):
with
open
(
vocab_file
,
"w"
,
encoding
=
"utf-8"
)
as
vocab_writer
:
vocab_writer
.
write
(
""
.
join
([
x
+
"
\n
"
for
x
in
self
.
vocab_tokens
]))
tokenizer
=
BertTokenizer
(
vocab_file
)
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model"
,
use_auth_token
=
self
.
_token
)
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-tokenizer"
,
use_auth_token
=
self
.
_token
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
f
"
{
USER
}
/test-
model
"
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
f
"
{
USER
}
/test-
tokenizer
"
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
def
test_push_to_hub_in_organization
(
self
):
...
...
@@ -2914,10 +2916,10 @@ class TokenzierPushToHubTester(unittest.TestCase):
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
model
-org"
,
repo_name
=
"test-
tokenizer
-org"
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
"valid_org/test-
model
-org"
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
"valid_org/test-
tokenizer
-org"
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
tests/test_trainer.py
View file @
7959d835
...
...
@@ -1099,12 +1099,12 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
trainer
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
trainer
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -1112,14 +1112,14 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
.
save_model
()
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-
model
"
,
use_auth_token
=
self
.
_token
)
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-
trainer
"
,
use_auth_token
=
self
.
_token
)
# Extract repo_name from the url
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
self
.
assertTrue
(
re_search
is
not
None
)
repo_name
=
re_search
.
groups
()[
0
]
self
.
assertEqual
(
repo_name
,
f
"
{
USER
}
/test-
model
"
)
self
.
assertEqual
(
repo_name
,
f
"
{
USER
}
/test-
trainer
"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
repo_name
)
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
...
...
@@ -1129,15 +1129,17 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
.
save_model
()
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-model-org"
,
organization
=
"valid_org"
,
use_auth_token
=
self
.
_token
)
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-trainer-org"
,
organization
=
"valid_org"
,
use_auth_token
=
self
.
_token
)
# Extract repo_name from the url
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
self
.
assertTrue
(
re_search
is
not
None
)
repo_name
=
re_search
.
groups
()[
0
]
self
.
assertEqual
(
repo_name
,
"valid_org/test-
model
-org"
)
self
.
assertEqual
(
repo_name
,
"valid_org/test-
trainer
-org"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
"valid_org/test-
model
-org"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
"valid_org/test-
trainer
-org"
)
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
self
.
assertEqual
(
model
.
b
.
item
(),
trainer
.
model
.
b
.
item
())
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment