Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
7959d835
"vscode:/vscode.git/clone" did not exist on "d311f87bcabb0485e7cd6baa78ebfa4e96f1f4ad"
Unverified
Commit
7959d835
authored
Apr 26, 2021
by
Sylvain Gugger
Committed by
GitHub
Apr 26, 2021
Browse files
Give each test a different repo name (#11453)
parent
b03b2a65
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
30 additions
and
26 deletions
+30
-26
tests/test_configuration_common.py
tests/test_configuration_common.py
+6
-6
tests/test_modeling_tf_common.py
tests/test_modeling_tf_common.py
+6
-6
tests/test_tokenization_common.py
tests/test_tokenization_common.py
+9
-7
tests/test_trainer.py
tests/test_trainer.py
+9
-7
No files found.
tests/test_configuration_common.py
View file @
7959d835
...
...
@@ -99,12 +99,12 @@ class ConfigPushToHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
config
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
config
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -113,9 +113,9 @@ class ConfigPushToHubTester(unittest.TestCase):
vocab_size
=
99
,
hidden_size
=
32
,
num_hidden_layers
=
5
,
num_attention_heads
=
4
,
intermediate_size
=
37
)
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
model
"
,
use_auth_token
=
self
.
_token
)
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
config
"
,
use_auth_token
=
self
.
_token
)
new_config
=
BertConfig
.
from_pretrained
(
f
"
{
USER
}
/test-
model
"
)
new_config
=
BertConfig
.
from_pretrained
(
f
"
{
USER
}
/test-
config
"
)
for
k
,
v
in
config
.
__dict__
.
items
():
if
k
!=
"transformers_version"
:
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
...
...
@@ -129,12 +129,12 @@ class ConfigPushToHubTester(unittest.TestCase):
config
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
model
-org"
,
repo_name
=
"test-
config
-org"
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
)
new_config
=
BertConfig
.
from_pretrained
(
"valid_org/test-
model
-org"
)
new_config
=
BertConfig
.
from_pretrained
(
"valid_org/test-
config
-org"
)
for
k
,
v
in
config
.
__dict__
.
items
():
if
k
!=
"transformers_version"
:
self
.
assertEqual
(
v
,
getattr
(
new_config
,
k
))
tests/test_modeling_tf_common.py
View file @
7959d835
...
...
@@ -1357,12 +1357,12 @@ class TFModelPushToHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model
-tf
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-model-
tf-
org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -1374,9 +1374,9 @@ class TFModelPushToHubTester(unittest.TestCase):
# Make sure model is properly initialized
_
=
model
(
model
.
dummy_inputs
)
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model"
,
use_auth_token
=
self
.
_token
)
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model
-tf
"
,
use_auth_token
=
self
.
_token
)
new_model
=
TFBertModel
.
from_pretrained
(
f
"
{
USER
}
/test-model"
)
new_model
=
TFBertModel
.
from_pretrained
(
f
"
{
USER
}
/test-model
-tf
"
)
models_equal
=
True
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
...
...
@@ -1392,12 +1392,12 @@ class TFModelPushToHubTester(unittest.TestCase):
model
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model-org"
,
repo_name
=
"test-model-
tf-
org"
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
)
new_model
=
TFBertModel
.
from_pretrained
(
"valid_org/test-model-org"
)
new_model
=
TFBertModel
.
from_pretrained
(
"valid_org/test-model-
tf-
org"
)
models_equal
=
True
for
p1
,
p2
in
zip
(
model
.
weights
,
new_model
.
weights
):
if
tf
.
math
.
reduce_sum
(
tf
.
math
.
abs
(
p1
-
p2
))
>
0
:
...
...
tests/test_tokenization_common.py
View file @
7959d835
...
...
@@ -2874,7 +2874,7 @@ class TokenizerTesterMixin:
@
is_staging_test
class
Token
z
ierPushToHubTester
(
unittest
.
TestCase
):
class
Tokeni
z
erPushToHubTester
(
unittest
.
TestCase
):
vocab_tokens
=
[
"[UNK]"
,
"[CLS]"
,
"[SEP]"
,
"[PAD]"
,
"[MASK]"
,
"bla"
,
"blou"
]
@
classmethod
...
...
@@ -2885,12 +2885,12 @@ class TokenzierPushToHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
tokenizer
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
tokenizer
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -2900,9 +2900,11 @@ class TokenzierPushToHubTester(unittest.TestCase):
with
open
(
vocab_file
,
"w"
,
encoding
=
"utf-8"
)
as
vocab_writer
:
vocab_writer
.
write
(
""
.
join
([
x
+
"
\n
"
for
x
in
self
.
vocab_tokens
]))
tokenizer
=
BertTokenizer
(
vocab_file
)
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-model"
,
use_auth_token
=
self
.
_token
)
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-tokenizer"
,
use_auth_token
=
self
.
_token
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
f
"
{
USER
}
/test-
model
"
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
f
"
{
USER
}
/test-
tokenizer
"
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
def
test_push_to_hub_in_organization
(
self
):
...
...
@@ -2914,10 +2916,10 @@ class TokenzierPushToHubTester(unittest.TestCase):
tokenizer
.
save_pretrained
(
tmp_dir
,
push_to_hub
=
True
,
repo_name
=
"test-
model
-org"
,
repo_name
=
"test-
tokenizer
-org"
,
use_auth_token
=
self
.
_token
,
organization
=
"valid_org"
,
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
"valid_org/test-
model
-org"
)
new_tokenizer
=
BertTokenizer
.
from_pretrained
(
"valid_org/test-
tokenizer
-org"
)
self
.
assertDictEqual
(
new_tokenizer
.
vocab
,
tokenizer
.
vocab
)
tests/test_trainer.py
View file @
7959d835
...
...
@@ -1099,12 +1099,12 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
@
classmethod
def
tearDownClass
(
cls
):
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
trainer
"
)
except
HTTPError
:
pass
try
:
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
model
-org"
,
organization
=
"valid_org"
)
cls
.
_api
.
delete_repo
(
token
=
cls
.
_token
,
name
=
"test-
trainer
-org"
,
organization
=
"valid_org"
)
except
HTTPError
:
pass
...
...
@@ -1112,14 +1112,14 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
.
save_model
()
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-
model
"
,
use_auth_token
=
self
.
_token
)
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-
trainer
"
,
use_auth_token
=
self
.
_token
)
# Extract repo_name from the url
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
self
.
assertTrue
(
re_search
is
not
None
)
repo_name
=
re_search
.
groups
()[
0
]
self
.
assertEqual
(
repo_name
,
f
"
{
USER
}
/test-
model
"
)
self
.
assertEqual
(
repo_name
,
f
"
{
USER
}
/test-
trainer
"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
repo_name
)
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
...
...
@@ -1129,15 +1129,17 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with
tempfile
.
TemporaryDirectory
()
as
tmp_dir
:
trainer
=
get_regression_trainer
(
output_dir
=
tmp_dir
)
trainer
.
save_model
()
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-model-org"
,
organization
=
"valid_org"
,
use_auth_token
=
self
.
_token
)
url
=
trainer
.
push_to_hub
(
repo_name
=
"test-trainer-org"
,
organization
=
"valid_org"
,
use_auth_token
=
self
.
_token
)
# Extract repo_name from the url
re_search
=
re
.
search
(
ENDPOINT_STAGING
+
r
"/([^/]+/[^/]+)/"
,
url
)
self
.
assertTrue
(
re_search
is
not
None
)
repo_name
=
re_search
.
groups
()[
0
]
self
.
assertEqual
(
repo_name
,
"valid_org/test-
model
-org"
)
self
.
assertEqual
(
repo_name
,
"valid_org/test-
trainer
-org"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
"valid_org/test-
model
-org"
)
model
=
RegressionPreTrainedModel
.
from_pretrained
(
"valid_org/test-
trainer
-org"
)
self
.
assertEqual
(
model
.
a
.
item
(),
trainer
.
model
.
a
.
item
())
self
.
assertEqual
(
model
.
b
.
item
(),
trainer
.
model
.
b
.
item
())
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment