Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
1a948d70
Commit
1a948d70
authored
Dec 22, 2019
by
Aymeric Augustin
Browse files
Switch from comments to annotations for types.
parent
1c62e87b
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
9 additions
and
33 deletions
+9
-33
src/transformers/hf_api.py
src/transformers/hf_api.py
+9
-33
No files found.
src/transformers/hf_api.py
View file @
1a948d70
...
@@ -27,14 +27,7 @@ ENDPOINT = "https://huggingface.co"
...
@@ -27,14 +27,7 @@ ENDPOINT = "https://huggingface.co"
class
S3Obj
:
class
S3Obj
:
def
__init__
(
def
__init__
(
self
,
filename
:
str
,
LastModified
:
str
,
ETag
:
str
,
Size
:
int
,
**
kwargs
):
self
,
filename
,
# type: str
LastModified
,
# type: str
ETag
,
# type: str
Size
,
# type: int
**
kwargs
):
self
.
filename
=
filename
self
.
filename
=
filename
self
.
LastModified
=
LastModified
self
.
LastModified
=
LastModified
self
.
ETag
=
ETag
self
.
ETag
=
ETag
...
@@ -42,13 +35,7 @@ class S3Obj:
...
@@ -42,13 +35,7 @@ class S3Obj:
class
PresignedUrl
:
class
PresignedUrl
:
def
__init__
(
def
__init__
(
self
,
write
:
str
,
access
:
str
,
type
:
str
,
**
kwargs
):
self
,
write
,
# type: str
access
,
# type: str
type
,
# type: str
**
kwargs
):
self
.
write
=
write
self
.
write
=
write
self
.
access
=
access
self
.
access
=
access
self
.
type
=
type
# mime-type to send to S3.
self
.
type
=
type
# mime-type to send to S3.
...
@@ -58,12 +45,7 @@ class HfApi:
...
@@ -58,12 +45,7 @@ class HfApi:
def
__init__
(
self
,
endpoint
=
None
):
def
__init__
(
self
,
endpoint
=
None
):
self
.
endpoint
=
endpoint
if
endpoint
is
not
None
else
ENDPOINT
self
.
endpoint
=
endpoint
if
endpoint
is
not
None
else
ENDPOINT
def
login
(
def
login
(
self
,
username
:
str
,
password
:
str
)
->
str
:
self
,
username
,
# type: str
password
,
# type: str
):
# type: (...) -> str
"""
"""
Call HF API to sign in a user and get a token if credentials are valid.
Call HF API to sign in a user and get a token if credentials are valid.
...
@@ -79,10 +61,7 @@ class HfApi:
...
@@ -79,10 +61,7 @@ class HfApi:
d
=
r
.
json
()
d
=
r
.
json
()
return
d
[
"token"
]
return
d
[
"token"
]
def
whoami
(
def
whoami
(
self
,
token
:
str
)
->
str
:
self
,
token
,
# type: str
):
# type: (...) -> str
"""
"""
Call HF API to know "whoami"
Call HF API to know "whoami"
"""
"""
...
@@ -92,8 +71,7 @@ class HfApi:
...
@@ -92,8 +71,7 @@ class HfApi:
d
=
r
.
json
()
d
=
r
.
json
()
return
d
[
"user"
]
return
d
[
"user"
]
def
logout
(
self
,
token
):
def
logout
(
self
,
token
:
str
)
->
None
:
# type: (...) -> None
"""
"""
Call HF API to log out.
Call HF API to log out.
"""
"""
...
@@ -101,19 +79,17 @@ class HfApi:
...
@@ -101,19 +79,17 @@ class HfApi:
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)})
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)})
r
.
raise_for_status
()
r
.
raise_for_status
()
def
presign
(
self
,
token
,
filename
):
def
presign
(
self
,
token
:
str
,
filename
)
->
PresignedUrl
:
# type: (...) -> PresignedUrl
"""
"""
Call HF API to get a presigned url to upload `filename` to S3.
Call HF API to get a presigned url to upload `filename` to S3.
"""
"""
path
=
"{}/api/presign"
.
format
(
self
.
endpoint
)
path
=
"{}/api/presign"
.
format
(
self
.
endpoint
)
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)},
json
=
{
"filename"
:
filename
}
,
)
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)},
json
=
{
"filename"
:
filename
})
r
.
raise_for_status
()
r
.
raise_for_status
()
d
=
r
.
json
()
d
=
r
.
json
()
return
PresignedUrl
(
**
d
)
return
PresignedUrl
(
**
d
)
def
presign_and_upload
(
self
,
token
,
filename
,
filepath
):
def
presign_and_upload
(
self
,
token
:
str
,
filename
,
filepath
)
->
str
:
# type: (...) -> str
"""
"""
Get a presigned url, then upload file to S3.
Get a presigned url, then upload file to S3.
...
@@ -157,7 +133,7 @@ class TqdmProgressFileReader:
...
@@ -157,7 +133,7 @@ class TqdmProgressFileReader:
def
__init__
(
self
,
f
:
io
.
BufferedReader
):
def
__init__
(
self
,
f
:
io
.
BufferedReader
):
self
.
f
=
f
self
.
f
=
f
self
.
total_size
=
os
.
fstat
(
f
.
fileno
()).
st_size
# type: int
self
.
total_size
=
os
.
fstat
(
f
.
fileno
()).
st_size
self
.
pbar
=
tqdm
(
total
=
self
.
total_size
,
leave
=
False
)
self
.
pbar
=
tqdm
(
total
=
self
.
total_size
,
leave
=
False
)
self
.
read
=
f
.
read
self
.
read
=
f
.
read
f
.
read
=
self
.
_read
f
.
read
=
self
.
_read
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment