Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
1a948d70
"vscode:/vscode.git/clone" did not exist on "da1e4e53fcd52bc281bfecef2ca0c0f420caf38f"
Commit
1a948d70
authored
Dec 22, 2019
by
Aymeric Augustin
Browse files
Switch from comments to annotations for types.
parent
1c62e87b
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
9 additions
and
33 deletions
+9
-33
src/transformers/hf_api.py
src/transformers/hf_api.py
+9
-33
No files found.
src/transformers/hf_api.py
View file @
1a948d70
...
@@ -27,14 +27,7 @@ ENDPOINT = "https://huggingface.co"
...
@@ -27,14 +27,7 @@ ENDPOINT = "https://huggingface.co"
class
S3Obj
:
class
S3Obj
:
def
__init__
(
def
__init__
(
self
,
filename
:
str
,
LastModified
:
str
,
ETag
:
str
,
Size
:
int
,
**
kwargs
):
self
,
filename
,
# type: str
LastModified
,
# type: str
ETag
,
# type: str
Size
,
# type: int
**
kwargs
):
self
.
filename
=
filename
self
.
filename
=
filename
self
.
LastModified
=
LastModified
self
.
LastModified
=
LastModified
self
.
ETag
=
ETag
self
.
ETag
=
ETag
...
@@ -42,13 +35,7 @@ class S3Obj:
...
@@ -42,13 +35,7 @@ class S3Obj:
class
PresignedUrl
:
class
PresignedUrl
:
def
__init__
(
def
__init__
(
self
,
write
:
str
,
access
:
str
,
type
:
str
,
**
kwargs
):
self
,
write
,
# type: str
access
,
# type: str
type
,
# type: str
**
kwargs
):
self
.
write
=
write
self
.
write
=
write
self
.
access
=
access
self
.
access
=
access
self
.
type
=
type
# mime-type to send to S3.
self
.
type
=
type
# mime-type to send to S3.
...
@@ -58,12 +45,7 @@ class HfApi:
...
@@ -58,12 +45,7 @@ class HfApi:
def
__init__
(
self
,
endpoint
=
None
):
def
__init__
(
self
,
endpoint
=
None
):
self
.
endpoint
=
endpoint
if
endpoint
is
not
None
else
ENDPOINT
self
.
endpoint
=
endpoint
if
endpoint
is
not
None
else
ENDPOINT
def
login
(
def
login
(
self
,
username
:
str
,
password
:
str
)
->
str
:
self
,
username
,
# type: str
password
,
# type: str
):
# type: (...) -> str
"""
"""
Call HF API to sign in a user and get a token if credentials are valid.
Call HF API to sign in a user and get a token if credentials are valid.
...
@@ -79,10 +61,7 @@ class HfApi:
...
@@ -79,10 +61,7 @@ class HfApi:
d
=
r
.
json
()
d
=
r
.
json
()
return
d
[
"token"
]
return
d
[
"token"
]
def
whoami
(
def
whoami
(
self
,
token
:
str
)
->
str
:
self
,
token
,
# type: str
):
# type: (...) -> str
"""
"""
Call HF API to know "whoami"
Call HF API to know "whoami"
"""
"""
...
@@ -92,8 +71,7 @@ class HfApi:
...
@@ -92,8 +71,7 @@ class HfApi:
d
=
r
.
json
()
d
=
r
.
json
()
return
d
[
"user"
]
return
d
[
"user"
]
def
logout
(
self
,
token
):
def
logout
(
self
,
token
:
str
)
->
None
:
# type: (...) -> None
"""
"""
Call HF API to log out.
Call HF API to log out.
"""
"""
...
@@ -101,19 +79,17 @@ class HfApi:
...
@@ -101,19 +79,17 @@ class HfApi:
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)})
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)})
r
.
raise_for_status
()
r
.
raise_for_status
()
def
presign
(
self
,
token
,
filename
):
def
presign
(
self
,
token
:
str
,
filename
)
->
PresignedUrl
:
# type: (...) -> PresignedUrl
"""
"""
Call HF API to get a presigned url to upload `filename` to S3.
Call HF API to get a presigned url to upload `filename` to S3.
"""
"""
path
=
"{}/api/presign"
.
format
(
self
.
endpoint
)
path
=
"{}/api/presign"
.
format
(
self
.
endpoint
)
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)},
json
=
{
"filename"
:
filename
}
,
)
r
=
requests
.
post
(
path
,
headers
=
{
"authorization"
:
"Bearer {}"
.
format
(
token
)},
json
=
{
"filename"
:
filename
})
r
.
raise_for_status
()
r
.
raise_for_status
()
d
=
r
.
json
()
d
=
r
.
json
()
return
PresignedUrl
(
**
d
)
return
PresignedUrl
(
**
d
)
def
presign_and_upload
(
self
,
token
,
filename
,
filepath
):
def
presign_and_upload
(
self
,
token
:
str
,
filename
,
filepath
)
->
str
:
# type: (...) -> str
"""
"""
Get a presigned url, then upload file to S3.
Get a presigned url, then upload file to S3.
...
@@ -157,7 +133,7 @@ class TqdmProgressFileReader:
...
@@ -157,7 +133,7 @@ class TqdmProgressFileReader:
def
__init__
(
self
,
f
:
io
.
BufferedReader
):
def
__init__
(
self
,
f
:
io
.
BufferedReader
):
self
.
f
=
f
self
.
f
=
f
self
.
total_size
=
os
.
fstat
(
f
.
fileno
()).
st_size
# type: int
self
.
total_size
=
os
.
fstat
(
f
.
fileno
()).
st_size
self
.
pbar
=
tqdm
(
total
=
self
.
total_size
,
leave
=
False
)
self
.
pbar
=
tqdm
(
total
=
self
.
total_size
,
leave
=
False
)
self
.
read
=
f
.
read
self
.
read
=
f
.
read
f
.
read
=
self
.
_read
f
.
read
=
self
.
_read
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment