Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
599db139
Unverified
Commit
599db139
authored
Dec 26, 2019
by
Anthony MOI
Browse files
Code style update
parent
835b76a4
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
4 additions
and
8 deletions
+4
-8
src/transformers/tokenization_bert.py
src/transformers/tokenization_bert.py
+2
-4
src/transformers/tokenization_gpt2.py
src/transformers/tokenization_gpt2.py
+2
-4
No files found.
src/transformers/tokenization_bert.py
View file @
599db139
...
@@ -22,7 +22,7 @@ import unicodedata
...
@@ -22,7 +22,7 @@ import unicodedata
import
tokenizers
as
tk
import
tokenizers
as
tk
from
.tokenization_utils
import
PreTrainedTokenizer
Fast
,
PreTrainedTokenizer
from
.tokenization_utils
import
PreTrainedTokenizer
,
PreTrainedTokenizer
Fast
logger
=
logging
.
getLogger
(
__name__
)
logger
=
logging
.
getLogger
(
__name__
)
...
@@ -583,9 +583,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
...
@@ -583,9 +583,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
)
)
)
)
if
max_length
is
not
None
:
if
max_length
is
not
None
:
self
.
_tokenizer
.
with_truncation
(
max_length
,
self
.
_tokenizer
.
with_truncation
(
max_length
,
stride
=
stride
,
strategy
=
truncation_strategy
)
stride
=
stride
,
strategy
=
truncation_strategy
)
self
.
_tokenizer
.
with_padding
(
self
.
_tokenizer
.
with_padding
(
max_length
=
max_length
if
pad_to_max_length
else
None
,
max_length
=
max_length
if
pad_to_max_length
else
None
,
direction
=
self
.
padding_side
,
direction
=
self
.
padding_side
,
...
...
src/transformers/tokenization_gpt2.py
View file @
599db139
...
@@ -23,7 +23,7 @@ from functools import lru_cache
...
@@ -23,7 +23,7 @@ from functools import lru_cache
import
regex
as
re
import
regex
as
re
import
tokenizers
as
tk
import
tokenizers
as
tk
from
.tokenization_utils
import
PreTrainedTokenizer
Fast
,
PreTrainedTokenizer
from
.tokenization_utils
import
PreTrainedTokenizer
,
PreTrainedTokenizer
Fast
logger
=
logging
.
getLogger
(
__name__
)
logger
=
logging
.
getLogger
(
__name__
)
...
@@ -277,9 +277,7 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
...
@@ -277,9 +277,7 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
self
.
_tokenizer
.
with_pre_tokenizer
(
tk
.
pre_tokenizers
.
ByteLevel
.
new
(
add_prefix_space
=
add_prefix_space
))
self
.
_tokenizer
.
with_pre_tokenizer
(
tk
.
pre_tokenizers
.
ByteLevel
.
new
(
add_prefix_space
=
add_prefix_space
))
self
.
_tokenizer
.
with_decoder
(
tk
.
decoders
.
ByteLevel
.
new
())
self
.
_tokenizer
.
with_decoder
(
tk
.
decoders
.
ByteLevel
.
new
())
if
max_length
:
if
max_length
:
self
.
_tokenizer
.
with_truncation
(
max_length
,
self
.
_tokenizer
.
with_truncation
(
max_length
,
stride
=
stride
,
strategy
=
truncation_strategy
)
stride
=
stride
,
strategy
=
truncation_strategy
)
self
.
_tokenizer
.
with_padding
(
self
.
_tokenizer
.
with_padding
(
max_length
=
max_length
if
pad_to_max_length
else
None
,
max_length
=
max_length
if
pad_to_max_length
else
None
,
direction
=
self
.
padding_side
,
direction
=
self
.
padding_side
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment