Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
chenpangpang
transformers
Commits
b8142753
"docs/source/vscode:/vscode.git/clone" did not exist on "434022adac8a06f74a9adc4d110301c417d6424b"
Unverified
Commit
b8142753
authored
Jun 29, 2022
by
Leon Derczynski
Committed by
GitHub
Jun 29, 2022
Browse files
Add missing comment quotes (#17379)
parent
e113c5cb
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
3 additions
and
3 deletions
+3
-3
templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/tokenization_{{cookiecutter.lowercase_modelname}}.py
...me}}/tokenization_{{cookiecutter.lowercase_modelname}}.py
+3
-3
No files found.
templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/tokenization_{{cookiecutter.lowercase_modelname}}.py
View file @
b8142753
...
@@ -144,14 +144,14 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(PreTrainedTokenizer):
...
@@ -144,14 +144,14 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(PreTrainedTokenizer):
unk_token
=
AddedToken
(
unk_token
,
lstrip
=
False
,
rstrip
=
False
)
if
isinstance
(
unk_token
,
str
)
else
unk_token
unk_token
=
AddedToken
(
unk_token
,
lstrip
=
False
,
rstrip
=
False
)
if
isinstance
(
unk_token
,
str
)
else
unk_token
super
().
__init__
(
bos_token
=
bos_token
,
eos_token
=
eos_token
,
unk_token
=
unk_token
,
**
kwargs
)
super
().
__init__
(
bos_token
=
bos_token
,
eos_token
=
eos_token
,
unk_token
=
unk_token
,
**
kwargs
)
"Initialisation"
"
""
Initialisation
""
"
@
property
@
property
def
vocab_size
(
self
):
def
vocab_size
(
self
):
"Returns vocab size"
"
""
Returns vocab size
""
"
def
get_vocab
(
self
):
def
get_vocab
(
self
):
"Returns vocab as a dict"
"
""
Returns vocab as a dict
""
"
def
_tokenize
(
self
,
text
):
def
_tokenize
(
self
,
text
):
""" Returns a tokenized string. """
""" Returns a tokenized string. """
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment