Unverified Commit b8142753 authored by Leon Derczynski's avatar Leon Derczynski Committed by GitHub
Browse files

Add missing comment quotes (#17379)

parent e113c5cb
......@@ -144,14 +144,14 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(PreTrainedTokenizer):
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
"Initialisation"
""" Initialisation """
@property
def vocab_size(self):
"Returns vocab size"
""" Returns vocab size """
def get_vocab(self):
"Returns vocab as a dict"
""" Returns vocab as a dict """
def _tokenize(self, text):
""" Returns a tokenized string. """
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment