"docs/source/vscode:/vscode.git/clone" did not exist on "434022adac8a06f74a9adc4d110301c417d6424b"
Unverified Commit b8142753 authored by Leon Derczynski's avatar Leon Derczynski Committed by GitHub
Browse files

Add missing comment quotes (#17379)

parent e113c5cb
...@@ -144,14 +144,14 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(PreTrainedTokenizer): ...@@ -144,14 +144,14 @@ class {{cookiecutter.camelcase_modelname}}Tokenizer(PreTrainedTokenizer):
unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token
super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs) super().__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs)
"Initialisation" """ Initialisation """
@property @property
def vocab_size(self): def vocab_size(self):
"Returns vocab size" """ Returns vocab size """
def get_vocab(self): def get_vocab(self):
"Returns vocab as a dict" """ Returns vocab as a dict """
def _tokenize(self, text): def _tokenize(self, text):
""" Returns a tokenized string. """ """ Returns a tokenized string. """
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment