Unverified Commit cd40cb88 authored by Lysandre Debut's avatar Lysandre Debut Committed by GitHub
Browse files

Fix special token doc (#4292)

parent 82601f4c
...@@ -274,7 +274,7 @@ class AlbertTokenizer(PreTrainedTokenizer): ...@@ -274,7 +274,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
......
...@@ -269,7 +269,7 @@ class BertTokenizer(PreTrainedTokenizer): ...@@ -269,7 +269,7 @@ class BertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
......
...@@ -182,7 +182,7 @@ class CamembertTokenizer(PreTrainedTokenizer): ...@@ -182,7 +182,7 @@ class CamembertTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
if token_ids_1 is not None: if token_ids_1 is not None:
......
...@@ -193,7 +193,7 @@ class RobertaTokenizer(GPT2Tokenizer): ...@@ -193,7 +193,7 @@ class RobertaTokenizer(GPT2Tokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
if token_ids_1 is not None: if token_ids_1 is not None:
......
...@@ -893,7 +893,7 @@ class XLMTokenizer(PreTrainedTokenizer): ...@@ -893,7 +893,7 @@ class XLMTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
......
...@@ -217,7 +217,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer): ...@@ -217,7 +217,7 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
......
...@@ -278,7 +278,7 @@ class XLNetTokenizer(PreTrainedTokenizer): ...@@ -278,7 +278,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
Set to True if the token list is already formatted with special tokens for the model Set to True if the token list is already formatted with special tokens for the model
Returns: Returns:
:obj:`List[int]`: A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
""" """
if already_has_special_tokens: if already_has_special_tokens:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment