"git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "054db06b1bf388798230a999a53af80ff8a2798a"
Unverified Commit a0dfc2d3 authored by cronoik's avatar cronoik Committed by GitHub
Browse files

fixes #10303 (#10304)

parent 9a7e6372
......@@ -3054,12 +3054,13 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
def convert_tokens_to_string(self, tokens: List[str]) -> str:
"""
Converts a sequence of token ids in a single string. The most simple way to do it is ``" ".join(tokens)`` but
we often want to remove sub-word tokenization artifacts at the same time
Converts a sequence of tokens in a single string. The most simple way to do it is ``" ".join(tokens)`` but
we often want to remove sub-word tokenization artifacts at the same time.
Args:
tokens (:obj:`List[str]`): The token to join in a string.
Return: The joined tokens.
Returns:
:obj:`str`: The joined tokens.
"""
raise NotImplementedError
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment