Unverified Commit 841f0715 authored by Yohei Tamura's avatar Yohei Tamura Committed by GitHub
Browse files

Add typing.overload for convert_ids_tokens (#6637)

* add overload for type checker

* black
parent 0f16dd0a
...@@ -20,7 +20,7 @@ import itertools ...@@ -20,7 +20,7 @@ import itertools
import logging import logging
import re import re
import unicodedata import unicodedata
from typing import Any, Dict, List, Optional, Tuple, Union from typing import Any, Dict, List, Optional, Tuple, Union, overload
from .file_utils import add_end_docstrings from .file_utils import add_end_docstrings
from .tokenization_utils_base import ( from .tokenization_utils_base import (
...@@ -657,6 +657,14 @@ class PreTrainedTokenizer(PreTrainedTokenizerBase): ...@@ -657,6 +657,14 @@ class PreTrainedTokenizer(PreTrainedTokenizerBase):
""" """
return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0)) return [0] * ((len(token_ids_1) if token_ids_1 else 0) + len(token_ids_0))
@overload
def convert_ids_to_tokens(self, ids: int, skip_special_tokens: bool = False) -> str:
...
@overload
def convert_ids_to_tokens(self, ids: List[int], skip_special_tokens: bool = False) -> List[str]:
...
def convert_ids_to_tokens( def convert_ids_to_tokens(
self, ids: Union[int, List[int]], skip_special_tokens: bool = False self, ids: Union[int, List[int]], skip_special_tokens: bool = False
) -> Union[str, List[str]]: ) -> Union[str, List[str]]:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment