Unverified Commit ebfd7229 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Let inputs of fast tokenizers be tuples as well as lists (#19898)



* Let inputs of fast tokenizers be tuples as well as lists

* Update src/transformers/tokenization_utils_fast.py
Co-authored-by: default avatarLysandre Debut <lysandre.debut@reseau.eseo.fr>

* Style
Co-authored-by: default avatarLysandre Debut <lysandre.debut@reseau.eseo.fr>
parent 6c24443f
...@@ -412,8 +412,10 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase): ...@@ -412,8 +412,10 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
verbose: bool = True, verbose: bool = True,
) -> BatchEncoding: ) -> BatchEncoding:
if not isinstance(batch_text_or_text_pairs, list): if not isinstance(batch_text_or_text_pairs, (tuple, list)):
raise TypeError(f"batch_text_or_text_pairs has to be a list (got {type(batch_text_or_text_pairs)})") raise TypeError(
f"batch_text_or_text_pairs has to be a list or a tuple (got {type(batch_text_or_text_pairs)})"
)
# Set the truncation and padding strategy and restore the initial configuration # Set the truncation and padding strategy and restore the initial configuration
self.set_truncation_and_padding( self.set_truncation_and_padding(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment