"archive-del/tools/bash-completion" did not exist on "f23f8a0688557e3ca3cf8bbf8e7669eab9912434"
Unverified Commit ebfd7229 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Let inputs of fast tokenizers be tuples as well as lists (#19898)



* Let inputs of fast tokenizers be tuples as well as lists

* Update src/transformers/tokenization_utils_fast.py
Co-authored-by: default avatarLysandre Debut <lysandre.debut@reseau.eseo.fr>

* Style
Co-authored-by: default avatarLysandre Debut <lysandre.debut@reseau.eseo.fr>
parent 6c24443f
......@@ -412,8 +412,10 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
verbose: bool = True,
) -> BatchEncoding:
if not isinstance(batch_text_or_text_pairs, list):
raise TypeError(f"batch_text_or_text_pairs has to be a list (got {type(batch_text_or_text_pairs)})")
if not isinstance(batch_text_or_text_pairs, (tuple, list)):
raise TypeError(
f"batch_text_or_text_pairs has to be a list or a tuple (got {type(batch_text_or_text_pairs)})"
)
# Set the truncation and padding strategy and restore the initial configuration
self.set_truncation_and_padding(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment